add tests

swh/lister/f_droid/lister.py

@@ -11,7 +11,7 @@ from swh.scheduler.model import ListedOrigin
 
 
 from ..pattern import CredentialsType, StatelessLister
-from datetime import datetime
+from datetime import datetime, timezone
 
 logger = logging.getLogger(__name__)
 
@@ -60,11 +60,16 @@ class FDroidLister(StatelessLister[FDroidListerPage]):
 
         for item in page:
             for id, value in item.items():
-                metadata = value.get("metaData", {})
+                metadata = value.get("metadata", {})
                 versions = value.get("versions", {})
-                last_updated = metadata.get("lastUpdated", None)
-                if last_updated is not None:
-                    last_updated = datetime.fromtimestamp(last_updated)
+                last_updated = (
+                    datetime.fromtimestamp(
+                        metadata.get("lastUpdated", None) / 1000,
+                        tz=timezone.utc,
+                    )
+                    if metadata.get("lastUpdated", None) is not None
+                    else None
+                )
 
                 yield ListedOrigin(
                     lister_id=self.lister_obj.id,

swh/lister/f_droid/tests/test_lister.py

@@ -0,0 +1,160 @@
+from swh.lister.f_droid.lister import FDroidLister
+from datetime import datetime, timezone
+
+expected_origins = [
+    {
+        "url": "https://f-droid.org/packages/An.stop",
+        "visit_type": "f_droid",
+        "extra_loader_arguments": {
+            "metadata": {
+                "added": 1309737600000,
+                "categories": ["Time"],
+                "issueTracker": "https://github.com/jdmonin/anstop/issues",
+                "lastUpdated": 1344556800000,
+                "license": "GPL-2.0-only",
+                "sourceCode": "https://github.com/jdmonin/anstop",
+                "name": {"en-US": "Anstop"},
+                "summary": {
+                    "ca": "Un simple cronòmetre",
+                    "cs": "Jednoduché stopky",
+                    "da": "Et simpelt stopur",
+                    "de": "Eine einfache Stoppuhr",
+                    "en-US": "A simple stopwatch",
+                    "eo": "Simpla temp-mezurilo",
+                    "es": "Un cronómetro simple",
+                    "fr": "Un chronomètre simple",
+                    "ga": "Stopuaireadóir simplí",
+                    "gd": "Stad-uaireadair sìmplidh",
+                    "he": "שעון עצר פשוט",
+                    "id": "Stopwatch",
+                    "is": "Einföld skeiðklukka",
+                    "it": "Un semplice cronometro",
+                    "ja": "質素なストップウォッチ",
+                    "nb": "Enkel stoppeklokke",
+                    "pl": "Prosty stoper",
+                    "pt": "Um simples cronômetro",
+                    "pt-BR": "Um simples cronômetro",
+                    "pt-PT": "Um simples cronômetro",
+                    "ro": "Un cronometru simplu",
+                    "ru": "Простой секундомер",
+                    "sq": "Një kronometër i thjeshtë",
+                    "sr": "Једноставна штоперица",
+                    "sw": "Saa sahili ya michezo",
+                    "ta": "ஒரு எளிய ச்டாப்வாட்ச்",
+                    "tr": "Basit bir süreölçer (kronometre)",
+                    "uk": "Простий секундомір",
+                    "zh-CN": "简单的秒表",
+                    "zh-TW": "簡單的秒錶",
+                },
+                "description": {
+                    "en-US": "A simple stopwatch, that also supports lap timing and a countdown timer. The\ncountdown timer doesn't make an alarm so an eye will have to be kept on it."
+                },
+                "icon": {
+                    "en-US": {
+                        "name": "/icons/An.stop.10.png",
+                        "sha256": "934816dacabfcf3db0e2e30243bb7605b17c01cedce28f114a9da9cbb97c5cd4",
+                        "size": 4488,
+                    }
+                },
+                "preferredSigner": "1a7974d7d08f0038ee40fbf44a9deeddd0fee8b2cf2df5f6449e869c75ec70fb",
+            },
+            "versions": {
+                "78ec7805f5a49b156fbd5f6af174c1cd8ae9900c9c7af2b2df021aca8cd5eae9": {
+                    "src": {
+                        "name": "/An.stop_10_src.tar.gz",
+                        "size": 558337,
+                        "sha256": "d489eee14c4693a4aa742c490f2566d2d17170a3977cc04993d96ba4588384c8",
+                    },
+                    "file": {
+                        "name": "/An.stop_10.apk",
+                        "size": 66218,
+                        "sha256": "78ec7805f5a49b156fbd5f6af174c1cd8ae9900c9c7af2b2df021aca8cd5eae9",
+                    },
+                    "added": 1344556800000,
+                    "manifest": {
+                        "signer": {
+                            "sha256": [
+                                "1a7974d7d08f0038ee40fbf44a9deeddd0fee8b2cf2df5f6449e869c75ec70fb"
+                            ]
+                        },
+                        "usesSdk": {"minSdkVersion": 4, "targetSdkVersion": 4},
+                        "versionCode": 10,
+                        "versionName": "1.5",
+                        "usesPermission": [
+                            {"name": "android.permission.VIBRATE"},
+                            {"name": "android.permission.WRITE_EXTERNAL_STORAGE"},
+                            {"name": "android.permission.READ_EXTERNAL_STORAGE"},
+                        ],
+                    },
+                },
+                "79f5253bab33cf4030b01fec457fd6ffa4fd54b631ee0bc4c1549fbb69ca6680": {
+                    "src": {
+                        "name": "/An.stop_9_src.tar.gz",
+                        "size": 63674,
+                        "sha256": "af6baad5820f1b86e8aeeec00bd3a46ad929dbae28dd3615e9ef94a555bd309f",
+                    },
+                    "file": {
+                        "name": "/An.stop_9.apk",
+                        "size": 49763,
+                        "sha256": "79f5253bab33cf4030b01fec457fd6ffa4fd54b631ee0bc4c1549fbb69ca6680",
+                    },
+                    "added": 1309737600000,
+                    "manifest": {
+                        "signer": {
+                            "sha256": [
+                                "1a7974d7d08f0038ee40fbf44a9deeddd0fee8b2cf2df5f6449e869c75ec70fb"
+                            ]
+                        },
+                        "usesSdk": {"minSdkVersion": 4, "targetSdkVersion": 4},
+                        "versionCode": 9,
+                        "versionName": "1.4",
+                        "usesPermission": [{"name": "android.permission.VIBRATE"}],
+                    },
+                },
+            },
+        },
+        "last_update": datetime.fromtimestamp(1344556800000 / 1000, tz=timezone.utc),
+    }
+]
+
+
+def test_fdroid_lister(datadir, requests_mock_datadir, swh_scheduler):
+    lister = FDroidLister(scheduler=swh_scheduler)
+    res = lister.run()
+
+    # based on the test data
+    assert res.pages == 1
+    assert res.origins == 3764
+
+    scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).results
+
+    # Check that all origins have the correct basic structure
+    for scheduled in scheduler_origins:
+        assert scheduled.visit_type == "f_droid"
+        assert scheduled.url.startswith("https://f-droid.org/packages/")
+        assert "metadata" in scheduled.extra_loader_arguments
+        assert "versions" in scheduled.extra_loader_arguments
+
+    # Check if all expected origins are found in the results
+    found_expected = []
+    for expected in expected_origins:
+        matched_origins = [o for o in scheduler_origins if o.url == expected["url"]]
+        assert len(matched_origins) == 1
+        matched = matched_origins[0]
+        found_expected.append(matched)
+
+        assert matched.visit_type == expected["visit_type"]
+        assert matched.last_update == expected["last_update"]
+        assert (
+            matched.extra_loader_arguments["metadata"]
+            == expected["extra_loader_arguments"]["metadata"]
+        )
+        assert (
+            matched.extra_loader_arguments["versions"]
+            == expected["extra_loader_arguments"]["versions"]
+        )
+
+    # Make sure all expected origins were found
+    assert len(found_expected) == len(expected_origins), (
+        f"Only found {len(found_expected)}/{len(expected_origins)} expected origins"
+    )

swh/lister/f_droid/tests/test_tasks.py

@@ -0,0 +1,33 @@
+# Copyright (C) 2022  The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+from swh.lister.pattern import ListerStats
+
+
+def test_f_droid_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker):
+    res = swh_scheduler_celery_app.send_task("swh.lister.f_droid.tasks.ping")
+    assert res
+    res.wait()
+    assert res.successful()
+    assert res.result == "OK"
+
+
+def test_f_droid_lister(swh_scheduler_celery_app, swh_scheduler_celery_worker, mocker):
+    # setup the mocked FDroidLister
+    lister = mocker.patch("swh.lister.f_droid.tasks.FDroidLister")
+    lister.from_configfile.return_value = lister
+    stats = ListerStats(pages=1, origins=3764)
+    lister.run.return_value = stats
+
+    res = swh_scheduler_celery_app.send_task(
+        "swh.lister.f_droid.tasks.FDroidListerTask"
+    )
+    assert res
+    res.wait()
+    assert res.successful()
+    assert res.result == stats.dict()
+
+    lister.from_configfile.assert_called_once_with()
+    lister.run.assert_called_once_with()