maven: Fix undef last_update in ListedOrigins.

swh/lister/maven/lister.py

@@ -4,6 +4,7 @@
 # See top-level LICENSE file for more information
 
 from dataclasses import asdict, dataclass
+from datetime import datetime, timezone
 import logging
 import re
 from typing import Any, Dict, Iterator, Optional
@@ -298,14 +299,25 @@ class MavenLister(Lister[MavenListerState, RepoPage]):
                     yield origin
         else:
             # Origin is a source archive:
+            last_update_dt = None
+            last_update_iso = ""
+            last_update_seconds = str(page["time"])[:-3]
+            try:
+                last_update_dt = datetime.fromtimestamp(int(last_update_seconds))
+                last_update_dt_tz = last_update_dt.astimezone(timezone.utc)
+            except OverflowError:
+                logger.warning("- Failed to convert datetime %s.", last_update_seconds)
+            if last_update_dt:
+                last_update_iso = last_update_dt_tz.isoformat()
             origin = ListedOrigin(
                 lister_id=self.lister_obj.id,
                 url=page["url"],
                 visit_type=page["type"],
+                last_update=last_update_dt,
                 extra_loader_arguments={
                     "artifacts": [
                         {
-                            "time": page["time"],
+                            "time": last_update_iso,
                             "gid": page["gid"],
                             "aid": page["aid"],
                             "version": page["version"],

swh/lister/maven/tests/test_lister.py

@@ -3,8 +3,10 @@
 # License: GNU General Public License version 3, or any later version
 # See top-level LICENSE file for more information
 
+from datetime import timezone
 from pathlib import Path
 
+import iso8601
 import pytest
 import requests
 
@@ -34,7 +36,7 @@ LIST_SRC_DATA = (
         "type": "maven",
         "url": "https://repo1.maven.org/maven2/al/aldi/sprova4j"
         + "/0.1.0/sprova4j-0.1.0-sources.jar",
-        "time": 1626109619335,
+        "time": "2021-07-12T17:06:59+00:00",
         "gid": "al.aldi",
         "aid": "sprova4j",
         "version": "0.1.0",
@@ -43,7 +45,7 @@ LIST_SRC_DATA = (
         "type": "maven",
         "url": "https://repo1.maven.org/maven2/al/aldi/sprova4j"
         + "/0.1.1/sprova4j-0.1.1-sources.jar",
-        "time": 1626111425534,
+        "time": "2021-07-12T17:37:05+00:00",
         "gid": "al.aldi",
         "aid": "sprova4j",
         "version": "0.1.1",
@@ -53,38 +55,32 @@ LIST_SRC_DATA = (
 
 @pytest.fixture
 def maven_index(datadir) -> str:
-    text = Path(datadir, "http_indexes", "export.fld").read_text()
-    return text
+    return Path(datadir, "http_indexes", "export.fld").read_text()
 
 
 @pytest.fixture
 def maven_index_incr(datadir) -> str:
-    text = Path(datadir, "http_indexes", "export_incr.fld").read_text()
-    return text
+    return Path(datadir, "http_indexes", "export_incr.fld").read_text()
 
 
 @pytest.fixture
 def maven_pom_1(datadir) -> str:
-    text = Path(datadir, "https_maven.org", "sprova4j-0.1.0.pom").read_text()
-    return text
+    return Path(datadir, "https_maven.org", "sprova4j-0.1.0.pom").read_text()
 
 
 @pytest.fixture
 def maven_pom_1_malformed(datadir) -> str:
-    text = Path(datadir, "https_maven.org", "sprova4j-0.1.0.malformed.pom").read_text()
-    return text
+    return Path(datadir, "https_maven.org", "sprova4j-0.1.0.malformed.pom").read_text()
 
 
 @pytest.fixture
 def maven_pom_2(datadir) -> str:
-    text = Path(datadir, "https_maven.org", "sprova4j-0.1.1.pom").read_text()
-    return text
+    return Path(datadir, "https_maven.org", "sprova4j-0.1.1.pom").read_text()
 
 
 @pytest.fixture
 def maven_pom_3(datadir) -> str:
-    text = Path(datadir, "https_maven.org", "arangodb-graphql-1.2.pom").read_text()
-    return text
+    return Path(datadir, "https_maven.org", "arangodb-graphql-1.2.pom").read_text()
 
 
 def test_maven_full_listing(
@@ -123,6 +119,10 @@ def test_maven_full_listing(
         if origin.visit_type == "maven":
             for src in LIST_SRC_DATA:
                 if src.get("url") == origin.url:
+                    last_update_src = iso8601.parse_date(src.get("time")).astimezone(
+                        tz=timezone.utc
+                    )
+                    assert last_update_src == origin.last_update
                     artifact = origin.extra_loader_arguments["artifacts"][0]
                     assert src.get("time") == artifact["time"]
                     assert src.get("gid") == artifact["gid"]
@@ -131,7 +131,9 @@ def test_maven_full_listing(
                     assert MVN_URL == artifact["base_url"]
                     break
             else:
-                raise AssertionError
+                raise AssertionError(
+                    "Could not find scheduler origin in referenced origins."
+                )
     scheduler_state = lister.get_state_from_scheduler()
     assert scheduler_state is not None
     assert scheduler_state.last_seen_doc == -1
@@ -188,7 +190,9 @@ def test_maven_full_listing_malformed(
                     assert MVN_URL == artifact["base_url"]
                     break
             else:
-                raise AssertionError
+                raise AssertionError(
+                    "Could not find scheduler origin in referenced origins."
+                )
     scheduler_state = lister.get_state_from_scheduler()
     assert scheduler_state is not None
     assert scheduler_state.last_seen_doc == -1