From c6372eea7e96474654b6b45e1bb71ca6200cbf79 Mon Sep 17 00:00:00 2001 From: "Antoine R. Dumont (@ardumont)" Date: Fri, 18 Oct 2019 03:18:32 +0200 Subject: [PATCH] gnu.lister: Unify timestamp formats to isoformat date in model Related T2023 --- requirements.txt | 1 + swh/lister/gnu/lister.py | 2 +- swh/lister/gnu/models.py | 4 ++-- swh/lister/gnu/tests/test_tree.py | 34 +++++++++++++++---------------- swh/lister/gnu/tree.py | 7 ++++--- 5 files changed, 25 insertions(+), 23 deletions(-) diff --git a/requirements.txt b/requirements.txt index 51e86f4..bf722f9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ setuptools xmltodict iso8601 beautifulsoup4 +pytz diff --git a/swh/lister/gnu/lister.py b/swh/lister/gnu/lister.py index 381c02c..5179462 100644 --- a/swh/lister/gnu/lister.py +++ b/swh/lister/gnu/lister.py @@ -97,6 +97,6 @@ class GNULister(SimpleLister): 'full_name': repo['name'], 'html_url': repo['url'], 'origin_url': repo['url'], - 'time_last_updated': int(repo['time_modified']), + 'time_last_updated': repo['time_modified'], 'origin_type': 'tar', } diff --git a/swh/lister/gnu/models.py b/swh/lister/gnu/models.py index 04b4a5e..bb78f85 100644 --- a/swh/lister/gnu/models.py +++ b/swh/lister/gnu/models.py @@ -2,7 +2,7 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from sqlalchemy import Column, String, Integer +from sqlalchemy import Column, DateTime, Integer, String from ..core.models import ModelBase @@ -14,4 +14,4 @@ class GNUModel(ModelBase): __tablename__ = 'gnu_repo' uid = Column(String, primary_key=True) - time_last_updated = Column(Integer) + time_last_updated = Column(DateTime) diff --git a/swh/lister/gnu/tests/test_tree.py b/swh/lister/gnu/tests/test_tree.py index 4137a74..0e6193b 100644 --- a/swh/lister/gnu/tests/test_tree.py +++ b/swh/lister/gnu/tests/test_tree.py @@ -46,25 +46,25 @@ def test_tree_json(requests_mock_datadir): assert tree_json.projects['https://ftp.gnu.org/gnu/8sync/'] == { 'name': '8sync', - 'time_modified': '1489817408', + 'time_modified': '2017-03-18T06:10:08+00:00', 'url': 'https://ftp.gnu.org/gnu/8sync/' } assert tree_json.projects['https://ftp.gnu.org/gnu/3dldf/'] == { 'name': '3dldf', - 'time_modified': '1386961236', + 'time_modified': '2013-12-13T19:00:36+00:00', 'url': 'https://ftp.gnu.org/gnu/3dldf/' } assert tree_json.projects['https://ftp.gnu.org/gnu/a2ps/'] == { 'name': 'a2ps', - 'time_modified': '1198900505', + 'time_modified': '2007-12-29T03:55:05+00:00', 'url': 'https://ftp.gnu.org/gnu/a2ps/' } assert tree_json.projects['https://ftp.gnu.org/old-gnu/xshogi/'] == { 'name': 'xshogi', - 'time_modified': '1059822922', + 'time_modified': '2003-08-02T11:15:22+00:00', 'url': 'https://ftp.gnu.org/old-gnu/xshogi/' } @@ -72,14 +72,14 @@ def test_tree_json(requests_mock_datadir): { 'url': 'https://ftp.gnu.org/old-gnu/zlibc/zlibc-0.9b.tar.gz', # noqa 'length': 90106, - 'time': '1997-03-10T09:00:00', + 'time': '1997-03-10T08:00:00+00:00', 'filename': 'zlibc-0.9b.tar.gz', 'version': '0.9b', }, { 'url': 'https://ftp.gnu.org/old-gnu/zlibc/zlibc-0.9e.tar.gz', # noqa 'length': 89625, - 'time': '1997-04-07T09:00:00', + 'time': '1997-04-07T07:00:00+00:00', 'filename': 'zlibc-0.9e.tar.gz', 'version': '0.9e', } @@ -101,28 +101,28 @@ def test_find_artifacts_small_sample(datadir): expected_artifacts = [ { 'url': '/root/artanis/artanis-0.2.1.tar.bz2', - 'time': '2017-05-19T16:59:39', + 'time': '2017-05-19T14:59:39+00:00', 'length': 424081, 'version': '0.2.1', 'filename': 'artanis-0.2.1.tar.bz2', }, { 'url': '/root/xboard/winboard/winboard-4_0_0-src.zip', # noqa - 'time': '1998-06-21T11:55:00', + 'time': '1998-06-21T09:55:00+00:00', 'length': 1514448, 'version': '4_0_0-src', 'filename': 'winboard-4_0_0-src.zip', }, { 'url': '/root/xboard/xboard-3.6.2.tar.gz', # noqa - 'time': '1997-07-25T09:00:00', + 'time': '1997-07-25T07:00:00+00:00', 'length': 450164, 'version': '3.6.2', 'filename': 'xboard-3.6.2.tar.gz', }, { 'url': '/root/xboard/xboard-4.0.0.tar.gz', # noqa - 'time': '1998-06-21T11:55:00', + 'time': '1998-06-21T09:55:00+00:00', 'length': 514951, 'version': '4.0.0', 'filename': 'xboard-4.0.0.tar.gz', @@ -207,14 +207,14 @@ def test_get_version(): def test_format_date(): - for t_str, expected_isoformat_date in [ - (1489817408, '2017-03-18T07:10:08'), - (1386961236, '2013-12-13T20:00:36'), - ('1198900505', '2007-12-29T04:55:05'), - (1059822922, '2003-08-02T13:15:22'), - ('1489817408', '2017-03-18T07:10:08'), + for timestamp, expected_isoformat_date in [ + (1489817408, '2017-03-18T06:10:08+00:00'), + (1386961236, '2013-12-13T19:00:36+00:00'), + ('1198900505', '2007-12-29T03:55:05+00:00'), + (1059822922, '2003-08-02T11:15:22+00:00'), + ('1489817408', '2017-03-18T06:10:08+00:00'), ]: - actual_date = format_date(t_str) + actual_date = format_date(timestamp) assert actual_date == expected_isoformat_date with pytest.raises(ValueError): diff --git a/swh/lister/gnu/tree.py b/swh/lister/gnu/tree.py index b47e1c5..5a4991f 100644 --- a/swh/lister/gnu/tree.py +++ b/swh/lister/gnu/tree.py @@ -3,15 +3,16 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -import datetime import gzip import json import logging import requests import re +from datetime import datetime from os import path from pathlib import Path +from pytz import utc from typing import Any, Dict, List, Mapping, Tuple from urllib.parse import urlparse @@ -72,7 +73,7 @@ class GNUTree: repo_details = { 'name': info['name'], 'url': package_url, - 'time_modified': info['time'], + 'time_modified': format_date(info['time']) } artifacts[package_url] = package_artifacts projects[package_url] = repo_details @@ -305,4 +306,4 @@ def format_date(timestamp: str) -> str: """Format a string timestamp to an isoformat string """ - return datetime.datetime.fromtimestamp(int(timestamp)).isoformat() + return datetime.fromtimestamp(int(timestamp), tz=utc).isoformat()