reonokiy/swh-lister
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

gnu.lister: Format timestamp to isoformat string for the tar loader

Browse source 
Related T2023
This commit is contained in:
Antoine R. Dumont (@ardumont) 2019-10-18 03:16:54 +02:00
parent 9fd648987e
commit 93e7afda3b
No known key found for this signature in database
GPG key ID: 52E2E9840D10C3B8
3 changed files with 41 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

8
swh/lister/gnu/tests/test_lister.py
View file

@ -40,4 +40,12 @@ def test_gnu_lister(swh_listers, requests_mock_datadir):
'url', 'length', 'time', 'filename', 'version'
}
for artifact in artifacts:
logger.debug(artifact)
# 'time' is an isoformat string now
for key in ['url', 'time', 'filename', 'version']:
assert isinstance(artifact[key], str)
assert isinstance(artifact['length'], int)
assert row['policy'] == 'oneshot'
assert row['priority'] is None

31
swh/lister/gnu/tests/test_tree.py
View file

@ -10,7 +10,7 @@ import pytest
from os import path
from swh.lister.gnu.tree import (
GNUTree, find_artifacts, check_filename_is_archive, load_raw_data,
get_version
get_version, format_date
)
@ -72,14 +72,14 @@ def test_tree_json(requests_mock_datadir):
{
'url': 'https://ftp.gnu.org/old-gnu/zlibc/zlibc-0.9b.tar.gz', # noqa
'length': 90106,
'time': 857980800,
'time': '1997-03-10T09:00:00',
'filename': 'zlibc-0.9b.tar.gz',
'version': '0.9b',
},
{
'url': 'https://ftp.gnu.org/old-gnu/zlibc/zlibc-0.9e.tar.gz', # noqa
'length': 89625,
'time': 860396400,
'time': '1997-04-07T09:00:00',
'filename': 'zlibc-0.9e.tar.gz',
'version': '0.9e',
}
@ -101,28 +101,28 @@ def test_find_artifacts_small_sample(datadir):
expected_artifacts = [
{
'url': '/root/artanis/artanis-0.2.1.tar.bz2',
'time': 1495205979,
'time': '2017-05-19T16:59:39',
'length': 424081,
'version': '0.2.1',
'filename': 'artanis-0.2.1.tar.bz2',
},
{
'url': '/root/xboard/winboard/winboard-4_0_0-src.zip', # noqa
'time': 898422900,
'time': '1998-06-21T11:55:00',
'length': 1514448,
'version': '4_0_0-src',
'filename': 'winboard-4_0_0-src.zip',
},
{
'url': '/root/xboard/xboard-3.6.2.tar.gz', # noqa
'time': 869814000,
'time': '1997-07-25T09:00:00',
'length': 450164,
'version': '3.6.2',
'filename': 'xboard-3.6.2.tar.gz',
},
{
'url': '/root/xboard/xboard-4.0.0.tar.gz', # noqa
'time': 898422900,
'time': '1998-06-21T11:55:00',
'length': 514951,
'version': '4.0.0',
'filename': 'xboard-4.0.0.tar.gz',
@ -204,3 +204,20 @@ def test_get_version():
actual_branchname = get_version(url)
assert actual_branchname == expected_branchname
def test_format_date():
for t_str, expected_isoformat_date in [
(1489817408, '2017-03-18T07:10:08'),
(1386961236, '2013-12-13T20:00:36'),
('1198900505', '2007-12-29T04:55:05'),
(1059822922, '2003-08-02T13:15:22'),
('1489817408', '2017-03-18T07:10:08'),
]:
actual_date = format_date(t_str)
assert actual_date == expected_isoformat_date
with pytest.raises(ValueError):
format_date('')
with pytest.raises(TypeError):
format_date(None)

10
swh/lister/gnu/tree.py
View file

@ -3,6 +3,7 @@
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import datetime
import gzip
import json
import logging
@ -133,7 +134,7 @@ def find_artifacts(
artifacts.append({
'url': uri,
'filename': filename,
'time': int(info_file['time']),
'time': format_date(info_file['time']),
'length': int(info_file['size']),
'version': get_version(filename),
})
@ -298,3 +299,10 @@ def load_raw_data(url: str) -> List[Dict]:
raw = f.read()
raw_data = json.loads(raw.decode('utf-8'))
return raw_data
def format_date(timestamp: str) -> str:
"""Format a string timestamp to an isoformat string
"""
return datetime.datetime.fromtimestamp(int(timestamp)).isoformat()