reonokiy/swh-lister
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

npm: Align lister's loader output tasks with expected format

Browse source 
Related to T2134
This commit is contained in:
Antoine R. Dumont (@ardumont) 2019-12-06 16:53:26 +01:00
parent 4a9608f31c
commit 5d096d511c
No known key found for this signature in database
GPG key ID: 52E2E9840D10C3B8
2 changed files with 7 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

22
swh/lister/npm/lister.py
View file

@ -2,8 +2,6 @@
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from urllib.parse import quote
from swh.lister.core.indexing_lister import IndexingHttpLister
from swh.lister.npm.models import NpmModel
from swh.scheduler.utils import create_task_dict
@ -36,13 +34,13 @@ class NpmListerBase(IndexingHttpLister):
"""(Override) Transform from npm package name to model
"""
package_url, package_metadata_url = self._compute_urls(repo_name)
package_url = 'https://www.npmjs.com/package/%s' % repo_name
return {
'uid': repo_name,
'indexable': repo_name,
'name': repo_name,
'full_name': repo_name,
'html_url': package_metadata_url,
'html_url': package_url,
'origin_url': package_url,
'origin_type': 'npm',
}
@ -57,12 +55,8 @@ class NpmListerBase(IndexingHttpLister):
"""
task_type = 'load-%s' % origin_type
task_policy = self.config['loading_task_policy']
package_name = kwargs.get('name')
package_metadata_url = kwargs.get('html_url')
return create_task_dict(task_type, task_policy,
package_name=package_name,
package_url=origin_url,
package_metadata_url=package_metadata_url)
url=origin_url)
def request_headers(self):
"""(Override) Set requests headers to send when querying the npm
@ -73,16 +67,6 @@ class NpmListerBase(IndexingHttpLister):
headers['Accept'] = 'application/json'
return headers
def _compute_urls(self, repo_name):
"""Return a tuple (package_url, package_metadata_url)
"""
return (
'https://www.npmjs.com/package/%s' % repo_name,
# package metadata url needs to be escaped otherwise some requests
# may fail (for instance when a package name contains '/')
'%s/%s' % (self.url, quote(repo_name, safe=''))
)
def string_pattern_check(self, inner, lower, upper=None):
""" (Override) Inhibit the effect of that method as packages indices
correspond to package names and thus do not respect any kind

10
swh/lister/npm/tests/test_lister.py
View file

@ -66,12 +66,10 @@ def check_tasks(tasks: List[Any]):
# kwargs
kwargs = row['arguments']['kwargs']
assert len(kwargs) == 3
package_name = kwargs['package_name']
package_url = kwargs['package_url']
assert package_url == 'https://www.npmjs.com/package/%s' % package_name
meta_url = kwargs['package_metadata_url']
assert meta_url == 'https://replicate.npmjs.com/%s' % package_name
assert len(kwargs) == 1
package_url = kwargs['url']
package_name = package_url.split('/')[-1]
assert package_url == f'https://www.npmjs.com/package/{package_name}'
assert row['policy'] == 'recurring'
assert row['priority'] is None