reonokiy/swh-lister
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

swh.lister.npm: Add an incremental npm lister

Browse source 
This new lister enables to get only new or updated npm packages
since the last listing operation.

Related T1378
Closes T1398
This commit is contained in:
Antoine Lambert 2018-11-30 14:45:22 +01:00
parent 605a67f51d
commit ffe4ac9a3c
7 changed files with 1106 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

109
swh/lister/npm/lister.py
View file

@ -9,20 +9,28 @@ from swh.lister.npm.models import NpmModel
from swh.scheduler.utils import create_task_dict
class NpmLister(SWHIndexingHttpLister):
"""List all packages available in the npm registry in a paginated way.
class NpmListerBase(SWHIndexingHttpLister):
"""List packages available in the npm registry in a paginated way
"""
PATH_TEMPLATE = '/_all_docs?startkey="%s"'
MODEL = NpmModel
LISTER_NAME = 'npm'
def __init__(self, api_baseurl='https://replicate.npmjs.com',
per_page=10000, override_config=None):
per_page=1000, override_config=None):
super().__init__(api_baseurl=api_baseurl,
override_config=override_config)
self.per_page = per_page + 1
self.PATH_TEMPLATE += '&limit=%s' % self.per_page
@property
def ADDITIONAL_CONFIG(self):
"""(Override) Add extra configuration
"""
default_config = super().ADDITIONAL_CONFIG
default_config['loading_task_policy'] = ('str', 'recurring')
return default_config
def get_model_from_repo(self, repo_name):
"""(Override) Transform from npm package name to model
@ -46,29 +54,14 @@ class NpmLister(SWHIndexingHttpLister):
needed for the ingestion task creation.
"""
_type = 'origin-update-%s' % origin_type
_policy = 'recurring'
task_type = 'origin-update-%s' % origin_type
task_policy = self.config['loading_task_policy']
package_name = kwargs.get('name')
package_metadata_url = kwargs.get('html_url')
return create_task_dict(_type, _policy, package_name, origin_url,
return create_task_dict(task_type, task_policy,
package_name, origin_url,
package_metadata_url=package_metadata_url)
def get_next_target_from_response(self, response):
"""(Override) Get next npm package name to continue the listing
"""
repos = response.json()['rows']
return repos[-1]['id'] if len(repos) == self.per_page else None
def transport_response_simplified(self, response):
"""(Override) Transform npm registry response to list for model manipulation
"""
repos = response.json()['rows']
if len(repos) == self.per_page:
repos = repos[:-1]
return [self.get_model_from_repo(repo['id']) for repo in repos]
def request_headers(self):
"""(Override) Set requests headers to send when querying the npm registry
@ -92,3 +85,73 @@ class NpmLister(SWHIndexingHttpLister):
of fixed length string pattern
"""
pass
class NpmLister(NpmListerBase):
"""List all packages available in the npm registry in a paginated way
"""
PATH_TEMPLATE = '/_all_docs?startkey="%s"'
def get_next_target_from_response(self, response):
"""(Override) Get next npm package name to continue the listing
"""
repos = response.json()['rows']
return repos[-1]['id'] if len(repos) == self.per_page else None
def transport_response_simplified(self, response):
"""(Override) Transform npm registry response to list for model manipulation
"""
repos = response.json()['rows']
if len(repos) == self.per_page:
repos = repos[:-1]
return [self.get_model_from_repo(repo['id']) for repo in repos]
class NpmIncrementalLister(NpmListerBase):
"""List packages in the npm registry, updated since a specific
update_seq value of the underlying CouchDB database, in a paginated way
"""
PATH_TEMPLATE = '/_changes?since=%s'
@property
def CONFIG_BASE_FILENAME(self): # noqa: N802
return 'lister-npm-incremental'
def get_next_target_from_response(self, response):
"""(Override) Get next npm package name to continue the listing
"""
repos = response.json()['results']
return repos[-1]['seq'] if len(repos) == self.per_page else None
def transport_response_simplified(self, response):
"""(Override) Transform npm registry response to list for model manipulation
"""
repos = response.json()['results']
if len(repos) == self.per_page:
repos = repos[:-1]
return [self.get_model_from_repo(repo['id']) for repo in repos]
def filter_before_inject(self, models_list):
"""(Override) Filter out documents in the CouchDB database
not related to a npm package
"""
models_filtered = []
for model in models_list:
package_name = model['name']
# document related to CouchDB internals
if package_name.startswith('_design/'):
continue
models_filtered.append(model)
return models_filtered
def disable_deleted_repo_tasks(self, start, end, keep_these):
"""(Override) Disable the processing performed by that method
as it is not relevant in this incremental lister context
and it raises and exception due to a different index type
(int instead of str)
"""
pass

27
swh/lister/npm/models.py
View file

@ -2,13 +2,34 @@
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from sqlalchemy import Column, String
from sqlalchemy import Column, String, DateTime, Integer, BigInteger, Sequence
from sqlalchemy.ext.declarative import declarative_base
from swh.lister.core.models import IndexingModelBase
from swh.lister.core.models import IndexingModelBase, ABCSQLMeta
SQLBase = declarative_base()
class NpmVisitModel(SQLBase, metaclass=ABCSQLMeta):
"""Table to store the npm registry state at the time of a
content listing by Software Heritage
"""
__tablename__ = 'npm_visit'
uid = Column(Integer, Sequence('npm_visit_id_seq'), primary_key=True)
visit_date = Column(DateTime, nullable=False)
doc_count = Column(BigInteger)
doc_del_count = Column(BigInteger)
update_seq = Column(BigInteger)
purge_seq = Column(BigInteger)
disk_size = Column(BigInteger)
data_size = Column(BigInteger)
committed_update_seq = Column(BigInteger)
compacted_seq = Column(BigInteger)
class NpmModel(IndexingModelBase):
"""a npm repository representation
"""A npm package representation
"""
__tablename__ = 'npm_repo'

69
swh/lister/npm/tasks.py
View file

@ -2,19 +2,76 @@
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from datetime import datetime
from swh.lister.core.tasks import ListerTaskBase
from swh.lister.npm.lister import NpmLister
from swh.lister.npm.lister import NpmLister, NpmIncrementalLister
from swh.lister.npm.models import NpmVisitModel
class NpmListerTask(ListerTaskBase):
"""Full npm lister (list all available packages from the npm registry).
class _NpmListerTaskBase(ListerTaskBase):
"""
task_queue = 'swh_lister_npm_refresh'
def _save_registry_state(self):
"""Query the root endpoint from the npm registry and
backup values of interest for future listing
"""
params = {'headers': self.lister.request_headers()}
registry_state = \
self.lister.session.get(self.lister.api_baseurl, **params)
registry_state = registry_state.json()
self.registry_state = {
'visit_date': datetime.now(),
}
for key in ('doc_count', 'doc_del_count', 'update_seq', 'purge_seq',
'disk_size', 'data_size', 'committed_update_seq',
'compacted_seq'):
self.registry_state[key] = registry_state[key]
def _store_registry_state(self):
"""Store the backup npm registry state to database.
"""
npm_visit = NpmVisitModel(**self.registry_state)
self.lister.db_session.add(npm_visit)
self.lister.db_session.commit()
class NpmListerTask(_NpmListerTaskBase):
"""Full npm lister (list all available packages from the npm registry)
"""
def new_lister(self):
return NpmLister()
def run_task(self):
lister = self.new_lister()
lister.run()
self.lister = self.new_lister()
self._save_registry_state()
self.lister.run()
self._store_registry_state()
class NpmIncrementalListerTask(_NpmListerTaskBase):
"""Incremental npm lister (list all updated packages since the last listing)
"""
def new_lister(self):
return NpmIncrementalLister()
def run_task(self):
self.lister = self.new_lister()
update_seq_start = self._get_last_update_seq()
self._save_registry_state()
self.lister.run(min_bound=update_seq_start)
self._store_registry_state()
def _get_last_update_seq(self):
"""Get latest ``update_seq`` value for listing only updated packages.
"""
query = self.lister.db_session.query(NpmVisitModel.update_seq)
row = query.order_by(NpmVisitModel.uid.desc()).first()
if not row:
raise ValueError('No npm registry listing previously performed ! '
'This is required prior to the execution of an '
'incremental listing.')
return row[0]

4
swh/lister/npm/tests/api_inc_empty_response.json Normal file
View file

@ -0,0 +1,4 @@
{
"results": [],
"last_seq": 6927821
}

906
swh/lister/npm/tests/api_inc_response.json Normal file
View file

@ -0,0 +1,906 @@
{
"results": [
{
"seq": 6920644,
"id": "electron-scripts",
"changes": [
{
"rev": "3-a19944df5a3636bb225af9e0c8f9eedc"
}
]
},
{
"seq": 6920649,
"id": "@crexi-dev/schematics",
"changes": [
{
"rev": "3-00188360eeca1f9123b2d7cd4b468c50"
}
]
},
{
"seq": 6920651,
"id": "botfactory-conversation",
"changes": [
{
"rev": "50-f3667cde87637505528c46adc87f44e3"
}
]
},
{
"seq": 6920667,
"id": "castle",
"changes": [
{
"rev": "3-d9adf9c9fd687cdaa2bf460c5bb523f0"
}
]
},
{
"seq": 6920671,
"id": "rbc-wm-framework-vuejs",
"changes": [
{
"rev": "111-32bed479afacdd88aed9ac16dd135843"
}
]
},
{
"seq": 6920678,
"id": "bitcoinfiles",
"changes": [
{
"rev": "22-ab3cd6b46f84d9aac1a24560cabdc9f0"
}
]
},
{
"seq": 6920679,
"id": "jovo-core",
"changes": [
{
"rev": "2-d7440f1d17823e1a0760d9b3d4537c6e"
}
]
},
{
"seq": 6920687,
"id": "jovo-framework",
"changes": [
{
"rev": "103-e4f46a3530514c2ee81a97d25fc8c8c9"
}
]
},
{
"seq": 6920690,
"id": "smart-form-lib",
"changes": [
{
"rev": "18-3b6b6b2b0ea2e114a3f1335a8e798ade"
}
]
},
{
"seq": 6920694,
"id": "bokehjs",
"changes": [
{
"rev": "18-115ce2d4bf4f281eb50c25f3203b3dd2"
}
]
},
{
"seq": 6920701,
"id": "guijarro",
"changes": [
{
"rev": "14-82ece581d6a35d4e1d78e5292ca245c0"
}
]
},
{
"seq": 6920702,
"id": "@kava-labs/crypto-rate-utils",
"changes": [
{
"rev": "3-cecc6a6c226a0590b1a685e3041028c6"
}
]
},
{
"seq": 6920703,
"id": "@riouxjean/test",
"changes": [
{
"rev": "10-01e97dc7d0241dc49ea93b3468ec7b29"
}
]
},
{
"seq": 6920704,
"id": "react-scrabblefy",
"changes": [
{
"rev": "7-970c8206f3b8744204f7dcb106f8462b"
}
]
},
{
"seq": 6920706,
"id": "molart",
"changes": [
{
"rev": "14-416cd3cec62dd46f9b59a3bbe35308f6"
}
]
},
{
"seq": 6920707,
"id": "@universal-material/angular",
"changes": [
{
"rev": "32-266ed3f67e1ddd0b4a37ca29f1cf5bf3"
}
]
},
{
"seq": 6920708,
"id": "cozy-doctypes",
"changes": [
{
"rev": "68-8e90cc26e25da6c9430d373e43ac3c25"
}
]
},
{
"seq": 6920710,
"id": "2o3t-ui",
"changes": [
{
"rev": "96-1e65d5320ea7c78525aba5daf328bd4b"
}
]
},
{
"seq": 6920712,
"id": "ark-ts",
"changes": [
{
"rev": "24-033183c2f7f9cbb6e44d553213e525b6"
}
]
},
{
"seq": 6920715,
"id": "mysqlconnector",
"changes": [
{
"rev": "19-f09bc0b82281ca486db5ebe83843679e"
}
]
},
{
"seq": 6920716,
"id": "@innovexa/ng-form-creator-lib",
"changes": [
{
"rev": "147-480665ee17fa889dfec1aee75b907ff2"
}
]
},
{
"seq": 6920717,
"id": "k-routes-example-basic",
"changes": [
{
"rev": "1-35142059e1c63cc724da71a9eebf229c"
}
]
},
{
"seq": 6920718,
"id": "wloggertojs",
"changes": [
{
"rev": "29-5b5aa74bd30ff0fc86b39fba799befe2"
}
]
},
{
"seq": 6920720,
"id": "wloggertofile",
"changes": [
{
"rev": "65-aa8d2005c1ecb90b8bd67b62daecfbb5"
}
]
},
{
"seq": 6920721,
"id": "@brightcove/flashls",
"changes": [
{
"rev": "62-fbadb49476a58e98f0f136c86b614734"
}
]
},
{
"seq": 6920722,
"id": "@brightcove/hls-fetcher",
"changes": [
{
"rev": "76-3341ed8ade38f3251a97c94c3a7af5ac"
}
]
},
{
"seq": 6920723,
"id": "@brightcove/kacl",
"changes": [
{
"rev": "33-d0bc6b639cccb301086114d548ecfdbf"
}
]
},
{
"seq": 6920724,
"id": "just-in-types",
"changes": [
{
"rev": "2-fc329aa885dc795aee340f36ec60f333"
}
]
},
{
"seq": 6920725,
"id": "@brightcove/player-loader",
"changes": [
{
"rev": "56-9ff5aebc9743a44d46c182746313877d"
}
]
},
{
"seq": 6920726,
"id": "@brightcove/player-loader-webpack-plugin",
"changes": [
{
"rev": "33-db8b4d6765f19e475e1c1d16843824cb"
}
]
},
{
"seq": 6920727,
"id": "@brightcove/player-url",
"changes": [
{
"rev": "28-2e5c7fecca46bf0f341395a57dc6b3bc"
}
]
},
{
"seq": 6920728,
"id": "@brightcove/react-player-loader",
"changes": [
{
"rev": "39-b7bf609de666ec7e71f517db53ab9c0a"
}
]
},
{
"seq": 6920729,
"id": "vscode-theme-generator",
"changes": [
{
"rev": "21-bcb92281d6f7e37548bb18113681df88"
}
]
},
{
"seq": 6920733,
"id": "@brightcove/typed-immutable-extensions",
"changes": [
{
"rev": "29-4f44b68fd5b8fdc0e499a8a93d8fbabe"
}
]
},
{
"seq": 6920734,
"id": "@brightcove/typed-immutable-proptypes",
"changes": [
{
"rev": "27-e4802afc947c55d34f778864476c17e4"
}
]
},
{
"seq": 6920737,
"id": "@brightcove/videojs-flashls-source-handler",
"changes": [
{
"rev": "59-faf69c49be866b2ab7faa7be9972e7a5"
}
]
},
{
"seq": 6920738,
"id": "@brightcove/videojs-flashls-swf",
"changes": [
{
"rev": "60-04908466eaac2194bc3061e91f463dab"
}
]
},
{
"seq": 6920739,
"id": "@noqcks/generated",
"changes": [
{
"rev": "2-e07d07614182d4beccc507ca199e612d"
}
]
},
{
"seq": 6920740,
"id": "pkcs7",
"changes": [
{
"rev": "60-65ba116f3b6b705f472971b5c6a8f8d2"
}
]
},
{
"seq": 6920741,
"id": "videojs-errors",
"changes": [
{
"rev": "57-c999abd162ca4b93412e363443aa688a"
}
]
},
{
"seq": 6920742,
"id": "videojs-flashls-source-handler",
"changes": [
{
"rev": "59-46d62e18971a8c800710a8fbf985c1c5"
}
]
},
{
"seq": 6920743,
"id": "videojs-playlist",
"changes": [
{
"rev": "97-d4b3492a94c1084c272162dd51901188"
}
]
},
{
"seq": 6920745,
"id": "videojs-playlist-ui",
"changes": [
{
"rev": "95-ba97c44c354b2262e639f8c515bed9bc"
}
]
},
{
"seq": 6920746,
"id": "fusion-apollo-universal-client",
"changes": [
{
"rev": "25-7123042a477cec67c7d5fc702254c7a3"
}
]
},
{
"seq": 6920749,
"id": "msg-fabric-core",
"changes": [
{
"rev": "20-17c33e06faca357526c7395aca1113d2"
}
]
},
{
"seq": 6920750,
"id": "@expo/schemer",
"changes": [
{
"rev": "62-3b1fc389ba4a6ecfc7a40f9c1b83016d"
}
]
},
{
"seq": 6920752,
"id": "mathjs",
"changes": [
{
"rev": "115-bff8ab85ac0812cad09d37ddcbd8ac18"
}
]
},
{
"seq": 6920758,
"id": "statesauce-ui",
"changes": [
{
"rev": "6-db9a39366c1a082c56a2212e368e3ae2"
}
]
},
{
"seq": 6920782,
"id": "@catchandrelease/arbor",
"changes": [
{
"rev": "19-925648432b398ecadc98993e6fba2353"
}
]
},
{
"seq": 6920784,
"id": "discover-shared-ebsco-ui-core",
"changes": [
{
"rev": "4-277063cbc6b71f969e5f0db8c371db65"
}
]
},
{
"seq": 6920807,
"id": "react-apexcharts",
"changes": [
{
"rev": "13-18505be8026a50390c1ff1ba522cb9bd"
}
]
},
{
"seq": 6920819,
"id": "zigbee-shepherd-converters",
"changes": [
{
"rev": "90-5819692a5a9679ff8669fb410e190515"
}
]
},
{
"seq": 6920835,
"id": "honeycomb-grid",
"changes": [
{
"rev": "36-edd6733c80b04a72600558dc55348c73"
}
]
},
{
"seq": 6920838,
"id": "pixl-config",
"changes": [
{
"rev": "7-5dd2b68d04fefb4039b3965b3497eda2"
}
]
},
{
"seq": 6920842,
"id": "discover-shared-ebsco-ui-theming",
"changes": [
{
"rev": "4-e9d083825b1eae46f28c4def2d0db79f"
}
]
},
{
"seq": 6920843,
"id": "common-oxgalaxy-lengua-app",
"changes": [
{
"rev": "66-8b64fa98b4c16b81fb906f0a1bb8539f"
}
]
},
{
"seq": 6920845,
"id": "discover-shared-ebsco-ui-grid",
"changes": [
{
"rev": "2-6f71cf625a5232075071952b2adaa8f2"
}
]
},
{
"seq": 6920847,
"id": "@auth0/cosmos-tokens",
"changes": [
{
"rev": "44-85cd3760dc5e7cfc2fa6330f12f04efb"
}
]
},
{
"seq": 6920848,
"id": "@auth0/babel-preset-cosmos",
"changes": [
{
"rev": "43-d05d3779db08f08726ba048da298e046"
}
]
},
{
"seq": 6920849,
"id": "jsrender",
"changes": [
{
"rev": "11-c949091592b3329d73ae564e45a3472d"
}
]
},
{
"seq": 6920850,
"id": "discover-shared-ebsco-ui-container",
"changes": [
{
"rev": "2-c32089f76b7f253bc0d765da8b9f670d"
}
]
},
{
"seq": 6920852,
"id": "@auth0/cosmos",
"changes": [
{
"rev": "42-5fdaf3d9063c20dac13dcf455c42773c"
}
]
},
{
"seq": 6920853,
"id": "discover-shared-ebsco-ui-checkbox",
"changes": [
{
"rev": "2-06d9521b86f0dbf4a398726faead1212"
}
]
},
{
"seq": 6920854,
"id": "@adunigan/toggles",
"changes": [
{
"rev": "1-c2a830cf814a9fe2d72084339c9c5d28"
}
]
},
{
"seq": 6920855,
"id": "@spriteful/spriteful-lazy-carousel",
"changes": [
{
"rev": "8-28a4bbfe2d1ff24cddcc5aeba6c77837"
}
]
},
{
"seq": 6920856,
"id": "react-modal-hook",
"changes": [
{
"rev": "2-364b39d6559364c41d5b852ccad4ce31"
}
],
"deleted": true
},
{
"seq": 6920859,
"id": "@bellese/angular-design-system",
"changes": [
{
"rev": "39-3e297f85ce2d6a6b6d15fc26420fc471"
}
]
},
{
"seq": 6920861,
"id": "@uifabric/styling",
"changes": [
{
"rev": "229-addf6cc0e74a335125c04d60047353f5"
}
]
},
{
"seq": 6920862,
"id": "@uifabric/file-type-icons",
"changes": [
{
"rev": "37-8a7e43399d1bb9f17334b10995f78df4"
}
]
},
{
"seq": 6920864,
"id": "throttlewrap",
"changes": [
{
"rev": "3-7ab31c0a6a02ed02b96734c747c8c6fa"
}
]
},
{
"seq": 6920865,
"id": "airtable",
"changes": [
{
"rev": "16-d8aee935f6fa4c88057d75a0542bc58c"
}
]
},
{
"seq": 6920866,
"id": "@csmart/ngc-smart-address",
"changes": [
{
"rev": "19-66a6ea868aae1912952f232d2c699f3a"
}
]
},
{
"seq": 6920868,
"id": "office-ui-fabric-react",
"changes": [
{
"rev": "744-8542f4e04c0e9230e2ba19c9e0d7b461"
}
]
},
{
"seq": 6920869,
"id": "@fuelrats/eslint-config",
"changes": [
{
"rev": "12-1b4c71b78fd078e3c1cba535e8541bed"
}
]
},
{
"seq": 6920870,
"id": "@uifabric/date-time",
"changes": [
{
"rev": "2-f955fd46e3b7d3b70d1c82eeadd3f2ed"
}
]
},
{
"seq": 6920872,
"id": "dark-client",
"changes": [
{
"rev": "11-a954c2a89a130ae73f064233d9b3bce2"
}
]
},
{
"seq": 6920873,
"id": "@uifabric/variants",
"changes": [
{
"rev": "59-391c720194c663b9a5c59fe2c10a1535"
}
]
},
{
"seq": 6920875,
"id": "discover-shared-ebsco-ui-header",
"changes": [
{
"rev": "2-efd8f0426a83422a6c8b7bff11054c72"
}
]
},
{
"seq": 6920876,
"id": "react-responsive-picture",
"changes": [
{
"rev": "14-32a6d0850c8af33412cfdb23afd2ecfa"
}
]
},
{
"seq": 6920877,
"id": "@uifabric/fluent-theme",
"changes": [
{
"rev": "16-39c29e00b81a0b654213a5a50d7e7f42"
}
]
},
{
"seq": 6920878,
"id": "@uifabric/dashboard",
"changes": [
{
"rev": "82-04d6dc25b33e811c1d8c24566127b09c"
}
]
},
{
"seq": 6920879,
"id": "ids-enterprise",
"changes": [
{
"rev": "201-dd709a3912f9832440320d448850b61a"
}
]
},
{
"seq": 6920880,
"id": "@uifabric/experiments",
"changes": [
{
"rev": "224-efd1ef07f7640952c286488eae282367"
}
]
},
{
"seq": 6920881,
"id": "@fuelrats/eslint-config-react",
"changes": [
{
"rev": "10-d872deb1eebced4d1d8c3ea6cb5d98bc"
}
]
},
{
"seq": 6920883,
"id": "jsviews",
"changes": [
{
"rev": "11-44d8bedffc98cf6ac4aa669ba8844746"
}
]
},
{
"seq": 6920885,
"id": "pixl-server",
"changes": [
{
"rev": "15-823f4598c3354500d8d2a266dd062502"
}
]
},
{
"seq": 6920887,
"id": "@rrpm/netlify-cms-core",
"changes": [
{
"rev": "17-0dc4eafba1098806dd4cc0cb631eb5fa"
}
]
},
{
"seq": 6920889,
"id": "lodash-a",
"changes": [
{
"rev": "2-6ee66153dbe611a080b40775387d2d45"
}
]
},
{
"seq": 6920891,
"id": "meshcentral",
"changes": [
{
"rev": "499-6677ca74525ed2aa77644c68001382fe"
}
]
},
{
"seq": 6920892,
"id": "vue-transition-collection",
"changes": [
{
"rev": "2-0510ee52c014c0d3b1e65f24376d76f0"
}
]
},
{
"seq": 6920894,
"id": "fury-adapter-swagger",
"changes": [
{
"rev": "47-09f0c55d8574d654c67f9244c21d7ef7"
}
]
},
{
"seq": 6920895,
"id": "@isobar-us/redux-form-gen",
"changes": [
{
"rev": "30-70d7d9210264a321092c832063934648"
}
]
},
{
"seq": 6920896,
"id": "atomizer",
"changes": [
{
"rev": "19-129774900cb2a67a46871cc2c40c34d3"
}
]
},
{
"seq": 6920904,
"id": "boom-js-client",
"changes": [
{
"rev": "15-fe8d703ddfdc0bd220c3c2f7ea46d2c9"
}
]
},
{
"seq": 6920905,
"id": "@ts-common/json-parser",
"changes": [
{
"rev": "17-fe8cc9bc4a5021fde8629a8f880f64b3"
}
]
},
{
"seq": 6920906,
"id": "rutt",
"changes": [
{
"rev": "13-78aab849cb00a6ef7ebc8165770b7d33"
}
]
},
{
"seq": 6920907,
"id": "linear-react-components-ui",
"changes": [
{
"rev": "171-0307f1d69843b270e687371c67cbd1b0"
}
]
},
{
"seq": 6920908,
"id": "@earnest/eslint-config",
"changes": [
{
"rev": "180-b5250dd803102cf7dbac8da9c1a403fd"
}
]
},
{
"seq": 6920909,
"id": "@earnest/eslint-config-es7",
"changes": [
{
"rev": "181-da26885e0baacaea95814857f459572d"
}
]
},
{
"seq": 6920910,
"id": "fuse-design",
"changes": [
{
"rev": "10-e2b78592872f680c05e55eb5b81a0cab"
}
]
}
],
"last_seq": 6920912
}

19
swh/lister/npm/tests/test_npm_lister.py
View file

@ -7,7 +7,7 @@ import requests_mock
import unittest
from swh.lister.core.tests.test_lister import HttpListerTesterBase
from swh.lister.npm.lister import NpmLister
from swh.lister.npm.lister import NpmLister, NpmIncrementalLister
class NpmListerTester(HttpListerTesterBase, unittest.TestCase):
@ -25,3 +25,20 @@ class NpmListerTester(HttpListerTesterBase, unittest.TestCase):
# it can not succeed for the npm lister due to the
# overriding of the string_pattern_check method
pass
class NpmIncrementalListerTester(HttpListerTesterBase, unittest.TestCase):
Lister = NpmIncrementalLister
test_re = re.compile(r'^.*/_changes\?since=([0-9]+).*')
lister_subdir = 'npm'
good_api_response_file = 'api_inc_response.json'
bad_api_response_file = 'api_inc_empty_response.json'
first_index = 6920642
entries_per_page = 100
@requests_mock.Mocker()
def test_is_within_bounds(self, http_mocker):
# disable this test from HttpListerTesterBase as
# it can not succeed for the npm lister due to the
# overriding of the string_pattern_check method
pass