From ff232f0d918000325f245657e43ce8fbea4d556e Mon Sep 17 00:00:00 2001 From: Antoine Lambert Date: Fri, 15 Jan 2021 17:48:00 +0100 Subject: [PATCH] npm: Reimplement lister using new Lister API Port npm lister to `swh.lister.pattern.Lister` API. As before, the lister can be run in full or incremental mode. When using incremental mode, only new and modified packages will be returned since the last incremental listing process. Otherwise, all packages will be listed in lexicographical order. One major improvement to be noted, latest package update date is now retrieved when available and sent to scheduler database. Closes T2972 --- swh/lister/npm/__init__.py | 5 +- swh/lister/npm/lister.py | 276 +++--- swh/lister/npm/models.py | 37 - swh/lister/npm/tasks.py | 57 +- swh/lister/npm/tests/conftest.py | 17 +- .../npm/tests/data/api_empty_response.json | 5 - .../tests/data/api_inc_empty_response.json | 4 - .../npm/tests/data/api_inc_response.json | 906 ------------------ .../_all_docs,startkey=\"\",limit=1001" | 807 ---------------- .../_all_docs,startkey=\"\",limit=11" | 83 -- ...accessible-simple-tooltip-aria\",limit=11" | 83 -- .../api_response.json | 1 - swh/lister/npm/tests/data/npm_full_page1.json | 235 +++++ swh/lister/npm/tests/data/npm_full_page2.json | 168 ++++ .../npm/tests/data/npm_incremental_page1.json | 175 ++++ .../npm/tests/data/npm_incremental_page2.json | 632 ++++++++++++ swh/lister/npm/tests/test_lister.py | 251 +++-- swh/lister/npm/tests/test_tasks.py | 51 +- 18 files changed, 1573 insertions(+), 2220 deletions(-) delete mode 100644 swh/lister/npm/models.py delete mode 100644 swh/lister/npm/tests/data/api_empty_response.json delete mode 100644 swh/lister/npm/tests/data/api_inc_empty_response.json delete mode 100644 swh/lister/npm/tests/data/api_inc_response.json delete mode 100644 "swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"\",limit=1001" delete mode 100644 "swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"\",limit=11" delete mode 100644 "swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"jquery-accessible-simple-tooltip-aria\",limit=11" delete mode 120000 swh/lister/npm/tests/data/https_replicate.npmjs.com/api_response.json create mode 100644 swh/lister/npm/tests/data/npm_full_page1.json create mode 100644 swh/lister/npm/tests/data/npm_full_page2.json create mode 100644 swh/lister/npm/tests/data/npm_incremental_page1.json create mode 100644 swh/lister/npm/tests/data/npm_incremental_page2.json diff --git a/swh/lister/npm/__init__.py b/swh/lister/npm/__init__.py index e2ec1bf..7544bd1 100644 --- a/swh/lister/npm/__init__.py +++ b/swh/lister/npm/__init__.py @@ -1,14 +1,13 @@ -# Copyright (C) 2019 the Software Heritage developers +# Copyright (C) 2019-2021 the Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information def register(): from .lister import NpmLister - from .models import NpmModel, NpmVisitModel return { - "models": [NpmVisitModel, NpmModel], + "models": [], "lister": NpmLister, "task_modules": ["%s.tasks" % __name__], "task_types": { diff --git a/swh/lister/npm/lister.py b/swh/lister/npm/lister.py index a1165e1..dfc6561 100644 --- a/swh/lister/npm/lister.py +++ b/swh/lister/npm/lister.py @@ -1,154 +1,190 @@ -# Copyright (C) 2018-2019 the Software Heritage developers +# Copyright (C) 2018-2021 the Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from typing import Any, Dict, List, Optional +from dataclasses import asdict, dataclass +import logging +from typing import Any, Dict, Iterator, List, Optional -from requests import Response +import iso8601 +import requests +from tenacity.before_sleep import before_sleep_log -from swh.core import config -from swh.lister.core.indexing_lister import IndexingHttpLister -from swh.lister.npm.models import NpmModel -from swh.scheduler.utils import create_task_dict +from swh.lister import USER_AGENT +from swh.lister.pattern import CredentialsType, Lister +from swh.lister.utils import throttling_retry +from swh.scheduler.interface import SchedulerInterface +from swh.scheduler.model import ListedOrigin -DEFAULT_CONFIG = { - "loading_task_policy": "recurring", -} +logger = logging.getLogger(__name__) -class NpmListerBase(IndexingHttpLister): - """List packages available in the npm registry in a paginated way +@dataclass +class NpmListerState: + """State of npm lister""" + + last_seq: Optional[int] = None + + +class NpmLister(Lister[NpmListerState, List[Dict[str, Any]]]): + """ + List all packages hosted on the npm registry. + + The lister is based on the npm replication API powered by a + CouchDB database (https://docs.couchdb.org/en/stable/api/database/). + + Args: + scheduler: a scheduler instance + page_size: number of packages info to return per page when querying npm API + incremental: defines if incremental listing should be used, in that case + only modified or new packages since last incremental listing operation + will be returned, otherwise all packages will be listed in lexicographical + order """ - MODEL = NpmModel LISTER_NAME = "npm" - instance = "npm" + INSTANCE = "npm" + + API_BASE_URL = "https://replicate.npmjs.com" + API_INCREMENTAL_LISTING_URL = f"{API_BASE_URL}/_changes" + API_FULL_LISTING_URL = f"{API_BASE_URL}/_all_docs" + PACKAGE_URL_TEMPLATE = "https://www.npmjs.com/package/{package_name}" def __init__( - self, url="https://replicate.npmjs.com", per_page=1000, override_config=None + self, + scheduler: SchedulerInterface, + page_size: int = 1000, + incremental: bool = False, + credentials: CredentialsType = None, ): - super().__init__(url=url, override_config=override_config) - self.config = config.merge_configs(DEFAULT_CONFIG, self.config) - self.per_page = per_page + 1 - self.PATH_TEMPLATE += "&limit=%s" % self.per_page + super().__init__( + scheduler=scheduler, + credentials=credentials, + url=self.API_INCREMENTAL_LISTING_URL + if incremental + else self.API_FULL_LISTING_URL, + instance=self.INSTANCE, + ) - def get_model_from_repo(self, repo_name: str) -> Dict[str, str]: - """(Override) Transform from npm package name to model + self.page_size = page_size + if not incremental: + # in full listing mode, first package in each page corresponds to the one + # provided as the startkey query parameter value, so we increment the page + # size by one to avoid double package processing + self.page_size += 1 + self.incremental = incremental - """ - package_url = "https://www.npmjs.com/package/%s" % repo_name - return { - "uid": repo_name, - "indexable": repo_name, - "name": repo_name, - "full_name": repo_name, - "html_url": package_url, - "origin_url": package_url, - "origin_type": "npm", - } + self.session = requests.Session() + self.session.headers.update( + {"Accept": "application/json", "User-Agent": USER_AGENT} + ) - def task_dict(self, origin_type: str, origin_url: str, **kwargs): - """(Override) Return task dict for loading a npm package into the - archive. + def state_from_dict(self, d: Dict[str, Any]) -> NpmListerState: + return NpmListerState(**d) - This is overridden from the lister_base as more information is - needed for the ingestion task creation. + def state_to_dict(self, state: NpmListerState) -> Dict[str, Any]: + return asdict(state) - """ - task_type = "load-%s" % origin_type - task_policy = self.config["loading_task_policy"] - return create_task_dict(task_type, task_policy, url=origin_url) + def request_params(self, last_package_id: str) -> Dict[str, Any]: + # include package JSON document to get its last update date + params = {"limit": self.page_size, "include_docs": "true"} + if self.incremental: + params["since"] = last_package_id + else: + params["startkey"] = last_package_id + return params - def request_headers(self) -> Dict[str, Any]: - """(Override) Set requests headers to send when querying the npm - registry. + @throttling_retry(before_sleep=before_sleep_log(logger, logging.WARNING)) + def page_request(self, last_package_id: str) -> requests.Response: + params = self.request_params(last_package_id) + logger.debug("Fetching URL %s with params %s", self.url, params) + response = self.session.get(self.url, params=params) + if response.status_code != 200: + logger.warning( + "Unexpected HTTP status code %s on %s: %s", + response.status_code, + response.url, + response.content, + ) + response.raise_for_status() + return response - """ - headers = super().request_headers() - headers["Accept"] = "application/json" - return headers + def get_pages(self) -> Iterator[List[Dict[str, Any]]]: + last_package_id: str = "0" if self.incremental else '""' + if ( + self.incremental + and self.state is not None + and self.state.last_seq is not None + ): + last_package_id = str(self.state.last_seq) - def string_pattern_check(self, inner: int, lower: int, upper: int = None): - """ (Override) Inhibit the effect of that method as packages indices - correspond to package names and thus do not respect any kind - of fixed length string pattern + while True: - """ - pass + response = self.page_request(last_package_id) + data = response.json() + page = data["results"] if self.incremental else data["rows"] -class NpmLister(NpmListerBase): - """List all packages available in the npm registry in a paginated way + if not page: + break - """ + if self.incremental or len(page) < self.page_size: + yield page + else: + yield page[:-1] - PATH_TEMPLATE = '/_all_docs?startkey="%s"' + if len(page) < self.page_size: + break - def get_next_target_from_response(self, response: Response) -> Optional[str]: - """(Override) Get next npm package name to continue the listing + last_package_id = ( + str(page[-1]["seq"]) if self.incremental else f'"{page[-1]["id"]}"' + ) - """ - repos = response.json()["rows"] - return repos[-1]["id"] if len(repos) == self.per_page else None + def get_origins_from_page( + self, page: List[Dict[str, Any]] + ) -> Iterator[ListedOrigin]: + """Convert a page of Npm repositories into a list of ListedOrigin.""" + assert self.lister_obj.id is not None - def transport_response_simplified(self, response: Response) -> List[Dict[str, str]]: - """(Override) Transform npm registry response to list for model manipulation - - """ - repos = response.json()["rows"] - if len(repos) == self.per_page: - repos = repos[:-1] - return [self.get_model_from_repo(repo["id"]) for repo in repos] - - -class NpmIncrementalLister(NpmListerBase): - """List packages in the npm registry, updated since a specific - update_seq value of the underlying CouchDB database, in a paginated way. - - """ - - PATH_TEMPLATE = "/_changes?since=%s" - - @property - def CONFIG_BASE_FILENAME(self): # noqa: N802 - return "lister_npm_incremental" - - def get_next_target_from_response(self, response: Response) -> Optional[str]: - """(Override) Get next npm package name to continue the listing. - - """ - repos = response.json()["results"] - return repos[-1]["seq"] if len(repos) == self.per_page else None - - def transport_response_simplified(self, response: Response) -> List[Dict[str, str]]: - """(Override) Transform npm registry response to list for model - manipulation. - - """ - repos = response.json()["results"] - if len(repos) == self.per_page: - repos = repos[:-1] - return [self.get_model_from_repo(repo["id"]) for repo in repos] - - def filter_before_inject(self, models_list: List[Dict[str, Any]]): - """(Override) Filter out documents in the CouchDB database - not related to a npm package. - - """ - models_filtered = [] - for model in models_list: - package_name = model["name"] - # document related to CouchDB internals - if package_name.startswith("_design/"): + for package in page: + # no source code to archive here + if not package["doc"].get("versions", {}): continue - models_filtered.append(model) - return models_filtered - def disable_deleted_repo_tasks(self, start, end, keep_these): - """(Override) Disable the processing performed by that method as it is - not relevant in this incremental lister context. It also raises an - exception due to a different index type (int instead of str). + package_name = package["doc"]["name"] + package_latest_version = ( + package["doc"].get("dist-tags", {}).get("latest", "") + ) - """ - pass + last_update = None + if package_latest_version in package["doc"].get("time", {}): + last_update = iso8601.parse_date( + package["doc"]["time"][package_latest_version] + ) + + yield ListedOrigin( + lister_id=self.lister_obj.id, + url=self.PACKAGE_URL_TEMPLATE.format(package_name=package_name), + visit_type="npm", + last_update=last_update, + ) + + def commit_page(self, page: List[Dict[str, Any]]): + """Update the currently stored state using the latest listed page.""" + if self.incremental: + last_package = page[-1] + last_seq = last_package["seq"] + + if self.state.last_seq is None or last_seq > self.state.last_seq: + self.state.last_seq = last_seq + + def finalize(self): + if self.incremental and self.state.last_seq is not None: + scheduler_state = self.get_state_from_scheduler() + + if ( + scheduler_state.last_seq is None + or self.state.last_seq > scheduler_state.last_seq + ): + self.updated = True diff --git a/swh/lister/npm/models.py b/swh/lister/npm/models.py deleted file mode 100644 index e8d88ab..0000000 --- a/swh/lister/npm/models.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright (C) 2018 the Software Heritage developers -# License: GNU General Public License version 3, or any later version -# See top-level LICENSE file for more information - -from sqlalchemy import BigInteger, Column, DateTime, Integer, Sequence, String - -from swh.lister.core.models import ABCSQLMeta, IndexingModelBase, SQLBase - - -class NpmVisitModel(SQLBase, metaclass=ABCSQLMeta): - """Table to store the npm registry state at the time of a - content listing by Software Heritage - """ - - __tablename__ = "npm_visit" - - uid = Column(Integer, Sequence("npm_visit_id_seq"), primary_key=True) - visit_date = Column(DateTime, nullable=False) - doc_count = Column(BigInteger) - doc_del_count = Column(BigInteger) - update_seq = Column(BigInteger) - purge_seq = Column(BigInteger) - disk_size = Column(BigInteger) - data_size = Column(BigInteger) - committed_update_seq = Column(BigInteger) - compacted_seq = Column(BigInteger) - - -class NpmModel(IndexingModelBase): - """A npm package representation - - """ - - __tablename__ = "npm_repo" - - uid = Column(String, primary_key=True) - indexable = Column(String, index=True) diff --git a/swh/lister/npm/tasks.py b/swh/lister/npm/tasks.py index 5e977b4..c276806 100644 --- a/swh/lister/npm/tasks.py +++ b/swh/lister/npm/tasks.py @@ -1,69 +1,24 @@ -# Copyright (C) 2018 the Software Heritage developers +# Copyright (C) 2018-2021 the Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from contextlib import contextmanager -from datetime import datetime - from celery import shared_task -from swh.lister.npm.lister import NpmIncrementalLister, NpmLister -from swh.lister.npm.models import NpmVisitModel - - -@contextmanager -def save_registry_state(lister): - params = {"headers": lister.request_headers()} - registry_state = lister.session.get(lister.url, **params) - registry_state = registry_state.json() - keys = ( - "doc_count", - "doc_del_count", - "update_seq", - "purge_seq", - "disk_size", - "data_size", - "committed_update_seq", - "compacted_seq", - ) - - state = {key: registry_state[key] for key in keys} - state["visit_date"] = datetime.now() - yield - npm_visit = NpmVisitModel(**state) - lister.db_session.add(npm_visit) - lister.db_session.commit() - - -def get_last_update_seq(lister): - """Get latest ``update_seq`` value for listing only updated packages. - """ - query = lister.db_session.query(NpmVisitModel.update_seq) - row = query.order_by(NpmVisitModel.uid.desc()).first() - if not row: - raise ValueError( - "No npm registry listing previously performed ! " - "This is required prior to the execution of an " - "incremental listing." - ) - return row[0] +from swh.lister.npm.lister import NpmLister @shared_task(name=__name__ + ".NpmListerTask") def list_npm_full(**lister_args): "Full lister for the npm (javascript) registry" - lister = NpmLister(**lister_args) - with save_registry_state(lister): - return lister.run() + lister = NpmLister.from_configfile(incremental=False, **lister_args) + return lister.run().dict() @shared_task(name=__name__ + ".NpmIncrementalListerTask") def list_npm_incremental(**lister_args): "Incremental lister for the npm (javascript) registry" - lister = NpmIncrementalLister(**lister_args) - update_seq_start = get_last_update_seq(lister) - with save_registry_state(lister): - return lister.run(min_bound=update_seq_start) + lister = NpmLister.from_configfile(incremental=True, **lister_args) + return lister.run().dict() @shared_task(name=__name__ + ".ping") diff --git a/swh/lister/npm/tests/conftest.py b/swh/lister/npm/tests/conftest.py index 81c3e1c..fe23d0b 100644 --- a/swh/lister/npm/tests/conftest.py +++ b/swh/lister/npm/tests/conftest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2019-2020 The Software Heritage developers +# Copyright (C) 2019-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -9,18 +9,3 @@ import pytest @pytest.fixture def lister_under_test(): return "npm" - - -@pytest.fixture -def lister_npm(swh_lister): - # Add the load-npm in the scheduler backend - swh_lister.scheduler.create_task_type( - { - "type": "load-npm", - "description": "Load npm package", - "backend_name": "swh.loader.package.tasks.LoadNpm", - "default_interval": "1 day", - } - ) - - return swh_lister diff --git a/swh/lister/npm/tests/data/api_empty_response.json b/swh/lister/npm/tests/data/api_empty_response.json deleted file mode 100644 index 23eb9e7..0000000 --- a/swh/lister/npm/tests/data/api_empty_response.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "total_rows": 839080, - "offset": 839080, - "rows": [] -} \ No newline at end of file diff --git a/swh/lister/npm/tests/data/api_inc_empty_response.json b/swh/lister/npm/tests/data/api_inc_empty_response.json deleted file mode 100644 index f820af2..0000000 --- a/swh/lister/npm/tests/data/api_inc_empty_response.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "results": [], - "last_seq": 6927821 -} \ No newline at end of file diff --git a/swh/lister/npm/tests/data/api_inc_response.json b/swh/lister/npm/tests/data/api_inc_response.json deleted file mode 100644 index 595298f..0000000 --- a/swh/lister/npm/tests/data/api_inc_response.json +++ /dev/null @@ -1,906 +0,0 @@ -{ - "results": [ - { - "seq": 6920644, - "id": "electron-scripts", - "changes": [ - { - "rev": "3-a19944df5a3636bb225af9e0c8f9eedc" - } - ] - }, - { - "seq": 6920649, - "id": "@crexi-dev/schematics", - "changes": [ - { - "rev": "3-00188360eeca1f9123b2d7cd4b468c50" - } - ] - }, - { - "seq": 6920651, - "id": "botfactory-conversation", - "changes": [ - { - "rev": "50-f3667cde87637505528c46adc87f44e3" - } - ] - }, - { - "seq": 6920667, - "id": "castle", - "changes": [ - { - "rev": "3-d9adf9c9fd687cdaa2bf460c5bb523f0" - } - ] - }, - { - "seq": 6920671, - "id": "rbc-wm-framework-vuejs", - "changes": [ - { - "rev": "111-32bed479afacdd88aed9ac16dd135843" - } - ] - }, - { - "seq": 6920678, - "id": "bitcoinfiles", - "changes": [ - { - "rev": "22-ab3cd6b46f84d9aac1a24560cabdc9f0" - } - ] - }, - { - "seq": 6920679, - "id": "jovo-core", - "changes": [ - { - "rev": "2-d7440f1d17823e1a0760d9b3d4537c6e" - } - ] - }, - { - "seq": 6920687, - "id": "jovo-framework", - "changes": [ - { - "rev": "103-e4f46a3530514c2ee81a97d25fc8c8c9" - } - ] - }, - { - "seq": 6920690, - "id": "smart-form-lib", - "changes": [ - { - "rev": "18-3b6b6b2b0ea2e114a3f1335a8e798ade" - } - ] - }, - { - "seq": 6920694, - "id": "bokehjs", - "changes": [ - { - "rev": "18-115ce2d4bf4f281eb50c25f3203b3dd2" - } - ] - }, - { - "seq": 6920701, - "id": "guijarro", - "changes": [ - { - "rev": "14-82ece581d6a35d4e1d78e5292ca245c0" - } - ] - }, - { - "seq": 6920702, - "id": "@kava-labs/crypto-rate-utils", - "changes": [ - { - "rev": "3-cecc6a6c226a0590b1a685e3041028c6" - } - ] - }, - { - "seq": 6920703, - "id": "@riouxjean/test", - "changes": [ - { - "rev": "10-01e97dc7d0241dc49ea93b3468ec7b29" - } - ] - }, - { - "seq": 6920704, - "id": "react-scrabblefy", - "changes": [ - { - "rev": "7-970c8206f3b8744204f7dcb106f8462b" - } - ] - }, - { - "seq": 6920706, - "id": "molart", - "changes": [ - { - "rev": "14-416cd3cec62dd46f9b59a3bbe35308f6" - } - ] - }, - { - "seq": 6920707, - "id": "@universal-material/angular", - "changes": [ - { - "rev": "32-266ed3f67e1ddd0b4a37ca29f1cf5bf3" - } - ] - }, - { - "seq": 6920708, - "id": "cozy-doctypes", - "changes": [ - { - "rev": "68-8e90cc26e25da6c9430d373e43ac3c25" - } - ] - }, - { - "seq": 6920710, - "id": "2o3t-ui", - "changes": [ - { - "rev": "96-1e65d5320ea7c78525aba5daf328bd4b" - } - ] - }, - { - "seq": 6920712, - "id": "ark-ts", - "changes": [ - { - "rev": "24-033183c2f7f9cbb6e44d553213e525b6" - } - ] - }, - { - "seq": 6920715, - "id": "mysqlconnector", - "changes": [ - { - "rev": "19-f09bc0b82281ca486db5ebe83843679e" - } - ] - }, - { - "seq": 6920716, - "id": "@innovexa/ng-form-creator-lib", - "changes": [ - { - "rev": "147-480665ee17fa889dfec1aee75b907ff2" - } - ] - }, - { - "seq": 6920717, - "id": "k-routes-example-basic", - "changes": [ - { - "rev": "1-35142059e1c63cc724da71a9eebf229c" - } - ] - }, - { - "seq": 6920718, - "id": "wloggertojs", - "changes": [ - { - "rev": "29-5b5aa74bd30ff0fc86b39fba799befe2" - } - ] - }, - { - "seq": 6920720, - "id": "wloggertofile", - "changes": [ - { - "rev": "65-aa8d2005c1ecb90b8bd67b62daecfbb5" - } - ] - }, - { - "seq": 6920721, - "id": "@brightcove/flashls", - "changes": [ - { - "rev": "62-fbadb49476a58e98f0f136c86b614734" - } - ] - }, - { - "seq": 6920722, - "id": "@brightcove/hls-fetcher", - "changes": [ - { - "rev": "76-3341ed8ade38f3251a97c94c3a7af5ac" - } - ] - }, - { - "seq": 6920723, - "id": "@brightcove/kacl", - "changes": [ - { - "rev": "33-d0bc6b639cccb301086114d548ecfdbf" - } - ] - }, - { - "seq": 6920724, - "id": "just-in-types", - "changes": [ - { - "rev": "2-fc329aa885dc795aee340f36ec60f333" - } - ] - }, - { - "seq": 6920725, - "id": "@brightcove/player-loader", - "changes": [ - { - "rev": "56-9ff5aebc9743a44d46c182746313877d" - } - ] - }, - { - "seq": 6920726, - "id": "@brightcove/player-loader-webpack-plugin", - "changes": [ - { - "rev": "33-db8b4d6765f19e475e1c1d16843824cb" - } - ] - }, - { - "seq": 6920727, - "id": "@brightcove/player-url", - "changes": [ - { - "rev": "28-2e5c7fecca46bf0f341395a57dc6b3bc" - } - ] - }, - { - "seq": 6920728, - "id": "@brightcove/react-player-loader", - "changes": [ - { - "rev": "39-b7bf609de666ec7e71f517db53ab9c0a" - } - ] - }, - { - "seq": 6920729, - "id": "vscode-theme-generator", - "changes": [ - { - "rev": "21-bcb92281d6f7e37548bb18113681df88" - } - ] - }, - { - "seq": 6920733, - "id": "@brightcove/typed-immutable-extensions", - "changes": [ - { - "rev": "29-4f44b68fd5b8fdc0e499a8a93d8fbabe" - } - ] - }, - { - "seq": 6920734, - "id": "@brightcove/typed-immutable-proptypes", - "changes": [ - { - "rev": "27-e4802afc947c55d34f778864476c17e4" - } - ] - }, - { - "seq": 6920737, - "id": "@brightcove/videojs-flashls-source-handler", - "changes": [ - { - "rev": "59-faf69c49be866b2ab7faa7be9972e7a5" - } - ] - }, - { - "seq": 6920738, - "id": "@brightcove/videojs-flashls-swf", - "changes": [ - { - "rev": "60-04908466eaac2194bc3061e91f463dab" - } - ] - }, - { - "seq": 6920739, - "id": "@noqcks/generated", - "changes": [ - { - "rev": "2-e07d07614182d4beccc507ca199e612d" - } - ] - }, - { - "seq": 6920740, - "id": "pkcs7", - "changes": [ - { - "rev": "60-65ba116f3b6b705f472971b5c6a8f8d2" - } - ] - }, - { - "seq": 6920741, - "id": "videojs-errors", - "changes": [ - { - "rev": "57-c999abd162ca4b93412e363443aa688a" - } - ] - }, - { - "seq": 6920742, - "id": "videojs-flashls-source-handler", - "changes": [ - { - "rev": "59-46d62e18971a8c800710a8fbf985c1c5" - } - ] - }, - { - "seq": 6920743, - "id": "videojs-playlist", - "changes": [ - { - "rev": "97-d4b3492a94c1084c272162dd51901188" - } - ] - }, - { - "seq": 6920745, - "id": "videojs-playlist-ui", - "changes": [ - { - "rev": "95-ba97c44c354b2262e639f8c515bed9bc" - } - ] - }, - { - "seq": 6920746, - "id": "fusion-apollo-universal-client", - "changes": [ - { - "rev": "25-7123042a477cec67c7d5fc702254c7a3" - } - ] - }, - { - "seq": 6920749, - "id": "msg-fabric-core", - "changes": [ - { - "rev": "20-17c33e06faca357526c7395aca1113d2" - } - ] - }, - { - "seq": 6920750, - "id": "@expo/schemer", - "changes": [ - { - "rev": "62-3b1fc389ba4a6ecfc7a40f9c1b83016d" - } - ] - }, - { - "seq": 6920752, - "id": "mathjs", - "changes": [ - { - "rev": "115-bff8ab85ac0812cad09d37ddcbd8ac18" - } - ] - }, - { - "seq": 6920758, - "id": "statesauce-ui", - "changes": [ - { - "rev": "6-db9a39366c1a082c56a2212e368e3ae2" - } - ] - }, - { - "seq": 6920782, - "id": "@catchandrelease/arbor", - "changes": [ - { - "rev": "19-925648432b398ecadc98993e6fba2353" - } - ] - }, - { - "seq": 6920784, - "id": "discover-shared-ebsco-ui-core", - "changes": [ - { - "rev": "4-277063cbc6b71f969e5f0db8c371db65" - } - ] - }, - { - "seq": 6920807, - "id": "react-apexcharts", - "changes": [ - { - "rev": "13-18505be8026a50390c1ff1ba522cb9bd" - } - ] - }, - { - "seq": 6920819, - "id": "zigbee-shepherd-converters", - "changes": [ - { - "rev": "90-5819692a5a9679ff8669fb410e190515" - } - ] - }, - { - "seq": 6920835, - "id": "honeycomb-grid", - "changes": [ - { - "rev": "36-edd6733c80b04a72600558dc55348c73" - } - ] - }, - { - "seq": 6920838, - "id": "pixl-config", - "changes": [ - { - "rev": "7-5dd2b68d04fefb4039b3965b3497eda2" - } - ] - }, - { - "seq": 6920842, - "id": "discover-shared-ebsco-ui-theming", - "changes": [ - { - "rev": "4-e9d083825b1eae46f28c4def2d0db79f" - } - ] - }, - { - "seq": 6920843, - "id": "common-oxgalaxy-lengua-app", - "changes": [ - { - "rev": "66-8b64fa98b4c16b81fb906f0a1bb8539f" - } - ] - }, - { - "seq": 6920845, - "id": "discover-shared-ebsco-ui-grid", - "changes": [ - { - "rev": "2-6f71cf625a5232075071952b2adaa8f2" - } - ] - }, - { - "seq": 6920847, - "id": "@auth0/cosmos-tokens", - "changes": [ - { - "rev": "44-85cd3760dc5e7cfc2fa6330f12f04efb" - } - ] - }, - { - "seq": 6920848, - "id": "@auth0/babel-preset-cosmos", - "changes": [ - { - "rev": "43-d05d3779db08f08726ba048da298e046" - } - ] - }, - { - "seq": 6920849, - "id": "jsrender", - "changes": [ - { - "rev": "11-c949091592b3329d73ae564e45a3472d" - } - ] - }, - { - "seq": 6920850, - "id": "discover-shared-ebsco-ui-container", - "changes": [ - { - "rev": "2-c32089f76b7f253bc0d765da8b9f670d" - } - ] - }, - { - "seq": 6920852, - "id": "@auth0/cosmos", - "changes": [ - { - "rev": "42-5fdaf3d9063c20dac13dcf455c42773c" - } - ] - }, - { - "seq": 6920853, - "id": "discover-shared-ebsco-ui-checkbox", - "changes": [ - { - "rev": "2-06d9521b86f0dbf4a398726faead1212" - } - ] - }, - { - "seq": 6920854, - "id": "@adunigan/toggles", - "changes": [ - { - "rev": "1-c2a830cf814a9fe2d72084339c9c5d28" - } - ] - }, - { - "seq": 6920855, - "id": "@spriteful/spriteful-lazy-carousel", - "changes": [ - { - "rev": "8-28a4bbfe2d1ff24cddcc5aeba6c77837" - } - ] - }, - { - "seq": 6920856, - "id": "react-modal-hook", - "changes": [ - { - "rev": "2-364b39d6559364c41d5b852ccad4ce31" - } - ], - "deleted": true - }, - { - "seq": 6920859, - "id": "@bellese/angular-design-system", - "changes": [ - { - "rev": "39-3e297f85ce2d6a6b6d15fc26420fc471" - } - ] - }, - { - "seq": 6920861, - "id": "@uifabric/styling", - "changes": [ - { - "rev": "229-addf6cc0e74a335125c04d60047353f5" - } - ] - }, - { - "seq": 6920862, - "id": "@uifabric/file-type-icons", - "changes": [ - { - "rev": "37-8a7e43399d1bb9f17334b10995f78df4" - } - ] - }, - { - "seq": 6920864, - "id": "throttlewrap", - "changes": [ - { - "rev": "3-7ab31c0a6a02ed02b96734c747c8c6fa" - } - ] - }, - { - "seq": 6920865, - "id": "airtable", - "changes": [ - { - "rev": "16-d8aee935f6fa4c88057d75a0542bc58c" - } - ] - }, - { - "seq": 6920866, - "id": "@csmart/ngc-smart-address", - "changes": [ - { - "rev": "19-66a6ea868aae1912952f232d2c699f3a" - } - ] - }, - { - "seq": 6920868, - "id": "office-ui-fabric-react", - "changes": [ - { - "rev": "744-8542f4e04c0e9230e2ba19c9e0d7b461" - } - ] - }, - { - "seq": 6920869, - "id": "@fuelrats/eslint-config", - "changes": [ - { - "rev": "12-1b4c71b78fd078e3c1cba535e8541bed" - } - ] - }, - { - "seq": 6920870, - "id": "@uifabric/date-time", - "changes": [ - { - "rev": "2-f955fd46e3b7d3b70d1c82eeadd3f2ed" - } - ] - }, - { - "seq": 6920872, - "id": "dark-client", - "changes": [ - { - "rev": "11-a954c2a89a130ae73f064233d9b3bce2" - } - ] - }, - { - "seq": 6920873, - "id": "@uifabric/variants", - "changes": [ - { - "rev": "59-391c720194c663b9a5c59fe2c10a1535" - } - ] - }, - { - "seq": 6920875, - "id": "discover-shared-ebsco-ui-header", - "changes": [ - { - "rev": "2-efd8f0426a83422a6c8b7bff11054c72" - } - ] - }, - { - "seq": 6920876, - "id": "react-responsive-picture", - "changes": [ - { - "rev": "14-32a6d0850c8af33412cfdb23afd2ecfa" - } - ] - }, - { - "seq": 6920877, - "id": "@uifabric/fluent-theme", - "changes": [ - { - "rev": "16-39c29e00b81a0b654213a5a50d7e7f42" - } - ] - }, - { - "seq": 6920878, - "id": "@uifabric/dashboard", - "changes": [ - { - "rev": "82-04d6dc25b33e811c1d8c24566127b09c" - } - ] - }, - { - "seq": 6920879, - "id": "ids-enterprise", - "changes": [ - { - "rev": "201-dd709a3912f9832440320d448850b61a" - } - ] - }, - { - "seq": 6920880, - "id": "@uifabric/experiments", - "changes": [ - { - "rev": "224-efd1ef07f7640952c286488eae282367" - } - ] - }, - { - "seq": 6920881, - "id": "@fuelrats/eslint-config-react", - "changes": [ - { - "rev": "10-d872deb1eebced4d1d8c3ea6cb5d98bc" - } - ] - }, - { - "seq": 6920883, - "id": "jsviews", - "changes": [ - { - "rev": "11-44d8bedffc98cf6ac4aa669ba8844746" - } - ] - }, - { - "seq": 6920885, - "id": "pixl-server", - "changes": [ - { - "rev": "15-823f4598c3354500d8d2a266dd062502" - } - ] - }, - { - "seq": 6920887, - "id": "@rrpm/netlify-cms-core", - "changes": [ - { - "rev": "17-0dc4eafba1098806dd4cc0cb631eb5fa" - } - ] - }, - { - "seq": 6920889, - "id": "lodash-a", - "changes": [ - { - "rev": "2-6ee66153dbe611a080b40775387d2d45" - } - ] - }, - { - "seq": 6920891, - "id": "meshcentral", - "changes": [ - { - "rev": "499-6677ca74525ed2aa77644c68001382fe" - } - ] - }, - { - "seq": 6920892, - "id": "vue-transition-collection", - "changes": [ - { - "rev": "2-0510ee52c014c0d3b1e65f24376d76f0" - } - ] - }, - { - "seq": 6920894, - "id": "fury-adapter-swagger", - "changes": [ - { - "rev": "47-09f0c55d8574d654c67f9244c21d7ef7" - } - ] - }, - { - "seq": 6920895, - "id": "@isobar-us/redux-form-gen", - "changes": [ - { - "rev": "30-70d7d9210264a321092c832063934648" - } - ] - }, - { - "seq": 6920896, - "id": "atomizer", - "changes": [ - { - "rev": "19-129774900cb2a67a46871cc2c40c34d3" - } - ] - }, - { - "seq": 6920904, - "id": "boom-js-client", - "changes": [ - { - "rev": "15-fe8d703ddfdc0bd220c3c2f7ea46d2c9" - } - ] - }, - { - "seq": 6920905, - "id": "@ts-common/json-parser", - "changes": [ - { - "rev": "17-fe8cc9bc4a5021fde8629a8f880f64b3" - } - ] - }, - { - "seq": 6920906, - "id": "rutt", - "changes": [ - { - "rev": "13-78aab849cb00a6ef7ebc8165770b7d33" - } - ] - }, - { - "seq": 6920907, - "id": "linear-react-components-ui", - "changes": [ - { - "rev": "171-0307f1d69843b270e687371c67cbd1b0" - } - ] - }, - { - "seq": 6920908, - "id": "@earnest/eslint-config", - "changes": [ - { - "rev": "180-b5250dd803102cf7dbac8da9c1a403fd" - } - ] - }, - { - "seq": 6920909, - "id": "@earnest/eslint-config-es7", - "changes": [ - { - "rev": "181-da26885e0baacaea95814857f459572d" - } - ] - }, - { - "seq": 6920910, - "id": "fuse-design", - "changes": [ - { - "rev": "10-e2b78592872f680c05e55eb5b81a0cab" - } - ] - } - ], - "last_seq": 6920912 -} \ No newline at end of file diff --git "a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"\",limit=1001" "b/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"\",limit=1001" deleted file mode 100644 index b1c78ef..0000000 --- "a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"\",limit=1001" +++ /dev/null @@ -1,807 +0,0 @@ -{ - "total_rows": 839080, - "offset": 422482, - "rows": [ - - { - "id": "jquery", - "key": "jquery", - "value": { - "rev": "212-2eac7c93af4c8bccdf7317739f0319b6" - } - }, - - { - "id": "jquery-1.8", - "key": "jquery-1.8", - "value": { - "rev": "1-711ded49a7453adce85ce7a51c2157de" - } - }, - - { - "id": "jquery-1x", - "key": "jquery-1x", - "value": { - "rev": "1-c53fa04d9c8fb231336704508732c287" - } - }, - - { - "id": "jquery-2-typescript-async-await-adapter", - "key": "jquery-2-typescript-async-await-adapter", - "value": { - "rev": "8-5cfb484e9afaa6e326a97240fccd8f93" - } - }, - - { - "id": "jquery-accessible-accordion-aria", - "key": "jquery-accessible-accordion-aria", - "value": { - "rev": "15-9fc0df7cb2f1cd1001e2da302443b56e" - } - }, - - { - "id": "jquery-accessible-autocomplete-list-aria", - "key": "jquery-accessible-autocomplete-list-aria", - "value": { - "rev": "8-961b382442c1a5bafe58f0e05424701d" - } - }, - - { - "id": "jquery-accessible-carrousel-aria", - "key": "jquery-accessible-carrousel-aria", - "value": { - "rev": "9-f33f59d7f601bafe023bd711b551282b" - } - }, - - { - "id": "jquery-accessible-dialog-tooltip-aria", - "key": "jquery-accessible-dialog-tooltip-aria", - "value": { - "rev": "12-0a7b5ba6f7717c2c6603cabdb29de9ba" - } - }, - - { - "id": "jquery-accessible-hide-show-aria", - "key": "jquery-accessible-hide-show-aria", - "value": { - "rev": "10-5a03c47a8995b08246e4bc103782dafa" - } - }, - - { - "id": "jquery-accessible-modal-window-aria", - "key": "jquery-accessible-modal-window-aria", - "value": { - "rev": "18-50266e260f6b807019cfcfcd3a3685ab" - } - }, - - { - "id": "jquery-accessible-simple-tooltip-aria", - "key": "jquery-accessible-simple-tooltip-aria", - "value": { - "rev": "6-ea71aa37760790dc603b56117f054e1b" - } - }, - - { - "id": "jquery-accessible-subnav-dropdown", - "key": "jquery-accessible-subnav-dropdown", - "value": { - "rev": "2-496f017a9ac243655225e43b5697b09b" - } - }, - - { - "id": "jquery-accessible-tabpanel-aria", - "key": "jquery-accessible-tabpanel-aria", - "value": { - "rev": "11-659971471e6ac0fbb3b2f78ad208722a" - } - }, - - { - "id": "jquery-accessible-tabs-umd", - "key": "jquery-accessible-tabs-umd", - "value": { - "rev": "1-f92015de5bb36e411d8c0940cca2883f" - } - }, - - { - "id": "jquery-active-descendant", - "key": "jquery-active-descendant", - "value": { - "rev": "8-79aed7a6cbca4e1f3c3ac0570d0290de" - } - }, - - { - "id": "jquery-ada-validation", - "key": "jquery-ada-validation", - "value": { - "rev": "1-9aab9629027c29fbece90485dd9d3112" - } - }, - - { - "id": "jquery-adaptText", - "key": "jquery-adaptText", - "value": { - "rev": "3-2e15fc801ea8235b9180a3defc782ed0" - } - }, - - { - "id": "jquery-adapttr", - "key": "jquery-adapttr", - "value": { - "rev": "6-74585f2d4be60b3f493585a6d28b90bc" - } - }, - - { - "id": "jquery-add-prefixed-class", - "key": "jquery-add-prefixed-class", - "value": { - "rev": "1-9e43aee9758504b3f5271e9804a95f20" - } - }, - - { - "id": "jquery-address", - "key": "jquery-address", - "value": { - "rev": "1-64173ede32157b26f4de910ad0f49590" - } - }, - - { - "id": "jquery-address-suggestion", - "key": "jquery-address-suggestion", - "value": { - "rev": "6-18d9df51d472c365bcd84a61c9105774" - } - }, - - { - "id": "jquery-advscrollevent", - "key": "jquery-advscrollevent", - "value": { - "rev": "1-f6033de9ba0f8e364c42826441d93119" - } - }, - - { - "id": "jquery-affix", - "key": "jquery-affix", - "value": { - "rev": "6-777371f67df59abf18ec1fe326df3b82" - } - }, - - { - "id": "jquery-airload", - "key": "jquery-airload", - "value": { - "rev": "7-136d513d2604a25238eb88709d6d9003" - } - }, - - { - "id": "jquery-ajax", - "key": "jquery-ajax", - "value": { - "rev": "1-ee358f630d4c928b52c968c7667d0d31" - } - }, - - { - "id": "jquery-ajax-cache", - "key": "jquery-ajax-cache", - "value": { - "rev": "2-ca31e0d43ae28e9cea968f1f538f06d3" - } - }, - - { - "id": "jquery-ajax-chain", - "key": "jquery-ajax-chain", - "value": { - "rev": "1-dc0e5aee651c0128b7f411aac96132a2" - } - }, - - { - "id": "jquery-ajax-file-upload", - "key": "jquery-ajax-file-upload", - "value": { - "rev": "1-96147d8bf69245c622e76583bb615d49" - } - }, - - { - "id": "jquery-ajax-json", - "key": "jquery-ajax-json", - "value": { - "rev": "1-b47eec12168e4cb39b45f1523d7cd397" - } - }, - - { - "id": "jquery-ajax-markup", - "key": "jquery-ajax-markup", - "value": { - "rev": "1-8e65dc822cb63be76c62a1323666265e" - } - }, - - { - "id": "jquery-ajax-native", - "key": "jquery-ajax-native", - "value": { - "rev": "2-9d67b8d43713e3546ad50f817c040139" - } - }, - - { - "id": "jquery-ajax-request", - "key": "jquery-ajax-request", - "value": { - "rev": "1-fdc0960ec73667bc2b46adf493c05db4" - } - }, - - { - "id": "jquery-ajax-retry", - "key": "jquery-ajax-retry", - "value": { - "rev": "1-27ca186953e346aa9c0ca2310c732751" - } - }, - - { - "id": "jquery-ajax-tracking", - "key": "jquery-ajax-tracking", - "value": { - "rev": "3-d48876f3c115ee4743a6a94bb65bb01d" - } - }, - - { - "id": "jquery-ajax-transport-xdomainrequest", - "key": "jquery-ajax-transport-xdomainrequest", - "value": { - "rev": "1-ece69aa5b9f0c950a1fa2806cf74392d" - } - }, - - { - "id": "jquery-ajax-unobtrusive", - "key": "jquery-ajax-unobtrusive", - "value": { - "rev": "3-fb0daab8480b9a2cc9c6876e1c4874f4" - } - }, - - { - "id": "jquery-ajax-unobtrusive-multi", - "key": "jquery-ajax-unobtrusive-multi", - "value": { - "rev": "1-0a2ffdabaf5708d4ae3d9e29a3a9ef11" - } - }, - - { - "id": "jquery-ajaxreadystate", - "key": "jquery-ajaxreadystate", - "value": { - "rev": "1-5e618474fe2e77ad5869c206164f82bf" - } - }, - - { - "id": "jquery-albe-timeline", - "key": "jquery-albe-timeline", - "value": { - "rev": "2-3db2b43778b5c50db873e724d9940eb6" - } - }, - - { - "id": "jquery-all-attributes", - "key": "jquery-all-attributes", - "value": { - "rev": "1-89bb7e01ee312ad5d36d78a3aa2327e4" - } - }, - - { - "id": "jquery-alphaindex", - "key": "jquery-alphaindex", - "value": { - "rev": "4-7f61cde9cfb70617a6fbe992dfcbc10a" - } - }, - - { - "id": "jquery-always", - "key": "jquery-always", - "value": { - "rev": "1-0ad944881bbc39c67df0a694d80bebef" - } - }, - - { - "id": "jquery-amd", - "key": "jquery-amd", - "value": { - "rev": "1-931646c751bef740c361dd0f6e68653c" - } - }, - - { - "id": "jquery-anaglyph-image-effect", - "key": "jquery-anaglyph-image-effect", - "value": { - "rev": "1-9bf7afce2e1bc73747ef22abc859b22b" - } - }, - - { - "id": "jquery-analytics", - "key": "jquery-analytics", - "value": { - "rev": "1-d84b0c8ce886b9f01d2c5c1cf0a7317f" - } - }, - - { - "id": "jquery-ancestors", - "key": "jquery-ancestors", - "value": { - "rev": "1-49b30817a03558f1f585c8c0cd4b8afb" - } - }, - - { - "id": "jquery-angry-loader", - "key": "jquery-angry-loader", - "value": { - "rev": "1-31c9fd950d32b9d3a73829cde1dae577" - } - }, - - { - "id": "jquery-angular-shim", - "key": "jquery-angular-shim", - "value": { - "rev": "1-723e72b2981f02dd3abcfe6d2395d636" - } - }, - - { - "id": "jquery-animate-gradient", - "key": "jquery-animate-gradient", - "value": { - "rev": "5-a3e0fc89699237e7e7241cd608a0dcf7" - } - }, - - { - "id": "jquery-animate-scroll", - "key": "jquery-animate-scroll", - "value": { - "rev": "1-37d49d89fe99aa599540e6ff83b15888" - } - }, - - { - "id": "jquery-animated-headlines", - "key": "jquery-animated-headlines", - "value": { - "rev": "1-adf1d149bc83fa8445e141e3c900759e" - } - }, - - { - "id": "jquery-animation", - "key": "jquery-animation", - "value": { - "rev": "4-f51d0559010bbe9d74d70e58de9bd733" - } - }, - - { - "id": "jquery-animation-support", - "key": "jquery-animation-support", - "value": { - "rev": "1-9013bc4bdeb2bd70bedcc988a811fcc0" - } - }, - - { - "id": "jquery-aniview", - "key": "jquery-aniview", - "value": { - "rev": "3-5754524da237693458bcff19b626b875" - } - }, - - { - "id": "jquery-anything-clickable", - "key": "jquery-anything-clickable", - "value": { - "rev": "2-e1aaaf1a369f7796c438a3efbf05bcce" - } - }, - - { - "id": "jquery-app", - "key": "jquery-app", - "value": { - "rev": "6-4e0bf5abd71c72ced3c4cf3035116f70" - } - }, - - { - "id": "jquery-app-banner", - "key": "jquery-app-banner", - "value": { - "rev": "2-8a5b530eaab94315eb00c77acd13f2dd" - } - }, - - { - "id": "jquery-appear-poetic", - "key": "jquery-appear-poetic", - "value": { - "rev": "1-368094b72ed36d42cf2fca438fa4b344" - } - }, - - { - "id": "jquery-applyonscreen", - "key": "jquery-applyonscreen", - "value": { - "rev": "4-d76c18a6e66fffba01a9a774b40663f8" - } - }, - - { - "id": "jquery-apta", - "key": "jquery-apta", - "value": { - "rev": "1-c486380fedefd887e6293a00c3b6a222" - } - }, - - { - "id": "jquery-arrow-navigate", - "key": "jquery-arrow-navigate", - "value": { - "rev": "3-0efe881e01ef0eac24a92baf1eb6d8d1" - } - }, - - { - "id": "jquery-asAccordion", - "key": "jquery-asAccordion", - "value": { - "rev": "2-2d18d3fe9089dcf67de5f29d1763b4ce" - } - }, - - { - "id": "jquery-asBgPicker", - "key": "jquery-asBgPicker", - "value": { - "rev": "2-d1403cd306d5764ee0f5aa852c2bed8e" - } - }, - - { - "id": "jquery-asBreadcrumbs", - "key": "jquery-asBreadcrumbs", - "value": { - "rev": "2-77e566a07680005ce1cb322f2a733fe4" - } - }, - - { - "id": "jquery-asCheck", - "key": "jquery-asCheck", - "value": { - "rev": "2-d0b2741b70616c7d563419cc125d193d" - } - }, - - { - "id": "jquery-asChoice", - "key": "jquery-asChoice", - "value": { - "rev": "2-0eda5269cbd59976ee904b74da209389" - } - }, - - { - "id": "jquery-asColor", - "key": "jquery-asColor", - "value": { - "rev": "3-aa730d81322561c7a3174d5c7bb6b3b8" - } - }, - - { - "id": "jquery-asColorPicker", - "key": "jquery-asColorPicker", - "value": { - "rev": "2-6bbaecaf94a324331a3d1f5d3aad3b3d" - } - }, - - { - "id": "jquery-asDropdown", - "key": "jquery-asDropdown", - "value": { - "rev": "2-b29b187cdd0bdce502d11855415e6887" - } - }, - - { - "id": "jquery-asFontEditor", - "key": "jquery-asFontEditor", - "value": { - "rev": "2-132882375101062896413afdc93b4c8c" - } - }, - - { - "id": "jquery-asGalleryPicker", - "key": "jquery-asGalleryPicker", - "value": { - "rev": "1-864a80930d72c6150aa74969a28617e4" - } - }, - - { - "id": "jquery-asGmap", - "key": "jquery-asGmap", - "value": { - "rev": "2-b0c4330774137b2f1b91bd4686880f2a" - } - }, - - { - "id": "jquery-asGradient", - "key": "jquery-asGradient", - "value": { - "rev": "2-5184670a313d5e161cb62659de3db55c" - } - }, - - { - "id": "jquery-asHoverScroll", - "key": "jquery-asHoverScroll", - "value": { - "rev": "7-3f6efebf248bd27520d03eaac33d8ca2" - } - }, - - { - "id": "jquery-asIconPicker", - "key": "jquery-asIconPicker", - "value": { - "rev": "2-9070adda148ea75247c7cee810ae91e2" - } - }, - - { - "id": "jquery-asImagePicker", - "key": "jquery-asImagePicker", - "value": { - "rev": "2-fb3115c2296b0b07ed9e379176626e01" - } - }, - - { - "id": "jquery-asItemList", - "key": "jquery-asItemList", - "value": { - "rev": "2-88a7d2900f47c785c2a6cb764ac467d6" - } - }, - - { - "id": "jquery-asModal", - "key": "jquery-asModal", - "value": { - "rev": "2-1719b8e6a489e03cc3e22bd329148366" - } - }, - - { - "id": "jquery-asOffset", - "key": "jquery-asOffset", - "value": { - "rev": "2-e45a0077e5bc0bbf91b32dc76387c945" - } - }, - - { - "id": "jquery-asPaginator", - "key": "jquery-asPaginator", - "value": { - "rev": "2-0d279d2748fc5e875f5fb2a8d3d48377" - } - }, - - { - "id": "jquery-asPieProgress", - "key": "jquery-asPieProgress", - "value": { - "rev": "2-14dc464a19e9d3feaa532f62e45bbd26" - } - }, - - { - "id": "jquery-asProgress", - "key": "jquery-asProgress", - "value": { - "rev": "2-a58d7100f1a78f7753efcf0e34dfaf0e" - } - }, - - { - "id": "jquery-asRange", - "key": "jquery-asRange", - "value": { - "rev": "3-aa3d2f348a933161868ba6b6fd9eb881" - } - }, - - { - "id": "jquery-asScroll", - "key": "jquery-asScroll", - "value": { - "rev": "1-f4880ea057adbfebb912ba0157575ca1" - } - }, - - { - "id": "jquery-asScrollable", - "key": "jquery-asScrollable", - "value": { - "rev": "7-5c18eb2180d8aa85f0b5e940667c8344" - } - }, - - { - "id": "jquery-asScrollbar", - "key": "jquery-asScrollbar", - "value": { - "rev": "4-89420658c355a5584825b45ee4ef0beb" - } - }, - - { - "id": "jquery-asSelect", - "key": "jquery-asSelect", - "value": { - "rev": "2-caf3dc516665009b654236b876fe02bb" - } - }, - - { - "id": "jquery-asSpinner", - "key": "jquery-asSpinner", - "value": { - "rev": "2-bf26b5d9c77eb4b63acbf16019407834" - } - }, - - { - "id": "jquery-asSwitch", - "key": "jquery-asSwitch", - "value": { - "rev": "2-f738586946b432caa73297568b5f38ad" - } - }, - - { - "id": "jquery-asTooltip", - "key": "jquery-asTooltip", - "value": { - "rev": "2-80d3fe5cdae70d9310969723e7045384" - } - }, - - { - "id": "jquery-asTree", - "key": "jquery-asTree", - "value": { - "rev": "2-353063a563c0322cbc317af385f71b27" - } - }, - - { - "id": "jquery-ascolorpicker-flat", - "key": "jquery-ascolorpicker-flat", - "value": { - "rev": "11-1681d53cd475e7b6b9564baa51a79611" - } - }, - - { - "id": "jquery-aslider", - "key": "jquery-aslider", - "value": { - "rev": "1-2b3dd953493eeaa4dc329cbf0d81116a" - } - }, - - { - "id": "jquery-aspect-ratio-keeper", - "key": "jquery-aspect-ratio-keeper", - "value": { - "rev": "1-1ad8e5588218e1d38fff351858655eda" - } - }, - - { - "id": "jquery-assinadordigitaldiscus", - "key": "jquery-assinadordigitaldiscus", - "value": { - "rev": "1-897cd68ef3699551630bd3454dceb6f0" - } - }, - - { - "id": "jquery-async-gravatar", - "key": "jquery-async-gravatar", - "value": { - "rev": "3-a3192e741d14d57635f4ebfb41a904db" - } - }, - - { - "id": "jquery-asynclink", - "key": "jquery-asynclink", - "value": { - "rev": "1-2159a3c49e3c8fe9280c592770e83522" - } - }, - - { - "id": "jquery-atlas", - "key": "jquery-atlas", - "value": { - "rev": "1-6142c5a0af67a0470daf36151d3f9d8c" - } - }, - - { - "id": "jquery-atomic-nav", - "key": "jquery-atomic-nav", - "value": { - "rev": "1-18e4ef14be83a907cbee0cd0adee25d4" - } - }, - - { - "id": "jquery-attach", - "key": "jquery-attach", - "value": { - "rev": "8-da4f17596c25a02b0cce266e59706d5f" - } - } - - ] -} diff --git "a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"\",limit=11" "b/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"\",limit=11" deleted file mode 100644 index b6c202f..0000000 --- "a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"\",limit=11" +++ /dev/null @@ -1,83 +0,0 @@ -{ - "total_rows": 839080, - "offset": 422482, - "rows": [ - { - "id": "jquery", - "key": "jquery", - "value": { - "rev": "212-2eac7c93af4c8bccdf7317739f0319b6" - } - }, - { - "id": "jquery-1.8", - "key": "jquery-1.8", - "value": { - "rev": "1-711ded49a7453adce85ce7a51c2157de" - } - }, - { - "id": "jquery-1x", - "key": "jquery-1x", - "value": { - "rev": "1-c53fa04d9c8fb231336704508732c287" - } - }, - { - "id": "jquery-2-typescript-async-await-adapter", - "key": "jquery-2-typescript-async-await-adapter", - "value": { - "rev": "8-5cfb484e9afaa6e326a97240fccd8f93" - } - }, - { - "id": "jquery-accessible-accordion-aria", - "key": "jquery-accessible-accordion-aria", - "value": { - "rev": "15-9fc0df7cb2f1cd1001e2da302443b56e" - } - }, - { - "id": "jquery-accessible-autocomplete-list-aria", - "key": "jquery-accessible-autocomplete-list-aria", - "value": { - "rev": "8-961b382442c1a5bafe58f0e05424701d" - } - }, - { - "id": "jquery-accessible-carrousel-aria", - "key": "jquery-accessible-carrousel-aria", - "value": { - "rev": "9-f33f59d7f601bafe023bd711b551282b" - } - }, - { - "id": "jquery-accessible-dialog-tooltip-aria", - "key": "jquery-accessible-dialog-tooltip-aria", - "value": { - "rev": "12-0a7b5ba6f7717c2c6603cabdb29de9ba" - } - }, - { - "id": "jquery-accessible-hide-show-aria", - "key": "jquery-accessible-hide-show-aria", - "value": { - "rev": "10-5a03c47a8995b08246e4bc103782dafa" - } - }, - { - "id": "jquery-accessible-modal-window-aria", - "key": "jquery-accessible-modal-window-aria", - "value": { - "rev": "18-50266e260f6b807019cfcfcd3a3685ab" - } - }, - { - "id": "jquery-accessible-simple-tooltip-aria", - "key": "jquery-accessible-simple-tooltip-aria", - "value": { - "rev": "6-ea71aa37760790dc603b56117f054e1b" - } - } - ] -} diff --git "a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"jquery-accessible-simple-tooltip-aria\",limit=11" "b/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"jquery-accessible-simple-tooltip-aria\",limit=11" deleted file mode 100644 index 0d73af8..0000000 --- "a/swh/lister/npm/tests/data/https_replicate.npmjs.com/_all_docs,startkey=\"jquery-accessible-simple-tooltip-aria\",limit=11" +++ /dev/null @@ -1,83 +0,0 @@ -{ - "total_rows": 839080, - "offset": 422482, - "rows": [ - { - "id": "jquery-accessible-simple-tooltip-aria", - "key": "jquery-accessible-simple-tooltip-aria", - "value": { - "rev": "6-ea71aa37760790dc603b56117f054e1b" - } - }, - { - "id": "jquery-accessible-subnav-dropdown", - "key": "jquery-accessible-subnav-dropdown", - "value": { - "rev": "2-496f017a9ac243655225e43b5697b09b" - } - }, - { - "id": "jquery-accessible-tabpanel-aria", - "key": "jquery-accessible-tabpanel-aria", - "value": { - "rev": "11-659971471e6ac0fbb3b2f78ad208722a" - } - }, - { - "id": "jquery-accessible-tabs-umd", - "key": "jquery-accessible-tabs-umd", - "value": { - "rev": "1-f92015de5bb36e411d8c0940cca2883f" - } - }, - { - "id": "jquery-active-descendant", - "key": "jquery-active-descendant", - "value": { - "rev": "8-79aed7a6cbca4e1f3c3ac0570d0290de" - } - }, - { - "id": "jquery-ada-validation", - "key": "jquery-ada-validation", - "value": { - "rev": "1-9aab9629027c29fbece90485dd9d3112" - } - }, - { - "id": "jquery-adaptText", - "key": "jquery-adaptText", - "value": { - "rev": "3-2e15fc801ea8235b9180a3defc782ed0" - } - }, - { - "id": "jquery-adapttr", - "key": "jquery-adapttr", - "value": { - "rev": "6-74585f2d4be60b3f493585a6d28b90bc" - } - }, - { - "id": "jquery-add-prefixed-class", - "key": "jquery-add-prefixed-class", - "value": { - "rev": "1-9e43aee9758504b3f5271e9804a95f20" - } - }, - { - "id": "jquery-address", - "key": "jquery-address", - "value": { - "rev": "1-64173ede32157b26f4de910ad0f49590" - } - }, - { - "id": "jquery-address-suggestion", - "key": "jquery-address-suggestion", - "value": { - "rev": "6-18d9df51d472c365bcd84a61c9105774" - } - } - ] -} diff --git a/swh/lister/npm/tests/data/https_replicate.npmjs.com/api_response.json b/swh/lister/npm/tests/data/https_replicate.npmjs.com/api_response.json deleted file mode 120000 index 21ff340..0000000 --- a/swh/lister/npm/tests/data/https_replicate.npmjs.com/api_response.json +++ /dev/null @@ -1 +0,0 @@ -_all_docs,startkey="",limit=1001 \ No newline at end of file diff --git a/swh/lister/npm/tests/data/npm_full_page1.json b/swh/lister/npm/tests/data/npm_full_page1.json new file mode 100644 index 0000000..9fc0447 --- /dev/null +++ b/swh/lister/npm/tests/data/npm_full_page1.json @@ -0,0 +1,235 @@ +{ + "total_rows": 1496481, + "offset": 0, + "rows": [{ + "id": "-", + "key": "-", + "value": { + "rev": "1-e340e64b090ea8c6b4e14edc0460c751" + }, + "doc": { + "_id": "-", + "_rev": "1-e340e64b090ea8c6b4e14edc0460c751", + "name": "-", + "dist-tags": { + "latest": "0.0.1" + }, + "versions": { + "0.0.1": { + "name": "-", + "version": "0.0.1", + "license": "UNLICENSED", + "keywords": [], + "main": "dist/index.js", + "types": "dist/index.d.ts", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1", + "start": "node --require ts-node/register src", + "build": "tsc" + }, + "devDependencies": { + "@types/node": "13.9.0", + "ts-node": "8.6.2", + "typescript": "3.8.3" + }, + "description": "> Created using https://github.com/parzh/create-package-typescript", + "_id": "-@0.0.1", + "_nodeVersion": "13.3.0", + "_npmVersion": "6.14.4", + "dist": { + "integrity": "sha512-3HfneK3DGAm05fpyj20sT3apkNcvPpCuccOThOPdzz8sY7GgQGe0l93XH9bt+YzibcTIgUAIMoyVJI740RtgyQ==", + "shasum": "db6db7cd866142880dd03e5b8781d1b4fac0e5bd", + "tarball": "https://registry.npmjs.org/-/-/--0.0.1.tgz", + "fileCount": 3, + "unpackedSize": 600, + "npm-signature": "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJehdEkCRA9TVsSAnZWagAAqcEQAIUCCWB2RzNV2p5CTeWf\nntknmFRbGh3K/gptyBnTC3MaNtk54Vf36XK1BfKgKz2rd/FOq1xcSQ8XUWt6\n7ZL9zSR3K/EBEgNr6p00fJAJ7ZdYu2kqq+uiZDKpSHLd6/bKaSdt0bAL78co\nmnzKOn9WULS1UC/ac8txNX9QdPg/aJezzxQadtKQR+SX6Fd/tTHKIsWv3ytI\nUtH/Lg9AM/tdTc94m2Ivo3+JxmkU+p8z7hjR97ym+AaZE8g1a47begVsSeZ/\nrIu1ZFCQ/XYFp0amNZ131qnW6FGnzpPlQTfkuJ1sPIORM9+o+6Ql6qa0mtde\nSJyAMKFfemmwkp56aZkL5NFX1snyJE4itfg99gsUrogHZIW3Jd1LqUe9yFZc\nVik1NZxqG4x6ktoog9IzG8ElOBWauKQAT+FIgNGea6IBv4wPQ8VoNOd4JKW/\nadvzus0hAdEZLyVsCyDw4wDCAK9JJXnSlJmJFc/vPtgPrbAj8xcBW6XIn6Nw\n0EVm993zlKmQ6/VmQUbHx8EP1zBm7M0vTcoI8dPIJkjGCZgWU571JARCetb2\npob+siYon3ndpoHG2OOWOuoRhwi4YeDW1ilc9isPve+S/iygKnfY8LTqHN1P\nOntz9Cu9YxHQ3WEiBuun7KP7FECbiSkozmzKWLHUhs2qXFSFYWeBLEI/HS1y\nJ9QO\r\n=lB+Q\r\n-----END PGP SIGNATURE-----\r\n" + }, + "maintainers": [{ + "name": "parzhitsky", + "email": "parzhitsky@gmail.com" + }], + "_npmUser": { + "name": "parzhitsky", + "email": "parzhitsky@gmail.com" + }, + "directories": {}, + "_npmOperationalInternal": { + "host": "s3://npm-registry-packages", + "tmp": "tmp/-_0.0.1_1585828132159_0.6185838126718755" + }, + "_hasShrinkwrap": false + } + }, + "time": { + "created": "2020-04-02T11:48:52.159Z", + "0.0.1": "2020-04-02T11:48:52.339Z", + "modified": "2020-04-02T11:49:00.248Z" + }, + "maintainers": [{ + "name": "parzhitsky", + "email": "parzhitsky@gmail.com" + }], + "description": "> Created using https://github.com/parzh/create-package-typescript", + "keywords": [], + "license": "UNLICENSED", + "readme": "# `-`\r\n\r\n> Created using https://github.com/parzh/create-package-typescript\r\n\r\n```\r\nnpm i -\r\n```\r\n", + "readmeFilename": "README.md" + } + }, + { + "id": "--file--1", + "key": "--file--1", + "value": { + "rev": "1-b23c0a584f925336378c29df0b858aff" + }, + "doc": { + "_id": "--file--1", + "_rev": "1-b23c0a584f925336378c29df0b858aff", + "name": "--file--1", + "dist-tags": { + "latest": "1.0.0" + }, + "versions": { + "1.0.0": { + "name": "--file--1", + "version": "1.0.0", + "description": "", + "main": "file.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "_id": "--file--1@1.0.0", + "_nodeVersion": "15.3.0", + "_npmVersion": "7.0.14", + "dist": { + "integrity": "sha512-OtGGxxqxDl1KCCUts3fN0gL4vW28u8WNIo+eqQON78IVxKJS3gYchaEZDrRubC4SyQOhDDKDtvswwLOkApuicQ==", + "shasum": "58ed9d9169197ded653d8df7153968d22c14ea59", + "tarball": "https://registry.npmjs.org/--file--1/-/--file--1-1.0.0.tgz", + "fileCount": 6, + "unpackedSize": 1327, + "npm-signature": "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJf2AGJCRA9TVsSAnZWagAALqcP+wZ9tzFsdQy4cEb5HGV6\nxV0LTnW+wxFYBUm2IfOwK7UH/Pl5Nym4yxlIjMEBinb9efepUAsucCyAka/v\n0OBGn+8I8kHDKvfhxiUVbi3CFAYcQTaaPffK5xJZi77netdsaLyS0cdx4RBm\nEWRIVK4axJBSUT6UVpmRqB73OlAv+JmNzBmL+CXHn9gksyvnm74pbT8tukLV\nX0TwkXNBtzn4lmydPRyecEZRDFydzp9UBQJyJACtpYxgTAwTT5ENtSymx4Ql\nlEqC9cqT9Q3n7z8lvntaCb6jgNN1aSWP3hOKCYQzezk5uSacJDWJOGAK7DE5\ndKUqsWa0HG84H+MMLxuepbrNYM5Fm7ulykl1yMrAf1G8T5pU/fUpq5uEHbqW\naVbW3GlCxJuUvlaTTFjGqDJx4tepY+Uel+TQigJoN2DGXTWiB+JeAAWLP+cu\nqcoZwkqoISmgru2g1+6TR//hod3CnCK4eQl/ph3VthtKyh4d1zUQAPcCGax6\nocJSWQsLRNW0QUmvpHeHQQOUGEb0XDTYkl9i8s8DY6JCqfgl9UIDZzNl2nK8\n385JseJKKLQR3tEk+HLZM2UGF6XgACBFTHYxlOrpzGgX+PJ1ns3noMrKo2jO\nbnvMN6Mp0GtoSzQj8FUcoU6BgOFHhHkB/i4Ke2VVeMYVj/CrFO13pZJOPR4O\n6uts\r\n=W7vP\r\n-----END PGP SIGNATURE-----\r\n" + }, + "_npmUser": { + "name": "guosuzhen", + "email": "1077655229@qq.com" + }, + "directories": {}, + "maintainers": [{ + "name": "guosuzhen", + "email": "1077655229@qq.com" + }], + "_npmOperationalInternal": { + "host": "s3://npm-registry-packages", + "tmp": "tmp/--file--1_1.0.0_1607991689533_0.6331916469772794" + }, + "_hasShrinkwrap": false + } + }, + "time": { + "created": "2020-12-15T00:21:29.533Z", + "1.0.0": "2020-12-15T00:21:29.657Z", + "modified": "2020-12-15T00:21:32.116Z" + }, + "maintainers": [{ + "name": "guosuzhen", + "email": "1077655229@qq.com" + }], + "keywords": [], + "license": "ISC", + "readme": "ERROR: No README data found!", + "readmeFilename": "" + } + }, + { + "id": "-keyboardevent", + "key": "-keyboardevent", + "value": { + "rev": "1-0fdff278618c4f8c1026889f1b525483" + }, + "doc": { + "_id": "-keyboardevent", + "_rev": "1-0fdff278618c4f8c1026889f1b525483", + "name": "-keyboardevent", + "dist-tags": { + "latest": "1.0.0" + }, + "versions": { + "1.0.0": { + "name": "-keyboardevent", + "version": "1.0.0", + "description": "监听键盘事件", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/RegExpRegExp/-KeyboardEvent.git" + }, + "author": { + "name": "yanggw" + }, + "license": "ISO", + "bugs": { + "url": "https://github.com/RegExpRegExp/-KeyboardEvent/issues" + }, + "homepage": "https://github.com/RegExpRegExp/-KeyboardEvent#readme", + "gitHead": "e8c5683b98d68b915eeb79656ce6717efca676b3", + "_id": "-keyboardevent@1.0.0", + "_nodeVersion": "12.10.0", + "_npmVersion": "6.10.3", + "dist": { + "integrity": "sha512-8G0t8AQlDyfE2xGIXpte6gFQs2lr48BoQYMGHTytshClyKUfzmX7YHYLxerqX2rp82KgsPCgq1r/3yC2UTmr/g==", + "shasum": "b81a787d8748ffaf882a73f3ae47f1d38a8da5d5", + "tarball": "https://registry.npmjs.org/-keyboardevent/-/-keyboardevent-1.0.0.tgz", + "fileCount": 3, + "unpackedSize": 2714, + "npm-signature": "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfbYnwCRA9TVsSAnZWagAAtDgP/0LCWxCot5zaM4KZXv/6\nlAdCfVDpFhmsV7I5v0pvVu4aY2OQt5VnPQD6nSfrlKflLSB88+Asoi4X2Xhh\nT1/zo8GE+RFUJ98Kh/1H53+6KQiMEZVWlxKMf9LxgBy8TRVcpZaj9BNCr7wG\nZ2t+NR6LaVvb1WkEB0ZtWzbHA3IMucIibkLC4W/JIZtzjmPl4FWUr1Aq/BMS\nr4dd7WROxObns+Na8p10UAEGYkEZjL68L7g2gm7u/70nCV+IvG/Q5ACufxxM\npxJycEto6v/3LbEmNxocnKZaYwLFc7Mhx5zIytvGS29e1FEoRx3Qyyq5opMA\nANWuv1zq1r/m4XtPL5ZN0PtkIRdWHz4xIdOVEI9GuCfFxiY/vd4/vYDWNUZL\nNk11LGR0VYeJaPYCdz0BrYK2R425/LVThkvy/n2EWQ81gpsnVDeerwVfNj2L\n6GsTUqceDnLLNEjhdQEGsCvMsF72974W0bolXdJjwKx8z+30lUmzOIvXPNrB\nt/O6kDCerhd0aMT6KADNMtijci9Mnm2AovcwWa9M8jfw2LsF2zbXYPfZ1CIK\nDbhgF9QUWZyOT2rahkN7Hn7G7OH8dndiuAa+liggB2DJkhZS1XelMJfShGJj\nMYmZ9LYykLocD3gbi4+pE/Y200UlE+MNrc8Jp+MWaXbwjTkfj+V/mAao0Q3z\nIL5/\r\n=Ye2g\r\n-----END PGP SIGNATURE-----\r\n" + }, + "maintainers": [{ + "name": "yanggw", + "email": "2993065178@qq.com" + }], + "_npmUser": { + "name": "yanggw", + "email": "2993065178@qq.com" + }, + "directories": {}, + "_npmOperationalInternal": { + "host": "s3://npm-registry-packages", + "tmp": "tmp/-keyboardevent_1.0.0_1601014256394_0.736974638502927" + }, + "_hasShrinkwrap": false + } + }, + "time": { + "created": "2020-09-25T06:10:56.357Z", + "1.0.0": "2020-09-25T06:10:56.513Z", + "modified": "2020-09-25T06:10:58.774Z" + }, + "maintainers": [{ + "name": "yanggw", + "email": "2993065178@qq.com" + }], + "description": "监听键盘事件", + "homepage": "https://github.com/RegExpRegExp/-KeyboardEvent#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/RegExpRegExp/-KeyboardEvent.git" + }, + "author": { + "name": "yanggw" + }, + "bugs": { + "url": "https://github.com/RegExpRegExp/-KeyboardEvent/issues" + }, + "license": "ISO", + "readme": "# -KeyboardEvent\r\n监听键盘事件\r\n", + "readmeFilename": "README.md" + } + } + ] +} \ No newline at end of file diff --git a/swh/lister/npm/tests/data/npm_full_page2.json b/swh/lister/npm/tests/data/npm_full_page2.json new file mode 100644 index 0000000..9175415 --- /dev/null +++ b/swh/lister/npm/tests/data/npm_full_page2.json @@ -0,0 +1,168 @@ +{ + "total_rows": 1496482, + "offset": 2, + "rows": [{ + "id": "-keyboardevent", + "key": "-keyboardevent", + "value": { + "rev": "1-0fdff278618c4f8c1026889f1b525483" + }, + "doc": { + "_id": "-keyboardevent", + "_rev": "1-0fdff278618c4f8c1026889f1b525483", + "name": "-keyboardevent", + "dist-tags": { + "latest": "1.0.0" + }, + "versions": { + "1.0.0": { + "name": "-keyboardevent", + "version": "1.0.0", + "description": "监听键盘事件", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/RegExpRegExp/-KeyboardEvent.git" + }, + "author": { + "name": "yanggw" + }, + "license": "ISO", + "bugs": { + "url": "https://github.com/RegExpRegExp/-KeyboardEvent/issues" + }, + "homepage": "https://github.com/RegExpRegExp/-KeyboardEvent#readme", + "gitHead": "e8c5683b98d68b915eeb79656ce6717efca676b3", + "_id": "-keyboardevent@1.0.0", + "_nodeVersion": "12.10.0", + "_npmVersion": "6.10.3", + "dist": { + "integrity": "sha512-8G0t8AQlDyfE2xGIXpte6gFQs2lr48BoQYMGHTytshClyKUfzmX7YHYLxerqX2rp82KgsPCgq1r/3yC2UTmr/g==", + "shasum": "b81a787d8748ffaf882a73f3ae47f1d38a8da5d5", + "tarball": "https://registry.npmjs.org/-keyboardevent/-/-keyboardevent-1.0.0.tgz", + "fileCount": 3, + "unpackedSize": 2714, + "npm-signature": "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJfbYnwCRA9TVsSAnZWagAAtDgP/0LCWxCot5zaM4KZXv/6\nlAdCfVDpFhmsV7I5v0pvVu4aY2OQt5VnPQD6nSfrlKflLSB88+Asoi4X2Xhh\nT1/zo8GE+RFUJ98Kh/1H53+6KQiMEZVWlxKMf9LxgBy8TRVcpZaj9BNCr7wG\nZ2t+NR6LaVvb1WkEB0ZtWzbHA3IMucIibkLC4W/JIZtzjmPl4FWUr1Aq/BMS\nr4dd7WROxObns+Na8p10UAEGYkEZjL68L7g2gm7u/70nCV+IvG/Q5ACufxxM\npxJycEto6v/3LbEmNxocnKZaYwLFc7Mhx5zIytvGS29e1FEoRx3Qyyq5opMA\nANWuv1zq1r/m4XtPL5ZN0PtkIRdWHz4xIdOVEI9GuCfFxiY/vd4/vYDWNUZL\nNk11LGR0VYeJaPYCdz0BrYK2R425/LVThkvy/n2EWQ81gpsnVDeerwVfNj2L\n6GsTUqceDnLLNEjhdQEGsCvMsF72974W0bolXdJjwKx8z+30lUmzOIvXPNrB\nt/O6kDCerhd0aMT6KADNMtijci9Mnm2AovcwWa9M8jfw2LsF2zbXYPfZ1CIK\nDbhgF9QUWZyOT2rahkN7Hn7G7OH8dndiuAa+liggB2DJkhZS1XelMJfShGJj\nMYmZ9LYykLocD3gbi4+pE/Y200UlE+MNrc8Jp+MWaXbwjTkfj+V/mAao0Q3z\nIL5/\r\n=Ye2g\r\n-----END PGP SIGNATURE-----\r\n" + }, + "maintainers": [{ + "name": "yanggw", + "email": "2993065178@qq.com" + }], + "_npmUser": { + "name": "yanggw", + "email": "2993065178@qq.com" + }, + "directories": {}, + "_npmOperationalInternal": { + "host": "s3://npm-registry-packages", + "tmp": "tmp/-keyboardevent_1.0.0_1601014256394_0.736974638502927" + }, + "_hasShrinkwrap": false + } + }, + "time": { + "created": "2020-09-25T06:10:56.357Z", + "1.0.0": "2020-09-25T06:10:56.513Z", + "modified": "2020-09-25T06:10:58.774Z" + }, + "maintainers": [{ + "name": "yanggw", + "email": "2993065178@qq.com" + }], + "description": "监听键盘事件", + "homepage": "https://github.com/RegExpRegExp/-KeyboardEvent#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/RegExpRegExp/-KeyboardEvent.git" + }, + "author": { + "name": "yanggw" + }, + "bugs": { + "url": "https://github.com/RegExpRegExp/-KeyboardEvent/issues" + }, + "license": "ISO", + "readme": "# -KeyboardEvent\r\n监听键盘事件\r\n", + "readmeFilename": "README.md" + } + }, + { + "id": "-lidonghui", + "key": "-lidonghui", + "value": { + "rev": "1-1aa03a85878671cdb74b789c2b124578" + }, + "doc": { + "_id": "-lidonghui", + "_rev": "1-1aa03a85878671cdb74b789c2b124578", + "name": "-lidonghui", + "dist-tags": { + "latest": "1.0.0" + }, + "versions": { + "1.0.0": { + "name": "-lidonghui", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "bin": { + "test1-cli": "bin/index.js" + }, + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "axios": "^0.19.2", + "commander": "^5.0.0", + "inquirer": "^7.1.0" + }, + "_id": "-lidonghui@1.0.0", + "_nodeVersion": "12.14.0", + "_npmVersion": "6.13.4", + "dist": { + "integrity": "sha512-1eUTEDU1kgAAsQ85uCEAp9tDsEIh3WNa5dB/ks8Uhf9Nh2RR37EeTGyxszn2A1B+MN+Q0fWrpCfBFzrrFyLFnA==", + "shasum": "be7ffedc328bd123ba13c468a89ebcf0cfe76777", + "tarball": "https://registry.npmjs.org/-lidonghui/-/-lidonghui-1.0.0.tgz", + "fileCount": 2, + "unpackedSize": 1024, + "npm-signature": "-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.4\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJenVH+CRA9TVsSAnZWagAAEIMP/ipex7Yo+T2LqfWtFuZz\n5NuXhcnaNyANXyJ7iQCYzq1ynq+tkwBOTM7FNLkTbIiIUsn8d6JlAt5CfIFZ\nrfaYDK7BhU4E8KfLoSC0VqyaprTSvAeqeQDEAguYM9Nbt4TWa7W2HoorgQd7\njUAuGpRqvkw8Omh3eOvhqEa0Y5mkRf853ndJHD425bAww2R0KCA7RuOBO1MW\nnPZqBe3XOgJmsyEW+yl2tSGVq2hEEeD3C0G9x5K2ZfxZKVU7lGfqRlUdo7na\nTLJU6YZPTB9AhDTM9b8XFUMWlEtvHcwvGvuB+gLupXqWwEfwSqWg/2TJ11AQ\nMBFeC1xSav0BCv6kFxFBRwz1k7IAdVYatLnmqwNsIGwjWsyfmIThSNzYTIq6\nE/E0iD2gSAsLW5r7U0t+rk4sWLmu4xbLDvT1xpEY7C00Zj4NXJhdXSo5BR45\n1IBiU96LqEA2bIw+TsyaFO7w/chgpvd8mU0g34tYQCcAts56ZhCr43x5kUSY\ns7hm1b3MB5qpOBbom1Ta24xjvoJwxv2Et/T7zfZoXzHDyX9BgUCiVIvupxd9\nzR5LdyXIh6JdiVBAgEftbm59FEVrxVfAAi/sFMNJKp8pE1ELrSp/gulzxDd2\nC5Hq1z5RNFRQUlbkMJHLIDOZLAvdkPKieE+uFJ7WTYCPEL9nvQoIJiO9UFH1\njKtU\r\n=Opv1\r\n-----END PGP SIGNATURE-----\r\n" + }, + "maintainers": [{ + "name": "tenderli", + "email": "l2369660482@163.com" + }], + "_npmUser": { + "name": "tenderli", + "email": "l2369660482@163.com" + }, + "directories": {}, + "_npmOperationalInternal": { + "host": "s3://npm-registry-packages", + "tmp": "tmp/-lidonghui_1.0.0_1587368445998_0.028137547296155807" + }, + "_hasShrinkwrap": false + } + }, + "time": { + "created": "2020-04-20T07:40:45.997Z", + "1.0.0": "2020-04-20T07:40:46.087Z", + "modified": "2020-04-20T07:40:48.525Z" + }, + "maintainers": [{ + "name": "tenderli", + "email": "l2369660482@163.com" + }], + "keywords": [], + "license": "ISC", + "readme": "ERROR: No README data found!", + "readmeFilename": "" + } + } + ] +} \ No newline at end of file diff --git a/swh/lister/npm/tests/data/npm_incremental_page1.json b/swh/lister/npm/tests/data/npm_incremental_page1.json new file mode 100644 index 0000000..e438eca --- /dev/null +++ b/swh/lister/npm/tests/data/npm_incremental_page1.json @@ -0,0 +1,175 @@ +{ + "results": [{ + "seq": 1, + "id": "overlay-fishnet", + "changes": [{ + "rev": "1-ae6eba8bb1a34b9d57d7285f097c5c9f" + }], + "doc": { + "_id": "overlay-fishnet", + "_rev": "1-ae6eba8bb1a34b9d57d7285f097c5c9f", + "name": "overlay-fishnet", + "description": "Overlay polygon with equal sized tiles (like a fishnet)", + "dist-tags": { + "latest": "0.0.0" + }, + "versions": { + "0.0.0": { + "name": "overlay-fishnet", + "version": "0.0.0", + "description": "Overlay polygon with equal sized tiles (like a fishnet)", + "main": "dist/index.js", + "scripts": { + "build": "rm -rf dist && mkdir dist && babel src --out-dir dist", + "prepublish": "npm run test && npm run build", + "test": "standard" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/yongjun21/fishnet.git" + }, + "keywords": [ + "geospatial", + "polygon", + "tiles", + "choropleth" + ], + "author": { + "name": "Yong Jun" + }, + "license": "ISC", + "bugs": { + "url": "https://github.com/yongjun21/fishnet/issues" + }, + "homepage": "https://github.com/yongjun21/fishnet#readme", + "dependencies": { + "babel-cli": "^6.18.0", + "babel-preset-es2015": "^6.18.0", + "babel-preset-stage-3": "^6.17.0", + "standard": "^8.6.0" + }, + "gitHead": "bc3d598445a767efefd094260c83d6f57f06e86d", + "_id": "overlay-fishnet@0.0.0", + "_shasum": "8fff6fa32c585f5d00a8bd5c0c332d76d69342d0", + "_from": ".", + "_npmVersion": "3.10.7", + "_nodeVersion": "6.2.2", + "_npmUser": { + "name": "yongjun21", + "email": "yjthong.2009@smu.edu.sg" + }, + "dist": { + "shasum": "8fff6fa32c585f5d00a8bd5c0c332d76d69342d0", + "tarball": "https://registry.npmjs.org/overlay-fishnet/-/overlay-fishnet-0.0.0.tgz" + }, + "maintainers": [{ + "name": "yongjun21", + "email": "yjthong.2009@smu.edu.sg" + }], + "_npmOperationalInternal": { + "host": "packages-12-west.internal.npmjs.com", + "tmp": "tmp/overlay-fishnet-0.0.0.tgz_1484192243466_0.4572450101841241" + } + } + }, + "readme": "# fishnet\nOverlay polygon with equal sized tiles (like a fishnet)\n", + "maintainers": [{ + "name": "yongjun21", + "email": "yjthong.2009@smu.edu.sg" + }], + "time": { + "modified": "2017-01-12T03:37:23.696Z", + "created": "2017-01-12T03:37:23.696Z", + "0.0.0": "2017-01-12T03:37:23.696Z" + }, + "homepage": "https://github.com/yongjun21/fishnet#readme", + "keywords": [ + "geospatial", + "polygon", + "tiles", + "choropleth" + ], + "repository": { + "type": "git", + "url": "git+https://github.com/yongjun21/fishnet.git" + }, + "author": { + "name": "Yong Jun" + }, + "bugs": { + "url": "https://github.com/yongjun21/fishnet/issues" + }, + "license": "ISC", + "readmeFilename": "README.md" + } + }, + { + "seq": 2, + "id": "shopping_site_detect", + "changes": [{ + "rev": "1-c62a780cdd9865b06b1e4eb0e6552560" + }], + "doc": { + "_id": "shopping_site_detect", + "_rev": "1-c62a780cdd9865b06b1e4eb0e6552560", + "name": "shopping_site_detect", + "description": "shopping site detect", + "dist-tags": { + "latest": "1.0.0" + }, + "versions": { + "1.0.0": { + "name": "shopping_site_detect", + "version": "1.0.0", + "description": "shopping site detect", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": { + "name": "Jayin Ton" + }, + "license": "MIT", + "gitHead": "223c4a7e12acf28310a106780652d1e7d0629d68", + "_id": "shopping_site_detect@1.0.0", + "_shasum": "0408f054053d7581869d42934e42d50794d20b8c", + "_from": ".", + "_npmVersion": "3.10.3", + "_nodeVersion": "6.4.0", + "_npmUser": { + "name": "jayinton", + "email": "tonjayin@gmail.com" + }, + "dist": { + "shasum": "0408f054053d7581869d42934e42d50794d20b8c", + "tarball": "https://registry.npmjs.org/shopping_site_detect/-/shopping_site_detect-1.0.0.tgz" + }, + "maintainers": [{ + "name": "jayinton", + "email": "tonjayin@gmail.com" + }], + "_npmOperationalInternal": { + "host": "packages-12-west.internal.npmjs.com", + "tmp": "tmp/shopping_site_detect-1.0.0.tgz_1484193398924_0.3402463826350868" + } + } + }, + "maintainers": [{ + "name": "jayinton", + "email": "tonjayin@gmail.com" + }], + "time": { + "modified": "2017-01-12T03:56:39.161Z", + "created": "2017-01-12T03:56:39.161Z", + "1.0.0": "2017-01-12T03:56:39.161Z" + }, + "author": { + "name": "Jayin Ton" + }, + "license": "MIT", + "readmeFilename": "readme.md" + } + } + ], + "last_seq": 2 +} \ No newline at end of file diff --git a/swh/lister/npm/tests/data/npm_incremental_page2.json b/swh/lister/npm/tests/data/npm_incremental_page2.json new file mode 100644 index 0000000..4a42b3b --- /dev/null +++ b/swh/lister/npm/tests/data/npm_incremental_page2.json @@ -0,0 +1,632 @@ +{ + "results": [{ + "seq": 3, + "id": "tinyviewpager", + "changes": [{ + "rev": "1-01e65a5602e9c43ed3482657f1309ef0" + }], + "doc": { + "_id": "tinyviewpager", + "_rev": "1-01e65a5602e9c43ed3482657f1309ef0", + "name": "tinyviewpager", + "time": { + "modified": "2017-01-12T04:05:34.847Z", + "created": "2017-01-11T10:52:41.047Z", + "1.0.0": "2017-01-11T10:52:41.047Z", + "1.0.1": "2017-01-11T14:02:03.232Z", + "1.0.2": "2017-01-11T14:31:04.820Z", + "1.0.3": "2017-01-11T14:38:41.688Z", + "1.0.4": "2017-01-11T14:45:57.970Z", + "1.0.5": "2017-01-11T14:49:04.778Z", + "1.0.6": "2017-01-11T14:54:06.876Z", + "1.0.7": "2017-01-11T15:07:13.271Z", + "1.0.8": "2017-01-12T04:05:34.847Z" + }, + "maintainers": [{ + "name": "tinylpc", + "email": "1150247879@qq.com" + }], + "dist-tags": { + "latest": "1.0.8" + }, + "description": "上下左右无限循环viewpager", + "versions": { + "1.0.1": { + "name": "tinyviewpager", + "version": "1.0.1", + "description": "上下左右无限循环viewpager", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/tinylpc/TinyViewPager.git" + }, + "keywords": [ + "viewpager" + ], + "author": { + "name": "tinylpc" + }, + "license": "ISC", + "bugs": { + "url": "https://github.com/tinylpc/TinyViewPager/issues" + }, + "homepage": "https://github.com/tinylpc/TinyViewPager#readme", + "dependencies": { + "react": "^15.4.2", + "react-native": "^0.40.0", + "react-timer-mixin": "^0.13.3" + }, + "gitHead": "fdb455a393c7d6b6d43d01f73362fa6e3ed0dbb9", + "_id": "tinyviewpager@1.0.1", + "_shasum": "b466c0c66e4dadc9706edb5f069d42ee3a8ed78b", + "_from": ".", + "_npmVersion": "3.10.3", + "_nodeVersion": "6.3.1", + "_npmUser": { + "name": "tinylpc", + "email": "1150247879@qq.com" + }, + "dist": { + "shasum": "b466c0c66e4dadc9706edb5f069d42ee3a8ed78b", + "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.1.tgz" + }, + "maintainers": [{ + "name": "tinylpc", + "email": "1150247879@qq.com" + }], + "_npmOperationalInternal": { + "host": "packages-12-west.internal.npmjs.com", + "tmp": "tmp/tinyviewpager-1.0.1.tgz_1484143322968_0.5517262546345592" + } + }, + "1.0.2": { + "name": "tinyviewpager", + "version": "1.0.2", + "description": "上下左右无限循环viewpager", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/tinylpc/TinyViewPager.git" + }, + "keywords": [ + "viewpager" + ], + "author": { + "name": "tinylpc" + }, + "license": "ISC", + "bugs": { + "url": "https://github.com/tinylpc/TinyViewPager/issues" + }, + "homepage": "https://github.com/tinylpc/TinyViewPager#readme", + "dependencies": { + "react": "^15.4.2", + "react-native": "^0.40.0", + "react-timer-mixin": "^0.13.3" + }, + "gitHead": "daae30e6ce7bea30a14619b5cef5f52a86cc5528", + "_id": "tinyviewpager@1.0.2", + "_shasum": "98a1e07ed4a100009a5e196a0aa958cb13d945e5", + "_from": ".", + "_npmVersion": "3.10.3", + "_nodeVersion": "6.3.1", + "_npmUser": { + "name": "tinylpc", + "email": "1150247879@qq.com" + }, + "dist": { + "shasum": "98a1e07ed4a100009a5e196a0aa958cb13d945e5", + "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.2.tgz" + }, + "maintainers": [{ + "name": "tinylpc", + "email": "1150247879@qq.com" + }], + "_npmOperationalInternal": { + "host": "packages-12-west.internal.npmjs.com", + "tmp": "tmp/tinyviewpager-1.0.2.tgz_1484145064595_0.49263988458551466" + } + }, + "1.0.3": { + "name": "tinyviewpager", + "version": "1.0.3", + "description": "上下左右无限循环viewpager", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/tinylpc/TinyViewPager.git" + }, + "keywords": [ + "viewpager" + ], + "author": { + "name": "tinylpc" + }, + "license": "ISC", + "bugs": { + "url": "https://github.com/tinylpc/TinyViewPager/issues" + }, + "homepage": "https://github.com/tinylpc/TinyViewPager#readme", + "dependencies": { + "react": "^15.4.2", + "react-native": "^0.40.0", + "react-timer-mixin": "^0.13.3" + }, + "gitHead": "d08aeab765b537d71ed21e7e37ae6546e9bdbef2", + "_id": "tinyviewpager@1.0.3", + "_shasum": "75f0c384cf1c17de0099b98ee17af5a4ff560459", + "_from": ".", + "_npmVersion": "3.10.3", + "_nodeVersion": "6.3.1", + "_npmUser": { + "name": "tinylpc", + "email": "1150247879@qq.com" + }, + "dist": { + "shasum": "75f0c384cf1c17de0099b98ee17af5a4ff560459", + "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.3.tgz" + }, + "maintainers": [{ + "name": "tinylpc", + "email": "1150247879@qq.com" + }], + "_npmOperationalInternal": { + "host": "packages-12-west.internal.npmjs.com", + "tmp": "tmp/tinyviewpager-1.0.3.tgz_1484145521456_0.8925717421807349" + } + }, + "1.0.4": { + "name": "tinyviewpager", + "version": "1.0.4", + "description": "上下左右无限循环viewpager", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/tinylpc/TinyViewPager.git" + }, + "keywords": [ + "viewpager" + ], + "author": { + "name": "tinylpc" + }, + "license": "ISC", + "bugs": { + "url": "https://github.com/tinylpc/TinyViewPager/issues" + }, + "homepage": "https://github.com/tinylpc/TinyViewPager#readme", + "dependencies": { + "react": "^15.4.2", + "react-native": "^0.40.0", + "react-timer-mixin": "^0.13.3" + }, + "gitHead": "cde72131c7adac616b79032a588c09d814816266", + "_id": "tinyviewpager@1.0.4", + "_shasum": "bc21669fa5e855cdfb8fceff23b43b832d0cfd55", + "_from": ".", + "_npmVersion": "3.10.3", + "_nodeVersion": "6.3.1", + "_npmUser": { + "name": "tinylpc", + "email": "1150247879@qq.com" + }, + "dist": { + "shasum": "bc21669fa5e855cdfb8fceff23b43b832d0cfd55", + "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.4.tgz" + }, + "maintainers": [{ + "name": "tinylpc", + "email": "1150247879@qq.com" + }], + "_npmOperationalInternal": { + "host": "packages-12-west.internal.npmjs.com", + "tmp": "tmp/tinyviewpager-1.0.4.tgz_1484145957736_0.7274783558677882" + } + }, + "1.0.5": { + "name": "tinyviewpager", + "version": "1.0.5", + "description": "上下左右无限循环viewpager", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/tinylpc/TinyViewPager.git" + }, + "keywords": [ + "viewpager" + ], + "author": { + "name": "tinylpc" + }, + "license": "ISC", + "bugs": { + "url": "https://github.com/tinylpc/TinyViewPager/issues" + }, + "homepage": "https://github.com/tinylpc/TinyViewPager#readme", + "dependencies": { + "react-timer-mixin": "^0.13.3" + }, + "gitHead": "cde72131c7adac616b79032a588c09d814816266", + "_id": "tinyviewpager@1.0.5", + "_shasum": "59a9f33bc4f7e39b21ed8c064bf9fd1bff47e15e", + "_from": ".", + "_npmVersion": "3.10.3", + "_nodeVersion": "6.3.1", + "_npmUser": { + "name": "tinylpc", + "email": "1150247879@qq.com" + }, + "dist": { + "shasum": "59a9f33bc4f7e39b21ed8c064bf9fd1bff47e15e", + "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.5.tgz" + }, + "maintainers": [{ + "name": "tinylpc", + "email": "1150247879@qq.com" + }], + "_npmOperationalInternal": { + "host": "packages-12-west.internal.npmjs.com", + "tmp": "tmp/tinyviewpager-1.0.5.tgz_1484146144523_0.9442351886536926" + } + }, + "1.0.6": { + "name": "tinyviewpager", + "version": "1.0.6", + "description": "上下左右无限循环viewpager", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/tinylpc/TinyViewPager.git" + }, + "keywords": [ + "viewpager" + ], + "author": { + "name": "tinylpc" + }, + "license": "ISC", + "bugs": { + "url": "https://github.com/tinylpc/TinyViewPager/issues" + }, + "homepage": "https://github.com/tinylpc/TinyViewPager#readme", + "dependencies": { + "react-timer-mixin": "^0.13.3" + }, + "gitHead": "cde72131c7adac616b79032a588c09d814816266", + "_id": "tinyviewpager@1.0.6", + "_shasum": "13a9cc5a65f703eecd3c6d6007670812190bba39", + "_from": ".", + "_npmVersion": "3.10.3", + "_nodeVersion": "6.3.1", + "_npmUser": { + "name": "tinylpc", + "email": "1150247879@qq.com" + }, + "dist": { + "shasum": "13a9cc5a65f703eecd3c6d6007670812190bba39", + "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.6.tgz" + }, + "maintainers": [{ + "name": "tinylpc", + "email": "1150247879@qq.com" + }], + "_npmOperationalInternal": { + "host": "packages-18-east.internal.npmjs.com", + "tmp": "tmp/tinyviewpager-1.0.6.tgz_1484146444932_0.7940177812706679" + } + }, + "1.0.7": { + "name": "tinyviewpager", + "version": "1.0.7", + "description": "上下左右无限循环viewpager", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/tinylpc/TinyViewPager.git" + }, + "keywords": [ + "viewpager" + ], + "author": { + "name": "tinylpc" + }, + "license": "ISC", + "bugs": { + "url": "https://github.com/tinylpc/TinyViewPager/issues" + }, + "homepage": "https://github.com/tinylpc/TinyViewPager#readme", + "dependencies": { + "react-timer-mixin": "^0.13.3" + }, + "gitHead": "cde72131c7adac616b79032a588c09d814816266", + "_id": "tinyviewpager@1.0.7", + "_shasum": "4fc4f6a3a4b33f3fb362bf2ea46d9e21cd92da16", + "_from": ".", + "_npmVersion": "3.10.3", + "_nodeVersion": "6.3.1", + "_npmUser": { + "name": "tinylpc", + "email": "1150247879@qq.com" + }, + "dist": { + "shasum": "4fc4f6a3a4b33f3fb362bf2ea46d9e21cd92da16", + "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.7.tgz" + }, + "maintainers": [{ + "name": "tinylpc", + "email": "1150247879@qq.com" + }], + "_npmOperationalInternal": { + "host": "packages-18-east.internal.npmjs.com", + "tmp": "tmp/tinyviewpager-1.0.7.tgz_1484147231500_0.17695364635437727" + } + }, + "1.0.8": { + "name": "tinyviewpager", + "version": "1.0.8", + "description": "上下左右无限循环viewpager", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/tinylpc/TinyViewPager.git" + }, + "keywords": [ + "viewpager" + ], + "author": { + "name": "tinylpc" + }, + "license": "ISC", + "bugs": { + "url": "https://github.com/tinylpc/TinyViewPager/issues" + }, + "homepage": "https://github.com/tinylpc/TinyViewPager#readme", + "dependencies": { + "react-timer-mixin": "^0.13.3" + }, + "gitHead": "54b9e9f1517a0385307065f96ef85e9b7b13868e", + "_id": "tinyviewpager@1.0.8", + "_shasum": "67234eebbae3a8205319c950fab62c8f84840061", + "_from": ".", + "_npmVersion": "3.10.3", + "_nodeVersion": "6.3.1", + "_npmUser": { + "name": "tinylpc", + "email": "1150247879@qq.com" + }, + "dist": { + "shasum": "67234eebbae3a8205319c950fab62c8f84840061", + "tarball": "https://registry.npmjs.org/tinyviewpager/-/tinyviewpager-1.0.8.tgz" + }, + "maintainers": [{ + "name": "tinylpc", + "email": "1150247879@qq.com" + }], + "_npmOperationalInternal": { + "host": "packages-18-east.internal.npmjs.com", + "tmp": "tmp/tinyviewpager-1.0.8.tgz_1484193932800_0.371786616044119" + } + } + }, + "homepage": "https://github.com/tinylpc/TinyViewPager#readme", + "keywords": [ + "viewpager" + ], + "repository": { + "type": "git", + "url": "git+https://github.com/tinylpc/TinyViewPager.git" + }, + "author": { + "name": "tinylpc" + }, + "bugs": { + "url": "https://github.com/tinylpc/TinyViewPager/issues" + }, + "license": "ISC", + "readmeFilename": "README.md" + } + }, + { + "seq": 4, + "id": "ubase-ext-wecloud", + "changes": [{ + "rev": "1-0a540144594976690ff540d57ed3ab3b" + }], + "doc": { + "_id": "ubase-ext-wecloud", + "_rev": "1-0a540144594976690ff540d57ed3ab3b", + "name": "ubase-ext-wecloud", + "dist-tags": { + "latest": "1.0.3" + }, + "versions": { + "1.0.0": { + "name": "ubase-ext-wecloud", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "dependencies": { + "copy-webpack-plugin": "^4.0.1", + "ubase-vue": "^1.1.14" + }, + "_id": "ubase-ext-wecloud@1.0.0", + "_shasum": "5e03bf173c77b0c696d0d68b8f7332a38da3c897", + "_from": ".", + "_npmVersion": "3.8.6", + "_nodeVersion": "6.1.0", + "_npmUser": { + "name": "askliujun", + "email": "547794529@qq.com" + }, + "dist": { + "shasum": "5e03bf173c77b0c696d0d68b8f7332a38da3c897", + "tarball": "https://registry.npmjs.org/ubase-ext-wecloud/-/ubase-ext-wecloud-1.0.0.tgz" + }, + "maintainers": [{ + "name": "askliujun", + "email": "547794529@qq.com" + }], + "_npmOperationalInternal": { + "host": "packages-18-east.internal.npmjs.com", + "tmp": "tmp/ubase-ext-wecloud-1.0.0.tgz_1484191505909_0.5115192488301545" + } + }, + "1.0.1": { + "name": "ubase-ext-wecloud", + "version": "1.0.1", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "dependencies": { + "copy-webpack-plugin": "^4.0.1", + "ubase-vue": "^1.1.14" + }, + "_id": "ubase-ext-wecloud@1.0.1", + "_shasum": "d7fea43f73ed0ca0001f26c13de659f994f883f8", + "_from": ".", + "_npmVersion": "3.8.6", + "_nodeVersion": "6.1.0", + "_npmUser": { + "name": "askliujun", + "email": "547794529@qq.com" + }, + "dist": { + "shasum": "d7fea43f73ed0ca0001f26c13de659f994f883f8", + "tarball": "https://registry.npmjs.org/ubase-ext-wecloud/-/ubase-ext-wecloud-1.0.1.tgz" + }, + "maintainers": [{ + "name": "askliujun", + "email": "547794529@qq.com" + }], + "_npmOperationalInternal": { + "host": "packages-12-west.internal.npmjs.com", + "tmp": "tmp/ubase-ext-wecloud-1.0.1.tgz_1484191845449_0.817801987985149" + } + }, + "1.0.2": { + "name": "ubase-ext-wecloud", + "version": "1.0.2", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "dependencies": { + "copy-webpack-plugin": "^4.0.1", + "ubase-vue": "^1.1.14" + }, + "_id": "ubase-ext-wecloud@1.0.2", + "_shasum": "f724f003da71ee2904a1f9f3da47644f68f87d73", + "_from": ".", + "_npmVersion": "3.8.6", + "_nodeVersion": "6.1.0", + "_npmUser": { + "name": "askliujun", + "email": "547794529@qq.com" + }, + "dist": { + "shasum": "f724f003da71ee2904a1f9f3da47644f68f87d73", + "tarball": "https://registry.npmjs.org/ubase-ext-wecloud/-/ubase-ext-wecloud-1.0.2.tgz" + }, + "maintainers": [{ + "name": "askliujun", + "email": "547794529@qq.com" + }], + "_npmOperationalInternal": { + "host": "packages-18-east.internal.npmjs.com", + "tmp": "tmp/ubase-ext-wecloud-1.0.2.tgz_1484192705755_0.1207962513435632" + } + }, + "1.0.3": { + "name": "ubase-ext-wecloud", + "version": "1.0.3", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "dependencies": { + "copy-webpack-plugin": "^4.0.1", + "ubase-vue": "^1.1.14" + }, + "gitHead": "a5b60823b706c4fd98ae798dbe76f18a7841eead", + "_id": "ubase-ext-wecloud@1.0.3", + "_shasum": "edd5ff2b0f52c5d5248c0649bdb79d02c9567e7b", + "_from": ".", + "_npmVersion": "3.8.6", + "_nodeVersion": "6.1.0", + "_npmUser": { + "name": "askliujun", + "email": "547794529@qq.com" + }, + "dist": { + "shasum": "edd5ff2b0f52c5d5248c0649bdb79d02c9567e7b", + "tarball": "https://registry.npmjs.org/ubase-ext-wecloud/-/ubase-ext-wecloud-1.0.3.tgz" + }, + "maintainers": [{ + "name": "askliujun", + "email": "547794529@qq.com" + }], + "_npmOperationalInternal": { + "host": "packages-12-west.internal.npmjs.com", + "tmp": "tmp/ubase-ext-wecloud-1.0.3.tgz_1484193826093_0.8929786814842373" + } + } + }, + "readme": "# ubase-ext-wecloud\n", + "maintainers": [{ + "name": "askliujun", + "email": "547794529@qq.com" + }], + "time": { + "modified": "2017-01-12T04:03:46.329Z", + "created": "2017-01-12T03:25:07.858Z", + "1.0.0": "2017-01-12T03:25:07.858Z", + "1.0.1": "2017-01-12T03:30:45.704Z", + "1.0.2": "2017-01-12T03:45:07.635Z", + "1.0.3": "2017-01-12T04:03:46.329Z" + }, + "license": "ISC", + "readmeFilename": "README.md" + } + } + ], + "last_seq": 4 +} \ No newline at end of file diff --git a/swh/lister/npm/tests/test_lister.py b/swh/lister/npm/tests/test_lister.py index 673e024..4472a2f 100644 --- a/swh/lister/npm/tests/test_lister.py +++ b/swh/lister/npm/tests/test_lister.py @@ -1,97 +1,200 @@ -# Copyright (C) 2018-2019 The Software Heritage developers +# Copyright (C) 2018-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -import logging -import re -from typing import Any, List -import unittest +from itertools import chain +import json +from pathlib import Path -import requests_mock +import iso8601 +import pytest +from requests.exceptions import HTTPError -from swh.lister.core.tests.test_lister import HttpListerTesterBase -from swh.lister.npm.lister import NpmIncrementalLister, NpmLister - -logger = logging.getLogger(__name__) +from swh.lister import USER_AGENT +from swh.lister.npm.lister import NpmLister, NpmListerState -class NpmListerTester(HttpListerTesterBase, unittest.TestCase): - Lister = NpmLister - test_re = re.compile(r'^.*/_all_docs\?startkey="(.+)".*') - lister_subdir = "npm" - good_api_response_file = "data/replicate.npmjs.com/api_response.json" - bad_api_response_file = "data/api_empty_response.json" - first_index = "jquery" - entries_per_page = 100 - - @requests_mock.Mocker() - def test_is_within_bounds(self, http_mocker): - # disable this test from HttpListerTesterBase as - # it can not succeed for the npm lister due to the - # overriding of the string_pattern_check method - pass +@pytest.fixture +def npm_full_listing_page1(datadir): + return json.loads(Path(datadir, "npm_full_page1.json").read_text()) -class NpmIncrementalListerTester(HttpListerTesterBase, unittest.TestCase): - Lister = NpmIncrementalLister - test_re = re.compile(r"^.*/_changes\?since=([0-9]+).*") - lister_subdir = "npm" - good_api_response_file = "data/api_inc_response.json" - bad_api_response_file = "data/api_inc_empty_response.json" - first_index = "6920642" - entries_per_page = 100 - - @requests_mock.Mocker() - def test_is_within_bounds(self, http_mocker): - # disable this test from HttpListerTesterBase as - # it can not succeed for the npm lister due to the - # overriding of the string_pattern_check method - pass +@pytest.fixture +def npm_full_listing_page2(datadir): + return json.loads(Path(datadir, "npm_full_page2.json").read_text()) -def check_tasks(tasks: List[Any]): - """Ensure scheduled tasks are in the expected format. +@pytest.fixture +def npm_incremental_listing_page1(datadir): + return json.loads(Path(datadir, "npm_incremental_page1.json").read_text()) - """ - for row in tasks: - logger.debug("row: %s", row) - assert row["type"] == "load-npm" - # arguments check - args = row["arguments"]["args"] - assert len(args) == 0 - - # kwargs - kwargs = row["arguments"]["kwargs"] - assert len(kwargs) == 1 - package_url = kwargs["url"] - package_name = package_url.split("/")[-1] - assert package_url == f"https://www.npmjs.com/package/{package_name}" - - assert row["policy"] == "recurring" - assert row["priority"] is None +@pytest.fixture +def npm_incremental_listing_page2(datadir): + return json.loads(Path(datadir, "npm_incremental_page2.json").read_text()) -def test_npm_lister_basic_listing(lister_npm, requests_mock_datadir): - lister_npm.run() +def _check_listed_npm_packages(lister, packages, scheduler_origins): + for package in packages: + package_name = package["doc"]["name"] + latest_version = package["doc"]["dist-tags"]["latest"] + package_last_update = iso8601.parse_date(package["doc"]["time"][latest_version]) + origin_url = lister.PACKAGE_URL_TEMPLATE.format(package_name=package_name) - tasks = lister_npm.scheduler.search_tasks(task_type="load-npm") - assert len(tasks) == 100 - - check_tasks(tasks) + scheduler_origin = [o for o in scheduler_origins if o.url == origin_url] + assert scheduler_origin + assert scheduler_origin[0].last_update == package_last_update -def test_npm_lister_listing_pagination(lister_npm, requests_mock_datadir): - lister = lister_npm - # Patch per page pagination - lister.per_page = 10 + 1 - lister.PATH_TEMPLATE = lister.PATH_TEMPLATE.replace( - "&limit=1001", "&limit=%s" % lister.per_page +def _match_request(request): + return request.headers.get("User-Agent") == USER_AGENT + + +def _url_params(page_size, **kwargs): + params = {"limit": page_size, "include_docs": "true"} + params.update(**kwargs) + return params + + +def test_npm_lister_full( + swh_scheduler, requests_mock, mocker, npm_full_listing_page1, npm_full_listing_page2 +): + """Simulate a full listing of four npm packages in two pages""" + page_size = 2 + lister = NpmLister(scheduler=swh_scheduler, page_size=page_size, incremental=False) + + requests_mock.get( + lister.API_FULL_LISTING_URL, + [{"json": npm_full_listing_page1}, {"json": npm_full_listing_page2},], + additional_matcher=_match_request, ) + + spy_get = mocker.spy(lister.session, "get") + + stats = lister.run() + assert stats.pages == 2 + assert stats.origins == page_size * stats.pages + + spy_get.assert_has_calls( + [ + mocker.call( + lister.API_FULL_LISTING_URL, + params=_url_params(page_size + 1, startkey='""'), + ), + mocker.call( + lister.API_FULL_LISTING_URL, + params=_url_params( + page_size + 1, + startkey=f'"{npm_full_listing_page1["rows"][-1]["id"]}"', + ), + ), + ] + ) + + scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).origins + + _check_listed_npm_packages( + lister, + chain(npm_full_listing_page1["rows"][:-1], npm_full_listing_page2["rows"]), + scheduler_origins, + ) + + assert lister.get_state_from_scheduler() == NpmListerState() + + +def test_npm_lister_incremental( + swh_scheduler, + requests_mock, + mocker, + npm_incremental_listing_page1, + npm_incremental_listing_page2, +): + """Simulate an incremental listing of four npm packages in two pages""" + page_size = 2 + lister = NpmLister(scheduler=swh_scheduler, page_size=page_size, incremental=True) + + requests_mock.get( + lister.API_INCREMENTAL_LISTING_URL, + [ + {"json": npm_incremental_listing_page1}, + {"json": npm_incremental_listing_page2}, + {"json": {"results": []}}, + ], + additional_matcher=_match_request, + ) + + spy_get = mocker.spy(lister.session, "get") + + assert lister.get_state_from_scheduler() == NpmListerState() + + stats = lister.run() + assert stats.pages == 2 + assert stats.origins == page_size * stats.pages + + last_seq = npm_incremental_listing_page2["results"][-1]["seq"] + + spy_get.assert_has_calls( + [ + mocker.call( + lister.API_INCREMENTAL_LISTING_URL, + params=_url_params(page_size, since="0"), + ), + mocker.call( + lister.API_INCREMENTAL_LISTING_URL, + params=_url_params( + page_size, + since=str(npm_incremental_listing_page1["results"][-1]["seq"]), + ), + ), + mocker.call( + lister.API_INCREMENTAL_LISTING_URL, + params=_url_params(page_size, since=str(last_seq)), + ), + ] + ) + + scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).origins + + _check_listed_npm_packages( + lister, + chain( + npm_incremental_listing_page1["results"], + npm_incremental_listing_page2["results"], + ), + scheduler_origins, + ) + + assert lister.get_state_from_scheduler() == NpmListerState(last_seq=last_seq) + + +def test_npm_lister_incremental_restart( + swh_scheduler, requests_mock, mocker, +): + """Check incremental npm listing will restart from saved state""" + page_size = 2 + last_seq = 67 + lister = NpmLister(scheduler=swh_scheduler, page_size=page_size, incremental=True) + lister.state = NpmListerState(last_seq=last_seq) + + requests_mock.get(lister.API_INCREMENTAL_LISTING_URL, json={"results": []}) + + spy_get = mocker.spy(lister.session, "get") + lister.run() - tasks = lister.scheduler.search_tasks(task_type="load-npm") - assert len(tasks) == 2 * 10 # only 2 files with 10 results each + spy_get.assert_called_with( + lister.API_INCREMENTAL_LISTING_URL, + params=_url_params(page_size, since=str(last_seq)), + ) - check_tasks(tasks) + +def test_npm_lister_http_error( + swh_scheduler, requests_mock, mocker, +): + lister = NpmLister(scheduler=swh_scheduler) + + requests_mock.get(lister.API_FULL_LISTING_URL, status_code=500) + + with pytest.raises(HTTPError): + lister.run() diff --git a/swh/lister/npm/tests/test_tasks.py b/swh/lister/npm/tests/test_tasks.py index 481d91d..cdee572 100644 --- a/swh/lister/npm/tests/test_tasks.py +++ b/swh/lister/npm/tests/test_tasks.py @@ -1,15 +1,9 @@ -# Copyright (C) 2019-2020 The Software Heritage developers +# Copyright (C) 2019-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from contextlib import contextmanager -from unittest.mock import patch - - -@contextmanager -def mock_save(lister): - yield +from swh.lister.pattern import ListerStats def test_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker): @@ -20,34 +14,31 @@ def test_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker): assert res.result == "OK" -@patch("swh.lister.npm.tasks.save_registry_state") -@patch("swh.lister.npm.tasks.NpmLister") -def test_lister(lister, save, swh_scheduler_celery_app, swh_scheduler_celery_worker): - # setup the mocked NpmLister - lister.return_value = lister - lister.run.return_value = None - save.side_effect = mock_save +def test_full_lister_task( + swh_scheduler_celery_app, swh_scheduler_celery_worker, mocker +): + stats = ListerStats(pages=10, origins=900) + mock_lister = mocker.patch("swh.lister.npm.tasks.NpmLister") + mock_lister.from_configfile.return_value = mock_lister + mock_lister.run.return_value = stats res = swh_scheduler_celery_app.send_task("swh.lister.npm.tasks.NpmListerTask") assert res res.wait() assert res.successful() - lister.assert_called_once_with() - lister.run.assert_called_once_with() + mock_lister.from_configfile.assert_called_once_with(incremental=False) + mock_lister.run.assert_called_once_with() + assert res.result == stats.dict() -@patch("swh.lister.npm.tasks.save_registry_state") -@patch("swh.lister.npm.tasks.get_last_update_seq") -@patch("swh.lister.npm.tasks.NpmIncrementalLister") -def test_incremental( - lister, seq, save, swh_scheduler_celery_app, swh_scheduler_celery_worker +def test_incremental_lister_task( + swh_scheduler_celery_app, swh_scheduler_celery_worker, mocker ): - # setup the mocked NpmLister - lister.return_value = lister - lister.run.return_value = None - seq.return_value = 42 - save.side_effect = mock_save + stats = ListerStats(pages=10, origins=900) + mock_lister = mocker.patch("swh.lister.npm.tasks.NpmLister") + mock_lister.from_configfile.return_value = mock_lister + mock_lister.run.return_value = stats res = swh_scheduler_celery_app.send_task( "swh.lister.npm.tasks.NpmIncrementalListerTask" @@ -56,6 +47,6 @@ def test_incremental( res.wait() assert res.successful() - lister.assert_called_once_with() - seq.assert_called_once_with(lister) - lister.run.assert_called_once_with(min_bound=42) + mock_lister.from_configfile.assert_called_once_with(incremental=True) + mock_lister.run.assert_called_once_with() + assert res.result == stats.dict()