From 4a09f660b35aa77744d9ed7b0ded84ba253305f3 Mon Sep 17 00:00:00 2001 From: Franck Bret Date: Mon, 12 Sep 2022 21:33:07 +0200 Subject: [PATCH] Crates.io: Add last_update for each version of a crate In order to reduce http api call amount made by the loader, download a crates.io database dump, and parse its csv files to get a last_update value for each versions of a Crate. Those values are sent to the loader through extra_loader_arguments 'crates_metadata'. 'artifacts' and 'crates_metadata' now uses "version" as key. Related T4104, D8171 --- swh/lister/crates/__init__.py | 70 ++-- swh/lister/crates/lister.py | 303 +++++++++--------- swh/lister/crates/tests/__init__.py | 26 -- .../tests/data/fake-crates-repository.tar.gz | Bin 9134 -> 0 bytes .../tests/data/fake_crates_repository_init.sh | 91 +++--- .../https_static.crates.io/db-dump.tar.gz | Bin 0 -> 1358 bytes .../db-dump.tar.gz_visit1 | Bin 0 -> 1534 bytes swh/lister/crates/tests/test_lister.py | 203 ++++++------ 8 files changed, 327 insertions(+), 366 deletions(-) delete mode 100644 swh/lister/crates/tests/data/fake-crates-repository.tar.gz create mode 100644 swh/lister/crates/tests/data/https_static.crates.io/db-dump.tar.gz create mode 100644 swh/lister/crates/tests/data/https_static.crates.io/db-dump.tar.gz_visit1 diff --git a/swh/lister/crates/__init__.py b/swh/lister/crates/__init__.py index 6fd08e6..31bd3e9 100644 --- a/swh/lister/crates/__init__.py +++ b/swh/lister/crates/__init__.py @@ -2,7 +2,6 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information - """ Crates lister ============= @@ -20,20 +19,24 @@ versions. Origins retrieving strategy --------------------------- -A json http api to list packages from crates.io but we choose a `different strategy`_ -in order to reduce to its bare minimum the amount of http call and bandwidth. -We clone a git repository which contains a tree of directories whose last child folder -name corresponds to the package name and contains a Cargo.toml file with some json data -to describe all existing versions of the package. -It takes a few seconds to clone the repository and browse it to build a full index of -existing package and related versions. -The lister is incremental, so the first time it clones and browses the repository as -previously described then stores the last seen commit id. -Next time, it retrieves the list of new and changed files since last commit id and -returns new or changed package with all of their related versions. +A json http api to list packages from crates.io exists but we choose a +`different strategy`_ in order to reduce to its bare minimum the amount +of http call and bandwidth. -Note that all Git related operations are done with `Dulwich`_, a Python -implementation of the Git file formats and protocols. +We download a `db-dump.tar.gz`_ archives which contains csv files as an export of +the crates.io database. Crates.csv list package names, versions.csv list versions +related to package names. +It takes a few seconds to download the archive and parse csv files to build a +full index of existing package and related versions. + +The archive also contains a metadata.json file with a timestamp corresponding to +the date the database dump started. The database dump is automatically generated +every 24 hours, around 02:00:00 UTC. + +The lister is incremental, so the first time it downloads the db-dump.tar.gz archive as +previously described and store the last seen database dump timestamp. +Next time, it downloads the db-dump.tar.gz but retrieves only the list of new and +changed packages since last seen timestamp with all of their related versions. Page listing ------------ @@ -48,56 +51,45 @@ The data schema for each line is: * **crate_file**: Package download url * **checksum**: Package download checksum * **yanked**: Whether the package is yanked or not -* **last_update**: Iso8601 last update date computed upon git commit date of the - related Cargo.toml file +* **last_update**: Iso8601 last update Origins from page ----------------- The lister yields one origin per page. The origin url corresponds to the http api url for a package, for example -"https://crates.io/api/v1/crates/{package}". +"https://crates.io/crates/{crate}". -Additionally we add some data set to "extra_loader_arguments": +Additionally we add some data for each version, set to "extra_loader_arguments": * **artifacts**: Represent data about the Crates to download, following :ref:`original-artifacts-json specification ` * **crates_metadata**: To store all other interesting attributes that do not belongs - to artifacts. For now it mainly indicate when a version is `yanked`_. + to artifacts. For now it mainly indicate when a version is `yanked`_, and the version + last_update timestamp. Origin data example:: { - "url": "https://crates.io/api/v1/crates/rand", + "url": "https://crates.io/api/v1/crates/regex-syntax", "artifacts": [ { + "version": "0.1.0", "checksums": { - "sha256": "48a45b46c2a8c38348adb1205b13c3c5eb0174e0c0fec52cc88e9fb1de14c54d", # noqa: B950 + "sha256": "398952a2f6cd1d22bc1774fd663808e32cf36add0280dee5cdd84a8fff2db944", # noqa: B950 }, - "filename": "rand-0.1.1.crate", - "url": "https://static.crates.io/crates/rand/rand-0.1.1.crate", - "version": "0.1.1", - }, - { - "checksums": { - "sha256": "6e229ed392842fa93c1d76018d197b7e1b74250532bafb37b0e1d121a92d4cf7", # noqa: B950 - }, - "filename": "rand-0.1.2.crate", - "url": "https://static.crates.io/crates/rand/rand-0.1.2.crate", - "version": "0.1.2", + "filename": "regex-syntax-0.1.0.crate", + "url": "https://static.crates.io/crates/regex-syntax/regex-syntax-0.1.0.crate", # noqa: B950 }, ], "crates_metadata": [ { - "version": "0.1.1", - "yanked": False, - }, - { - "version": "0.1.2", + "version": "0.1.0", + "last_update": "2017-11-30 03:37:17.449539", "yanked": False, }, ], - } + }, Running tests ------------- @@ -128,8 +120,8 @@ You can follow lister execution by displaying logs of swh-lister service:: .. _Cargo: https://doc.rust-lang.org/cargo/guide/why-cargo-exists.html#enter-cargo .. _Cargo.toml: https://doc.rust-lang.org/cargo/reference/manifest.html .. _different strategy: https://crates.io/data-access -.. _Dulwich: https://www.dulwich.io/ .. _yanked: https://doc.rust-lang.org/cargo/reference/publishing.html#cargo-yank +.. _db-dump.tar.gz: https://static.crates.io/db-dump.tar.gz """ diff --git a/swh/lister/crates/lister.py b/swh/lister/crates/lister.py index fbe3003..eca9f10 100644 --- a/swh/lister/crates/lister.py +++ b/swh/lister/crates/lister.py @@ -2,19 +2,20 @@ # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from dataclasses import asdict, dataclass -import datetime -import io + +import csv +from dataclasses import dataclass +from datetime import datetime import json import logging from pathlib import Path -import shutil +import tarfile +import tempfile from typing import Any, Dict, Iterator, List, Optional from urllib.parse import urlparse -from dulwich import porcelain -from dulwich.patch import write_tree_diff -from dulwich.repo import Repo +import iso8601 +from packaging.version import parse as parse_version from swh.scheduler.interface import SchedulerInterface from swh.scheduler.model import ListedOrigin @@ -30,36 +31,36 @@ CratesListerPage = List[Dict[str, Any]] @dataclass class CratesListerState: """Store lister state for incremental mode operations. - 'last_commit' represents a git commit hash + 'index_last_update' represents the UTC time the crates.io database dump was + started """ - last_commit: str = "" + index_last_update: Optional[datetime] = None class CratesLister(Lister[CratesListerState, CratesListerPage]): """List origins from the "crates.io" forge. - It basically fetches https://github.com/rust-lang/crates.io-index.git to a - temp directory and then walks through each file to get the crate's info on - the first run. + It downloads a tar.gz archive which contains crates.io database table content as + csv files which is automatically generated every 24 hours. + Parsing two csv files we can list all Crates.io package names and their related + versions. - In incremental mode, it relies on the same Git repository but instead of reading - each file of the repo, it get the differences through ``git log last_commit..HEAD``. - Resulting output string is parsed to build page entries. + In incremental mode, it check each entry comparing their 'last_update' value + with self.state.index_last_update """ - # Part of the lister API, that identifies this lister LISTER_NAME = "crates" - # (Optional) CVS type of the origins listed by this lister, if constant VISIT_TYPE = "crates" - INSTANCE = "crates" - INDEX_REPOSITORY_URL = "https://github.com/rust-lang/crates.io-index.git" - DESTINATION_PATH = Path("/tmp/crates.io-index") + + BASE_URL = "https://crates.io" + DB_DUMP_URL = "https://static.crates.io/db-dump.tar.gz" + CRATE_FILE_URL_PATTERN = ( "https://static.crates.io/crates/{crate}/{crate}-{version}.crate" ) - CRATE_API_URL_PATTERN = "https://crates.io/api/v1/crates/{crate}" + CRATE_URL_PATTERN = "https://crates.io/crates/{crate}" def __init__( self, @@ -69,172 +70,172 @@ class CratesLister(Lister[CratesListerState, CratesListerPage]): super().__init__( scheduler=scheduler, credentials=credentials, - url=self.INDEX_REPOSITORY_URL, + url=self.BASE_URL, instance=self.INSTANCE, ) + self.index_metadata: Dict[str, str] = {} def state_from_dict(self, d: Dict[str, Any]) -> CratesListerState: - if "last_commit" not in d: - d["last_commit"] = "" + index_last_update = d.get("index_last_update") + if index_last_update is not None: + d["index_last_update"] = iso8601.parse_date(index_last_update) return CratesListerState(**d) def state_to_dict(self, state: CratesListerState) -> Dict[str, Any]: - return asdict(state) + d: Dict[str, Optional[str]] = {"index_last_update": None} + index_last_update = state.index_last_update + if index_last_update is not None: + d["index_last_update"] = index_last_update.isoformat() + return d - def get_index_repository(self) -> None: - """Get crates.io-index repository up to date running git command.""" - if self.DESTINATION_PATH.exists(): - porcelain.pull( - self.DESTINATION_PATH, remote_location=self.INDEX_REPOSITORY_URL - ) - else: - porcelain.clone( - source=self.INDEX_REPOSITORY_URL, target=self.DESTINATION_PATH - ) - - def get_crates_index(self) -> List[Path]: - """Build a sorted list of file paths excluding dotted directories and - dotted files. - - Each file path corresponds to a crate that lists all available - versions. + def is_new(self, dt_str: str): + """Returns True when dt_str is greater than + self.state.index_last_update """ - crates_index = sorted( - path - for path in self.DESTINATION_PATH.rglob("*") - if not any(part.startswith(".") for part in path.parts) - and path.is_file() - and path != self.DESTINATION_PATH / "config.json" - ) + dt = iso8601.parse_date(dt_str) + last = self.state.index_last_update + return not last or (last is not None and last < dt) - return crates_index + def get_and_parse_db_dump(self) -> Dict[str, Any]: + """Download and parse csv files from db_dump_path. - def get_last_commit_hash(self, repository_path: Path) -> str: - """Returns the last commit hash of a git repository""" - assert repository_path.exists() - - repo = Repo(str(repository_path)) - head = repo.head() - last_commit = repo[head] - - return last_commit.id.decode() - - def get_last_update_by_file(self, filepath: Path) -> Optional[datetime.datetime]: - """Given a file path within a Git repository, returns its last commit - date as iso8601 + Returns a dict where each entry corresponds to a package name with its related versions. """ - repo = Repo(str(self.DESTINATION_PATH)) - # compute relative path otherwise it fails - relative_path = filepath.relative_to(self.DESTINATION_PATH) - walker = repo.get_walker(paths=[bytes(relative_path)], max_entries=1) - try: - commit = next(iter(walker)).commit - except StopIteration: - logger.error( - "Can not find %s related commits in repository %s", relative_path, repo - ) - return None - else: - last_update = datetime.datetime.fromtimestamp( - commit.author_time, datetime.timezone.utc - ) - return last_update + + with tempfile.TemporaryDirectory() as tmpdir: + + file_name = self.DB_DUMP_URL.split("/")[-1] + archive_path = Path(tmpdir) / file_name + + # Download the Db dump + with self.http_request(self.DB_DUMP_URL, stream=True) as res: + with open(archive_path, "wb") as out_file: + for chunk in res.iter_content(chunk_size=1024): + out_file.write(chunk) + + # Extract the Db dump + db_dump_path = Path(str(archive_path).split(".tar.gz")[0]) + tar = tarfile.open(archive_path) + tar.extractall(path=db_dump_path) + tar.close() + + csv.field_size_limit(1000000) + + (crates_csv_path,) = list(db_dump_path.glob("*/data/crates.csv")) + (versions_csv_path,) = list(db_dump_path.glob("*/data/versions.csv")) + (index_metadata_json_path,) = list(db_dump_path.rglob("*metadata.json")) + + with index_metadata_json_path.open("rb") as index_metadata_json: + self.index_metadata = json.load(index_metadata_json) + + crates: Dict[str, Any] = {} + with crates_csv_path.open() as crates_fd: + crates_csv = csv.DictReader(crates_fd) + for item in crates_csv: + if self.is_new(item["updated_at"]): + # crate 'id' as key + crates[item["id"]] = { + "name": item["name"], + "updated_at": item["updated_at"], + "versions": {}, + } + + data: Dict[str, Any] = {} + with versions_csv_path.open() as versions_fd: + versions_csv = csv.DictReader(versions_fd) + for version in versions_csv: + if version["crate_id"] in crates.keys(): + crate: Dict[str, Any] = crates[version["crate_id"]] + crate["versions"][version["num"]] = version + # crate 'name' as key + data[crate["name"]] = crate + return data def page_entry_dict(self, entry: Dict[str, Any]) -> Dict[str, Any]: """Transform package version definition dict to a suitable page entry dict """ + crate_file = self.CRATE_FILE_URL_PATTERN.format( + crate=entry["name"], version=entry["version"] + ) + filename = urlparse(crate_file).path.split("/")[-1] return dict( name=entry["name"], - version=entry["vers"], - checksum=entry["cksum"], - yanked=entry["yanked"], - crate_file=self.CRATE_FILE_URL_PATTERN.format( - crate=entry["name"], version=entry["vers"] - ), + version=entry["version"], + checksum=entry["checksum"], + yanked=True if entry["yanked"] == "t" else False, + crate_file=crate_file, + filename=filename, + last_update=entry["updated_at"], ) def get_pages(self) -> Iterator[CratesListerPage]: - """Yield an iterator sorted by name in ascending order of pages. - - Each page is a list of crate versions with: - - name: Name of the crate - - version: Version - - checksum: Checksum - - crate_file: Url of the crate file - - last_update: Date of the last commit of the corresponding index - file + """Each page is a list of crate versions with: + - name: Name of the crate + - version: Version + - checksum: Checksum + - yanked: Whether the package is yanked or not + - crate_file: Url of the crate file + - filename: File name of the crate file + - last_update: Last update for that version """ - # Fetch crates.io index repository - self.get_index_repository() - if not self.state.last_commit: - # First discovery - # List all crates files from the index repository - crates_index = self.get_crates_index() - else: - # Incremental case - # Get new package version by parsing a range of commits from index repository - repo = Repo(str(self.DESTINATION_PATH)) - head = repo[repo.head()] - last = repo[self.state.last_commit.encode()] - outstream = io.BytesIO() - write_tree_diff(outstream, repo.object_store, last.tree, head.tree) - raw_diff = outstream.getvalue() - crates_index = [] - for line in raw_diff.splitlines(): - if line.startswith(b"+++ b/"): - filepath = line.split(b"+++ b/", 1)[1] - crates_index.append(self.DESTINATION_PATH / filepath.decode()) - crates_index = sorted(crates_index) + # Fetch crates.io Db dump, then Parse the data. + dataset = self.get_and_parse_db_dump() - logger.debug("Found %s crates in crates_index", len(crates_index)) + logger.debug("Found %s crates in crates_index", len(dataset)) - # Each line of a crate file is a json entry describing released versions - # for a package - for crate in crates_index: + # Each entry from dataset will correspond to a page + for name, item in dataset.items(): page = [] - last_update = self.get_last_update_by_file(crate) + # sort crate versions + versions: list = sorted(item["versions"].keys(), key=parse_version) + + for version in versions: + v = item["versions"][version] + v["name"] = name + v["version"] = version + page.append(self.page_entry_dict(v)) - with crate.open("rb") as current_file: - for line in current_file: - data = json.loads(line) - entry = self.page_entry_dict(data) - entry["last_update"] = last_update - page.append(entry) yield page def get_origins_from_page(self, page: CratesListerPage) -> Iterator[ListedOrigin]: """Iterate on all crate pages and yield ListedOrigin instances.""" - assert self.lister_obj.id is not None - url = self.CRATE_API_URL_PATTERN.format(crate=page[0]["name"]) + url = self.CRATE_URL_PATTERN.format(crate=page[0]["name"]) last_update = page[0]["last_update"] + artifacts = [] crates_metadata = [] - for version in page: - filename = urlparse(version["crate_file"]).path.split("/")[-1] + for entry in page: # Build an artifact entry following original-artifacts-json specification # https://docs.softwareheritage.org/devel/swh-storage/extrinsic-metadata-specification.html#original-artifacts-json # noqa: B950 - artifact = { - "filename": f"{filename}", - "checksums": { - "sha256": f"{version['checksum']}", - }, - "url": version["crate_file"], - "version": version["version"], - } - artifacts.append(artifact) - data = {f"{version['version']}": {"yanked": f"{version['yanked']}"}} - crates_metadata.append(data) + artifacts.append( + { + "version": entry["version"], + "filename": entry["filename"], + "url": entry["crate_file"], + "checksums": { + "sha256": entry["checksum"], + }, + } + ) + + crates_metadata.append( + { + "version": entry["version"], + "yanked": entry["yanked"], + "last_update": entry["last_update"], + } + ) yield ListedOrigin( lister_id=self.lister_obj.id, visit_type=self.VISIT_TYPE, url=url, - last_update=last_update, + last_update=iso8601.parse_date(last_update), extra_loader_arguments={ "artifacts": artifacts, "crates_metadata": crates_metadata, @@ -242,18 +243,8 @@ class CratesLister(Lister[CratesListerState, CratesListerPage]): ) def finalize(self) -> None: - last = self.get_last_commit_hash(repository_path=self.DESTINATION_PATH) - if self.state.last_commit == last: - self.updated = False - else: - self.state.last_commit = last + last: datetime = iso8601.parse_date(self.index_metadata["timestamp"]) + + if not self.state.index_last_update: + self.state.index_last_update = last self.updated = True - - logger.debug("Listing crates origin completed with last commit id %s", last) - - # Cleanup by removing the repository directory - if self.DESTINATION_PATH.exists(): - shutil.rmtree(self.DESTINATION_PATH) - logger.debug( - "Successfully removed %s directory", str(self.DESTINATION_PATH) - ) diff --git a/swh/lister/crates/tests/__init__.py b/swh/lister/crates/tests/__init__.py index 8b98baa..68748ea 100644 --- a/swh/lister/crates/tests/__init__.py +++ b/swh/lister/crates/tests/__init__.py @@ -1,29 +1,3 @@ # Copyright (C) 2022 the Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information - -import os -from pathlib import PosixPath -import subprocess -from typing import Optional, Union - - -def prepare_repository_from_archive( - archive_path: str, - filename: Optional[str] = None, - tmp_path: Union[PosixPath, str] = "/tmp", -) -> str: - """Given an existing archive_path, uncompress it. - Returns a file repo url which can be used as origin url. - - This does not deal with the case where the archive passed along does not exist. - - """ - if not isinstance(tmp_path, str): - tmp_path = str(tmp_path) - # uncompress folder/repositories/dump for the loader to ingest - subprocess.check_output(["tar", "xf", archive_path, "-C", tmp_path]) - # build the origin url (or some derivative form) - _fname = filename if filename else os.path.basename(archive_path) - repo_url = f"file://{tmp_path}/{_fname}" - return repo_url diff --git a/swh/lister/crates/tests/data/fake-crates-repository.tar.gz b/swh/lister/crates/tests/data/fake-crates-repository.tar.gz deleted file mode 100644 index 498b10590d39b5bc909006831ea2f7ee2c739498..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 9134 zcmZ{JRan$t)HNtA9g@-w0t$#yL$`#aba$7MLkiN}-Q8V7BV7uTBi$klIRo>1`M>Y; ze0SfzI9KQ5tiAVIYwzdKMPs4-Z#+NE_3 zgU`0e5}2oY!U{19A@@v~inm&7kHmw0E9|KALS2*9dU)Tk z`{m?4o zWGm~yNv~jjugy@}HG?OKvLjJgMa(VjdO8bphlS=-EpWT>dE2<9ea-J*5f^6`A4VUD zCimQ`a3n9>Uo#f(_n$w1urlFhc>QL+=c`&BbkA+)6sm+B7}M{wf*j+eJ-pq|JkNGw z2t|;bdmLEV2q{m!bsBuoYah$D$P+198Tlke3E?F*9+CplXBXQ$bx! zK=Z`FjkTpA{1{wSGLc{^X&I|P!-sr3v#3JO8{QfqMNK3m42|Vvl(nVAHjxzh|<|}@g zas2s$f_l_f_0hY2WyZJMc404uB*Mmpmn`iu4209kQYCQ`>JDX4DI9XBq6P~8s9-tE zxk_oW^D_x}3VLbi3(!n=C6HzEEjFMYlo4?$q!KbIu#<9dYEjCogi37-msWAVXfAbi zw3XV=DmqEQ+Oa>D>YF(p6wQ=qp65!cQpb%f#Hk}07a$)NeD(I*^r$4NnUwsuK{SCN z)@;Edi4GFK?ac%;59 z#Y&tlXHcA|6E!jCtqFyrb65wHjvVEcP$;GW zn2X&EpyUCU@Jm=70@Aj(!0SS-N5y~L*6t=@RD-D}E^=Ys$XQ~gU!cXE@u-x$xVO_l z;`Ca0Ed)KK3Nraxl&A+gMG{=F3O5+syFNh8${k~zC%_^H(AoT6!hot=p!FMERnyg` z?c6Yb_CQ7T=8k{54st){0%vtZE6+^HT;WvK(A)} z&V$&QHh1fSuRI=+OKTQ5=u#@|(FmRGbh=&@X1M?Vbpe*F~Xqz6_}Y6bI>6X61r zq`=AvsAlUDZsR`1xQH_`2Sj)Svs8me`Jn;U{a#*HVbeg>$T^Tzw0du%++S8J8IBq! zh_0MMf}zF3@Ej|cSqit1UA@Av`TSgnHN71Snu$~=RL1}af}Ro`_URhjAf0_f#LZL# zMCe~!_z?=WCFV)( zaTP!ibqy3cjsY);B9EG!o!TQx*=XWAZq4!K5$=xMtR;_J^!iAXh5s;xEt(V)7j#)y z=;?f)@%oG|G1?WS&8Zul&!kksuHGuW<7LZ7FdeZFRj55VsO+O_Cg(&9n0z~LZ zGWDz_>eXb~^UBtJDmA=`2^V4Oi=f`zu2;_I5@J=nu{UIEGD!(HyvY?KK-C%Wnjr^? zD~0c=9fCuwu0V=|5ou|Q#f8E0%^8c~IZaoLE0T`g-VNY=Zi)bNy*~%Y!WH{9{-%}g zfNL+V?*@l)9FA-H0;Fz_Uc-Bdf!`r1X`Kwq2kA_1UPKB6q#FyMffXZ zzDYLdu=t#uE>q*yMKeD2{9TXY0UmWXspp3NoaNA{cC)jG{{v4B(HXCu$7iWkNR|9F zuyz8}JUvn(`GeFqC9m)FQ4PVkHBxv)WeltioW;a%EUpv6V1Rm}(l8*=`UoLIGcSu# zjRjG3${E= z94W4Et^#FV&~W+gn2@lFX3kybau^s3_tn^p>6V?U=(DJ;`q=~}hO=3ml$>Q{&DoAm z;yIi;8r@&9ui6fJz}DDeZMOoiReJ3`5Zroj+tR*@la>xT2miHewfsQ7+HZa3_h9&~ zUE<~Dfb1eQuG1goTJke^=RCF@p!MAt38e*^!(8E*x0gxbgM9m(Kego=UY-)E7ff~$ zev`o2LHow@L=Q}s2MEsDLp%$b`CmO~=>*$?)2pufJR@gUXx?-0cBFe;^I$}GcV0}U z4u9Rj@@v8v7ZkIt4E5Rv=M5_r%^zXIJn2KJ5^T4}7=y~tKRT(61Grq0u(dJX> z6?(T#i*RTCK%4ALuR&7@YB0Pn;Jo|hrhu-ctJSQ#Xvt>G66o5wf8DKW6R4@4Kq=Jp z4~BG5)pE>CSQlKqc5Nn}L0Xx5pI0)YjFnA$u)9q(23g;Yb%t5upb23cjiw%e$;OQV zGdz|{NUTV?cQ$Nzkbl~3;34M7XY9b)7J5FHDZjb#U>whhZ{c{IUK}v;E|Iz9vG|85 z?r0oPXEJy+TT%2(LaWHsXcIF10JI?vH=VL_h|bgiPh5?T+5=9x&PnAvZ6#+?tpy>0??d z61qd!ex0%B7wx)5CsUCW7m;ineutf6=Y&-AO(pGp8 zLG~@Q)$c}#fF_`KJwXV=@q=&E_Mu7An0z;@v+5IVFp7nc7XRUQg}*40M9-ed!@S*{ z6Zl*zY))a;Zy8JU*$QVz>G18~D7fm=I*<)|(%KN%1r%}c4UqRY^w>Rpc|ThkARq!W znKV-Djd=O70-`5+9KB+o$HK9?Y)|4NH4DV+19THvXFx4|4w877G@F+86SGzsbrUO* zGrcW^`tyVeiC=_W6Bl63TjC2)(+*gh^IToOt#3``No!N;zx3ilOPf`Ov%CTyo5?M)4q;#4ty?$xjsFq1C~67&i%kb3{f!C(3(D<*ToO9t=rFRdzKMM73H zhKkXO0ox}S`(Q1tb-4Qo7i^?dYov^#es59tju-J2lI?$C;QeJ(9`ONt-W(> zs@|$HCPOS$XoHtxcvf75GWbrg=fM+G`qi+5tq!J$^n;q;@7ru(L$6<46tL+!=Vf&} z!Q{utz5Q-wdZ}(LRZ4RSpj!y}>@l`ZvGz3Oe z@NBi~0?Q6ywm$(*AA#45IY>ftvdN?p_mCiZoKNWHxD`_9VGNqx58%fQ z3{t~D$Zz8~AYENz|B6=3SAL&-<97Y}S5G=^{?U8XOPkk!TOeDZ{7bK)xrZtyAKYg{ z1pDSag!-;k=GU$-OT(|%v>%}pzfRD+{cOA^W?q$fJGNSF5w`O)>>c*JioJ~oiC15K zS+Vd7NSt$Dx4FLi{SM|}wd6nWx!C+zEC6RA{%2qHfek_j^wktzw0>2-c!Gve>)VEG zZ1rgc77x;eI@+s;zrX`FI{G+bl#tq2;wY`oOy|JNi1{3nkd&$^cW`Y2gD< zn$d39?*GAlVwOqh*V#Wk>+)B(@4$^qpsKM_&|CnT2>*%7DcsuyLgDnfNb>MbtqBGk zbV{Cqa*#Nq7;{8(UX&b>gxJSFuJ&Yfw1Pe?Vu#2;Chj3D2VjYhZBMPs9)KKcqoli& z>^?h`xtz)ciGe%^-(JnEzayg|8`J3(S|xWR(U4jN#jM`d`}F)DE50$PkjHcAg#Xdf zMd5$HSM>~=RSVx4RjRf624EQjCyG4{P)c#Y=(U5dOYF<=HOz?@FVR_3XHDMndMV~u zSj?^so?e>6|iG zN}%sZ(T>rcF^MKzRJ3uQZ?@X#r?c0li)f!~?|jcq488-fX{F^M~!g@TEkMj`L&_ykgD{k0t7d<2%qrhCqc_mV{fAK|}|A>&lv>-}z%YLmfK5Ux>;+K)Cxi_&(s$=iYWMsCf~9Rn+DmLg*EK}C+UZyfLg1trzf%l5OBx%TCX2r zUl(|+{wp-uf<~V9(4Bsj%p;l57f>T=y)$V%0P8);_?txSkMG3Zrdbx*OA$(uelErd zqoMrCmte-O+YRt@HUh9tWd0dYFBY7^LH%jA7FbUOG51YRGOgzLaFkzABf(2z`(MG{ z<=zh6`%CKxkE}0WoA*w>)W7Pob>C~NcKDbjc+=XMYv2VBT#HGLmKPT_zdP!+9}89H zfv&nu(PvvRcpq3L}oaI($Qoq!5_-6r{~yv;7Rwda7_xu~H3?Uzbv$i=k2&+%?l zQ8Q@PZ|l!dp`ERyqv=NPPQL}#4F$1K(&?I_YD_LNCvg1e1!Q(Y<{ zgl9KtP0rDMvQ#Ij2(Y{l@D>tTbk4j7d(Ym&jyy(FZbc6Q_5(I;@@@n2BQz{3eyxBz zj*uYIy+?2?VF?M5!FI}X3OO1cI+||qt?ALoRYW%%iVW`Q-KdH^b(;I2cTHVo&6SE9 zQQ}SM*^Bgo7HXzIMzwIK??2MRxEE5LQfho#R%~fB+YqO-sKV(lXrps-FqP#MpCvZ^ zL@N5>Q?P-=`HNwVDfN@welA%aCuU1L%RFMD>l=o%M(?k8@?>uApG__pcfNUCSX*YH zotTdC-MSj|o;hgL{&D%S)tRxLq*c(_1RGydzGg{Lt@oOzlWqxVk9EQ9*N;5HM z(}(W^!#ZSZ+Nif=atpI%<56FJ@6M(r9YEii1#Hp0;PS)Vl2IShP!%!ID2cym#DwUj z(CZ}?E27y*rr6sd$I1|(lzJaTJvP!37{eDTl0mW_N?KL~RPFV_=aROs2I{@g=j@Vg zos_2*vnpr-{c2$H0Q{5i_-d9xAlB@?k|fPI1KRY0(jE$FWRQ7ONh%Z%D*;141K6`T zfv`i6)kIl!Kvt%KS|td+BQ(1VH&I@}_v>~0LhC7GQsAU|)4vnWc@6OB5oYipq3<#jnU8xqEeTPaPCx%pd<5VO1Disic%D5~v z0my5$FB2GIeq4MFUsj9Hknr+C_lKcX5diz&HB=x(_)(e>d}b~}jqw_nkB%gM6^MJ!0M0Tn$@1K`O;J?=wrj~PM8|MBJvYSvZFcmk7=|Sm zPp!Y>{-k7yHReH7>i2rpJMX(;4#){c9X(qdIUrp$!vZzppHDUtOFf1?U=;oq>F_|_ z?QZK{vXyh=t$kKUCuil16XJgCVm=<8j;U{Q%(+nofBo$|98$Gr5kNkh^RhQ!GyN5B zm>j0xND{$0h=b51wgFW>ofMB_rcL{fe3b^X#Iv+SH`Xb~S#=2Mj>2P$g2u!C;IOV-i z4qwgD9|qfoBhxykU>|+5U-v-EJ^;Hhk*-(-MS2KG5t-u*xCWuHr#&Nj_FMsr=sArH z3Zaba>ssI<@eULOyEhS#O1=ZH6;16V2BNC1)r(%P-(GKrV`rydH5qdeYRw4Z%(pd& z*2Wz_Kx=NcfhTHU9bd)M<4SJ{)V4t1zmVXLea06e!HwIPv4Fs|C6(-anGe7f9`a{_ zI-(Xoz9C<6n4M73*@jI(WWR{Pc!A7^wFm%X=RPu>K?I&m%fU z$JjJ0;YOYp+EQH#n|)pD)y}Yq@vlZo zu`@T-zl3?@JsvLBgz^=fQ{Z0M4X$zJ?TTG#vwBAv*_Gt)%z=w#n4@m(*Om`89+C46 zi}-AQAiXov>a)M_u|i)U|0N6pJAnL0IS+0Ob*$({ikNNgsYd3H#b}|?m7}Zmgv=NU zq>hD;@>{I;;M6ZWXAoAb1&+u`>gCayxy|C4pZ6EP|>p_tkRWROh?>tIM$k17-C$=ZeEe7U8^=t{*z5--LLkljg7F zX9sX|<%UlUayGkzJ2&=R*OzQg8P;li3dxOk>su-~zt|-h8$YN0@R%m*@5e|;{OVND zPOjcd$mp*GB7)^Gea&~@L(}6iINJ-qteJBslJFCobYq)dU%*1zJ6>UZ8P=SS6VlPs zezD8}pESGi1k!qqseggLe%>dTr7k)vZW~6=!#EMvP|cY1A>nJUX$DVhEV?7G1Bx6O z+cMhnRnL0U9#{vFsg^&imXkIJl4)X_tCx9h#08+7ZvNL9(|b(y!c$4L!r^MzLMtnk z%H30ssM;XvEo~?1Chm!rTBGX<8X#!t0o;eI{-$0IPQswXv9=46*1%$UgSCsLALB}c zlE@e3iXv-f`2q$3mmo=^E^kbj1U3Mu%Q*45?rju=)_eIT!Dhq3V5-LZFaJPlz^i>b z1D(93Cu1CIIRIeyP=U}V$)z9ICX5K-lTUsh?v_Hqtnkeo^<}7Sn5MnThvyZ2W6(%| z>4|W-tFZt9%=+Jyh@3MmM}6Su^*QGX9=&CLGQDymX(GJ(HcEa;S233V@OCi_6j9>HQjxjHE)s-W>H`-6OEB~2Yd-#b( zWHN@H$2>FE)>Eta`|7C8^n9Z0Doj1sl97J@*m38_54vV> zBz2&IVTv;$)n_q{sb z@*t8a544k7_@z}jLDta9$Ivr0!fNPg*0#>P@I7FA1&+#Q2F2ZMkmNH@cYO8_$e;X~ zIYU08+eSkp7;PApo2lRArCJzzrVJHUtUTXbu#(O7gZPr@8aeQt9EE^4eQKBFydKpg z{E#5X6YeKF82WC9^GuQ$ePEP6Aw%aFU*dC`ED@CjgHoknM>qnItgu6ZV{QS#!H7G8 zr&ZGg)Yr-i!&mt9%;YaG&CdO4?d0D{P=~iCLGpcb9)3#TOxf=>O}BavafFE9Dc1UL zb?wAfwk3JK{t~syHoC|I-v3I}`rzV^q{Ayz1q22XDU=y$l(x(hh%DZJm&6Sv#!?DG z3vXnfT6ld0fZC8R{#~00N-bWS6la;%^rlvt!TH@4v|NRJWQq1&SUX%c3f0$|E?dN_ z9zka#z?-N4tS7_BFLNy|CLk7gUTB`tuR%ea9F`G|ofgayLT`)9s1HH%V!`g|1VX%* z!Sw_kE@McmgnKr~>EOLJJ->#@?*{%>ZL_j)5wldMcwG$dbrKzQK~B zNMLabL$ggYmqdk3NDR0KqpFDH;m&jD4Y==Jk2^X0+FxG>j|UuB%|qjI{aj226gGWXP5$QVp^Xl-flA|5@?QzvXC*_ik>CrZvj*P;T#e$4UW&$-% z`u($@f*-^|CM{J-UP3;bEO69z0155B(# zbt>G8$OhSZ7Z~32a5=UWgk@Mu5SqJXSgPy9%%jC@v&Viyk$;0SE=cQIhW*WXg=9q_ z(@c>P=4_1~KAjdMO^;Qqha>BT@r_UAo{)WTP(T_5P1T(l!wjX?e?g6fli9HiO<=hv zX}AS^9zf z@b}{3H2}fGDJUP<<6c%${R90DNIu-V#1_Z_{GXHTF3SJZpU~;9QWyhS1)t%!EYC@6 z9$cV1ya<3W=-)j=;Q2WKLxTR_0Ra2J7hZ#NTKj!)!}z@kp_2;s#4JE%8WRe$-ILY5 zUrd__=-%RIzqo;CCp%I+#!J`Z^7p3$+hmcE z4z{rVo4}~T7%k^%_>4}^B-vG6?rWPJkRh}^eYNIh72*(|3~zw6c~sk8M4TYJ7qWiE zp`Nkrln_^cMh$Gbon8MOH+T3Nz6I~=y@a-xJ7la1eNg<9IWLkM@Y!77NvG$HwV7J5 zx~l#p#mQ);#=q;#?k(r*^MUGeuh{>#dh&$)$|%E;hcBUd ziFgT~D|YwM?-j;mDP7(98shTy?1=keyme+9UYr*prP8jqOJ%gm=aW|5-oGn!*4xfY zx9jQE)~;*(2`vNbrOrOO>Y)ao>AY|{ulFdSnj?wOCx^Wf6w=_qzVBW?^LD>Qjr8{M zdbY0TTKwi0GQ4joz#<%tTU*JE862hkW%Sls^sNuH6?fA016NQAFo{@qSZOuL data/crates.csv -touch .dot-file -touch config.json +echo -e '''checksum,crate_id,crate_size,created_at,downloads,features,id,license,links,num,published_by,updated_at,yanked +398952a2f6cd1d22bc1774fd663808e32cf36add0280dee5cdd84a8fff2db944,2233,,2015-05-27 23:19:16.848643,1961,{},10855,MIT/Apache-2.0,,0.1.0,,2017-11-30 03:37:17.449539,f +343bd0171ee23346506db6f4c64525de6d72f0e8cc533f83aea97f3e7488cbf9,545,,2014-12-18 06:56:46.88489,845,{},1321,MIT/Apache-2.0,,0.1.2,,2017-11-30 02:29:20.01125,f +6e229ed392842fa93c1d76018d197b7e1b74250532bafb37b0e1d121a92d4cf7,1339,,2015-02-03 11:15:19.001762,8211,{},4371,MIT/Apache-2.0,,0.1.2,,2017-11-30 03:14:27.545115,f +defb220c4054ca1b95fe8b0c9a6e782dda684c1bdf8694df291733ae8a3748e3,545,,2014-12-19 16:16:41.73772,1498,{},1363,MIT/Apache-2.0,,0.1.3,,2017-11-30 02:26:59.236947,f +48a45b46c2a8c38348adb1205b13c3c5eb0174e0c0fec52cc88e9fb1de14c54d,1339,,2015-02-03 06:17:14.169972,7963,{},4362,MIT/Apache-2.0,,0.1.1,,2017-11-30 03:33:14.186028,f +f0ff1ca641d3c9a2c30464dac30183a8b91cdcc959d616961be020cdea6255c5,545,,2014-12-13 22:10:11.329494,3204,{},1100,MIT/Apache-2.0,,0.1.0,,2017-11-30 02:51:27.240551,f +a07bef996bd38a73c21a8e345d2c16848b41aa7ec949e2fedffe9edf74cdfb36,545,,2014-12-15 20:31:48.571836,889,{},1178,MIT/Apache-2.0,,0.1.1,,2017-11-30 03:03:20.143103,f +''' > data/versions.csv -# Init as a git repository -git init -git add . -git commit -m "Init fake crates.io-index repository for tests purpose" +echo -e '''{ + "timestamp": "2022-08-08T02:00:27.645191645Z", + "crates_io_commit": "3e5f0b4d2a382ac0951898fd257f693734eadee2" +} +''' > metadata.json -echo '{"name":"rand","vers":"0.1.1","deps":[],"cksum":"48a45b46c2a8c38348adb1205b13c3c5eb0174e0c0fec52cc88e9fb1de14c54d","features":{},"yanked":false}' > ra/nd/rand -git add . -git commit -m " Updating crate rand#0.1.1" +cd ../../ +tar -czf db-dump.tar.gz -C crates.io-db-dump . -echo '{"name":"rand","vers":"0.1.2","deps":[{"name":"libc","req":"^0.1.1","features":[""],"optional":false,"default_features":true,"target":null,"kind":"normal"},{"name":"log","req":"^0.2.1","features":[""],"optional":false,"default_features":true,"target":null,"kind":"normal"}],"cksum":"6e229ed392842fa93c1d76018d197b7e1b74250532bafb37b0e1d121a92d4cf7","features":{},"yanked":false}' >> ra/nd/rand -git add . -git commit -m " Updating crate rand#0.1.2" +# A second db dump with a new entry and a different timestamp -echo '{"name":"regex","vers":"0.1.0","deps":[],"cksum":"f0ff1ca641d3c9a2c30464dac30183a8b91cdcc959d616961be020cdea6255c5","features":{},"yanked":false}' > re/ge/regex -git add . -git commit -m " Updating crate regex#0.1.0" +mkdir -p crates.io-db-dump_visit1 +cp -rf crates.io-db-dump/2022-08-08-020027 crates.io-db-dump_visit1/2022-09-05-020027 -echo '{"name":"regex","vers":"0.1.1","deps":[{"name":"regex_macros","req":"^0.1.0","features":[""],"optional":false,"default_features":true,"target":null,"kind":"dev"}],"cksum":"a07bef996bd38a73c21a8e345d2c16848b41aa7ec949e2fedffe9edf74cdfb36","features":{},"yanked":false}' >> re/ge/regex -git add . -git commit -m " Updating crate regex#0.1.1" +cd crates.io-db-dump_visit1/2022-09-05-020027/ -echo '{"name":"regex","vers":"0.1.2","deps":[{"name":"regex_macros","req":"^0.1.0","features":[""],"optional":false,"default_features":true,"target":null,"kind":"dev"}],"cksum":"343bd0171ee23346506db6f4c64525de6d72f0e8cc533f83aea97f3e7488cbf9","features":{},"yanked":false}' >> re/ge/regex -git add . -git commit -m " Updating crate regex#0.1.2" +echo -e '''{ + "timestamp": "2022-09-05T02:00:27.687167108Z", + "crates_io_commit": "d3652ad81bd8bd837f2d2442ee08484ee5d4bac3" +} +''' > metadata.json -echo '{"name":"regex","vers":"0.1.3","deps":[{"name":"regex_macros","req":"^0.1.0","features":[""],"optional":false,"default_features":true,"target":null,"kind":"dev"}],"cksum":"defb220c4054ca1b95fe8b0c9a6e782dda684c1bdf8694df291733ae8a3748e3","features":{},"yanked":false}' >> re/ge/regex -git add . -git commit -m " Updating crate regex#0.1.3" +echo -e '''2019-01-08 15:11:01.560092,"A crate for safe and ergonomic pin-projection.",,48353738,,107436,,pin-project,,https://github.com/taiki-e/pin-project,2022-08-15 13:52:11.642129 +''' >> data/crates.csv -echo '{"name":"regex-syntax","vers":"0.1.0","deps":[{"name":"rand","req":"^0.3","features":[""],"optional":false,"default_features":true,"target":null,"kind":"dev"},{"name":"quickcheck","req":"^0.2","features":[""],"optional":false,"default_features":true,"target":null,"kind":"dev"}],"cksum":"398952a2f6cd1d22bc1774fd663808e32cf36add0280dee5cdd84a8fff2db944","features":{},"yanked":false}' > re/ge/regex-syntax -git add . -git commit -m " Updating crate regex-syntax#0.1.0" +echo -e '''ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc,107436,56972,2022-08-15 13:52:11.642129,580330,{},602929,Apache-2.0 OR MIT,,1.0.12,33035,2022-08-15 13:52:11.642129,f +''' >> data/versions.csv -# Save some space -rm .git/hooks/*.sample +cd ../../ -# Compress git directory as a tar.gz archive -cd ../ -tar -cvzf fake-crates-repository.tar.gz crates.io-index -mv fake-crates-repository.tar.gz ../ +tar -czf db-dump.tar.gz_visit1 -C crates.io-db-dump_visit1 . + +# Move the generated tar.gz archives to a servable directory +mv db-dump.tar.gz ../https_static.crates.io/ +mv db-dump.tar.gz_visit1 ../https_static.crates.io/ # Clean up tmp_dir cd ../ diff --git a/swh/lister/crates/tests/data/https_static.crates.io/db-dump.tar.gz b/swh/lister/crates/tests/data/https_static.crates.io/db-dump.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..bd74c75b0fffe577720c990446d756eed21118f8 GIT binary patch literal 1358 zcmV-U1+n@ciwFP!000001MOJLa@#f(_1a&-(Hf$0FW%f`*+n;*th(`lcu1@zQbST{ z(oFunmz3)|j$Aua$4#d^qY)$$mpC~01*yEc`0n5YqtrBlQT+b+CoVt=;943kw0A4}Bs( z$7?^bn1;|dhq38e7Q1lVr*`zW-+pU%osR?CcKdYjo5Y&P_Wr~5@i1M#9-7}1YyCbk zdR&}`LpL;I*MDTkL%dz#63Md^WNR)8t}8GV$P%P6R==S$5PY*w(4X!~J+*ZO=(v(~%XYI62xdyr= z3d9O{1yBku1OQvTX^Uom*rhvhi!K*^+8lQjh3Uhgr`^$2+F{ohZ)wr8c(-kahepRC z(Uhib##H#@*zGB0XQehrPxQt_w|7mO=%Hh?PX*l%+os(VWb1dkqG=DuaX2Sj+N2N6 zDvjJZ{iqT<-)?T_Dyc}f`Qvk@pL8>8t_rXf=amqp0kl@#hKE|CDNt14D$u1B4#1FZ zXGAIvz8_Nm*e=$?N0N&+BhfiyLEv)c;pO*rsqZ9QSPQ_t#X`Pao!re!n@~wSY|5j+B8@3)nRw zwL_w@y%`wQZgx1lAv-Iz)(wbpXoD9JLmf+axe#p?!=T zEsrUw5Tlje=A1FsPRiQ^JzFWH)lx8U8rY}L47gQ_{ruCrt2YNvyIf<*8RI2P1KFE3 zz*_Kvi>Y{nDW!CZT0p)OQq++qKuWZj)Qam^>nwwo3RO%x8q7S|5R?$v3ZJ|)StKK^ z4Rv;>ig?!q?9}U&t|U<;j*jY)nh6Sov;6Q&epsQake3_)717fP(WNLHQen&9i2%`P z4mN@_wMkGLiHa+Mwa>LMHBS%$!8?pHWOLi;?)74T3KXeWPI;-3S%g_aDa^ANk_Tjk zrlfrXOcZ0vH6jm^D;Yf0PUU249-P<7Sd7tYD+AOqTkT}b=)efl(Rx85ChBWkmTBL{!?B2bKx^Q4LxfE;XV3ErC&Xt{(r#heobZZCL z15im@Dnnb;%u=$<1dV;3g!H0BnOlh(I3?u*MVppMy3J z`oEn1E?D~iE8x>*QLLt)2qeS)u&RpHU-9sca-I(8l=Ia2zyZDgvSO3L>Cb`drn{!o z@V*%*JCRi8wT$S6MIX3RV4ckomC4!(BP9Jaq?Wq6{Jdo5f&~i}ELgB$!GZ+~7A#n> QV8N^54*=fb>i{SK01MuucK`qY literal 0 HcmV?d00001 diff --git a/swh/lister/crates/tests/data/https_static.crates.io/db-dump.tar.gz_visit1 b/swh/lister/crates/tests/data/https_static.crates.io/db-dump.tar.gz_visit1 new file mode 100644 index 0000000000000000000000000000000000000000..0b7dd384a90ce172fa758391869b2d998bfd8b5c GIT binary patch literal 1534 zcmV*i^c(9Hai{%Fal*n%OSC6i*TGh?f-wr{rTF+OoT0Fh}i7QYNgi%&V zbtOP(VXv6}`>9=BoyI8)j9rx>bn*V>ao>9WKZ@!G1>#ottW9z#bW9H`Y>*;LNwyV|2~&fuK~P`IHNTst<5=I^5bv=X#+!L42kE_( zM#{&>14Vm#eoeJ&r@K>Awf#-g_xChddd`=5kHa5^#QyMf=K1C4ZG&4-Ot@OwTG>iS z?VX#=R9m1&k|EY0YJf@#DFOKP4;^a{$9;ZuF6#>$^6s>!EX<#d16fCR$-;hMA865v zeY|VO=aWujrXg+DPML+%)E{WeE}q(*0@0f?J>Iunrk76bAv1a&@7iw1Xl~f=S=${? z(|E~od6z$Nrwt14&5NGchi>;URx3?@TfV+#_}Ook$_)co3sFm1SwQFXLwe{fT86R) zUxTTf^Z=IpcEL&P7=|$qFIll2Ka;!27=_+jt6s$pDO>norhl(dw$`5Vp+f0E+Rbbg z6<62_$3Rmm)B-APgzz|9wUpT<*kdSp31J>~eb*n_$c}BdJr4cPdC7IMlPf2+w9;`7 z!YXMve=_DG)hV?1?Ka;$9e1Ydwm@`ct#M9JqY%(%X>k)&KWEyvy-^JRSH_h;B*N&tDcrf7%>8wg83hou~rOEx^8wxf?TGyZe!o zcJt$@*|*~zO=_M>KmQ!M`)2h~TXHUIG+A8&p*2C{OCmEc@dFkfiC-2!Z-U_#E~ z4YpFsYia??Mk?7P8UQ)dV#;V?Qe%pWMrqV3n`E(w>|)eX7AHdv-WHjyaxON-pB3@3 z32<|-qfM=dB60MjM|vSB5iat>zw*NxeT||L0H}$c$%sBD>5+u31TP~bYXrCi-ZnNv zV-;$lB{rcn(l#PP0)*f(saWhor}Ly2BTbr>$yf8fG{_yo;zA|us~A!QRE@SGy8&j3 zDVGLOL?yI}0UEDMc8!QWm~0)U6pT|5npB+eDi!o#CHc_>Nlwi2nFl=s>K4g^P+4g$ za!}r#U1;P*g7PaDQV2XrGA(C`pmIU!Mwy7gMd>8{P7TnZvjG`p)VZPADvKyejv8Zh zF8k6z%AlfF>1!W;>1Pc!8_2CEYKsrap_eGaS6L{R>CRB@6U8D*ff$T}B*}3YrBFtt zKv$rgbB%|XV)WW4Lu(pnGL`R`a-edmqkiVYr>!V@gykC_Y(|8*QV| zIbovRq61MhImSPmEN$kHzT7o&?WnyMGg*qBryF0WvY$RMB2NulMbwbfIH~{LyUM%v zV3+j&kf$*B0M*Z9-@U;a-_U;n#y-=3Wwl)O|2^>ghOz7Uh)8ZY9ItD3{Uskh(w?US z8tr-GECq=q^`F;#o^bwi=(g=|>4