From 213a4a152ffe2b089b6c0fed66b5e3bcb7ae8eb2 Mon Sep 17 00:00:00 2001 From: Antoine Lambert Date: Tue, 15 Apr 2025 12:15:03 +0200 Subject: [PATCH] crates: Bump chunk size when downloading database dump It allows faster download of the database dump located at https://static.crates.io/db-dump.tar.gz. --- swh/lister/crates/lister.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/swh/lister/crates/lister.py b/swh/lister/crates/lister.py index fe8b56a..d94834c 100644 --- a/swh/lister/crates/lister.py +++ b/swh/lister/crates/lister.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2024 The Software Heritage developers +# Copyright (C) 2022-2025 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -18,6 +18,7 @@ import iso8601 from looseversion import LooseVersion2 from swh.core.utils import grouper +from swh.model.hashutil import HASH_BLOCK_SIZE from swh.scheduler.interface import SchedulerInterface from swh.scheduler.model import ListedOrigin @@ -119,7 +120,7 @@ class CratesLister(Lister[CratesListerState, CratesListerPage]): # Download the Db dump with self.http_request(self.DB_DUMP_URL, stream=True) as res: with open(archive_path, "wb") as out_file: - for chunk in res.iter_content(chunk_size=1024): + for chunk in res.iter_content(chunk_size=HASH_BLOCK_SIZE): out_file.write(chunk) # Extract the Db dump