reonokiy/swh-lister
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

simple_lister: Split models into smaller chunks to avoid oversized db transactions

Browse source 
Related T1659
This commit is contained in:
Antoine Lambert 2019-06-28 15:43:15 +02:00
parent 5ea9d5ed39
commit d85bcdac5b
1 changed files with 1 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
swh/lister/core/simple_lister.py
View file

@ -45,7 +45,7 @@ class SimpleLister(ListerBase):
models_list = self.transport_response_simplified(response)
models_list = self.filter_before_inject(models_list)
all_injected = []
for models in utils.grouper(models_list, n=10000):
for models in utils.grouper(models_list, n=1000):
models = list(models)
logging.debug('models: %s' % len(models))
# inject into local db