reonokiy/swh-lister
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Remove debug logging from tasks' code

Browse source 
since this is now handled by the SWHTask itself.
This commit is contained in:
David Douard 2019-01-17 13:58:29 +01:00
parent 6957f3c435
commit f670de298f
6 changed files with 36 additions and 98 deletions
Download patch file Download diff file Expand all files Collapse all files

27
swh/lister/bitbucket/tasks.py
View file

@ -16,31 +16,20 @@ def new_lister(api_baseurl='https://api.bitbucket.org/2.0'):
return BitBucketLister(api_baseurl=api_baseurl)
@app.task(name='swh.lister.bitbucket.tasks.IncrementalBitBucketLister',
bind=True)
def incremental_bitbucket_lister(self, **lister_args):
self.log.debug('%s, lister_args=%s' % (
self.name, lister_args))
@app.task(name=__name__ + '.IncrementalBitBucketLister')
def incremental_bitbucket_lister(**lister_args):
lister = new_lister(**lister_args)
lister.run(min_bound=lister.db_last_index(), max_bound=None)
self.log.debug('%s OK' % (self.name))
@app.task(name='swh.lister.bitbucket.tasks.RangeBitBucketLister',
bind=True)
def range_bitbucket_lister(self, start, end, **lister_args):
self.log.debug('%s(start=%s, end=%s), lister_args=%s' % (
self.name, start, end, lister_args))
@app.task(name=__name__ + '.RangeBitBucketLister')
def range_bitbucket_lister(start, end, **lister_args):
lister = new_lister(**lister_args)
lister.run(min_bound=start, max_bound=end)
self.log.debug('%s OK' % (self.name))
@app.task(name='swh.lister.bitbucket.tasks.FullBitBucketRelister',
bind=True)
@app.task(name=__name__ + '.FullBitBucketRelister', bind=True)
def full_bitbucket_relister(self, split=None, **lister_args):
self.log.debug('%s, lister_args=%s' % (
self.name, lister_args))
lister = new_lister(**lister_args)
ranges = lister.db_partition_indices(split or GROUP_SPLIT)
random.shuffle(ranges)
@ -54,8 +43,6 @@ def full_bitbucket_relister(self, split=None, **lister_args):
return promise.id
@app.task(name='swh.lister.bitbucket.tasks.ping',
bind=True)
def ping(self):
self.log.debug(self.name)
@app.task(name=__name__ + '.ping')
def ping():
return 'OK'

14
swh/lister/debian/tasks.py
View file

@ -7,17 +7,11 @@ from swh.scheduler.celery_backend.config import app
from .lister import DebianLister
@app.task(name='swh.lister.debian.tasks.DebianListerTask',
bind=True)
def debian_lister(self, distribution, **lister_args):
self.log.debug('%s, lister_args=%s' % (
self.name, lister_args))
@app.task(name=__name__ + '.DebianListerTask')
def debian_lister(distribution, **lister_args):
DebianLister(**lister_args).run(distribution)
self.log.debug('%s OK' % (self.name))
@app.task(name='swh.lister.debian.tasks.ping',
bind=True)
def ping(self):
self.log.debug(self.name)
@app.task(name=__name__ + '.ping')
def ping():
return 'OK'

29
swh/lister/github/tasks.py
View file

@ -5,7 +5,6 @@
import random
from celery import group
from swh.scheduler.celery_backend.config import app
from swh.lister.github.lister import GitHubLister
@ -17,31 +16,20 @@ def new_lister(api_baseurl='https://api.github.com', **kw):
return GitHubLister(api_baseurl=api_baseurl, **kw)
@app.task(name='swh.lister.github.tasks.IncrementalGitHubLister',
bind=True)
def incremental_github_lister(self, **lister_args):
self.log.debug('%s, lister_args=%s' % (
self.name, lister_args))
@app.task(name=__name__ + '.IncrementalGitHubLister')
def incremental_github_lister(**lister_args):
lister = new_lister(**lister_args)
lister.run(min_bound=lister.db_last_index(), max_bound=None)
self.log.debug('%s OK' % (self.name))
@app.task(name='swh.lister.github.tasks.RangeGitHubLister',
bind=True)
def range_github_lister(self, start, end, **lister_args):
self.log.debug('%s(start=%s, end=%d), lister_args=%s' % (
self.name, start, end, lister_args))
@app.task(name=__name__ + '.RangeGitHubLister')
def range_github_lister(start, end, **lister_args):
lister = new_lister(**lister_args)
lister.run(min_bound=start, max_bound=end)
self.log.debug('%s OK' % (self.name))
@app.task(name='swh.lister.github.tasks.FullGitHubRelister',
bind=True)
@app.task(name=__name__ + '.FullGitHubRelister', bind=True)
def full_github_relister(self, split=None, **lister_args):
self.log.debug('%s, lister_args=%s' % (
self.name, lister_args))
lister = new_lister(**lister_args)
ranges = lister.db_partition_indices(split or GROUP_SPLIT)
random.shuffle(ranges)
@ -52,11 +40,10 @@ def full_github_relister(self, split=None, **lister_args):
promise.save() # so that we can restore the GroupResult in tests
except NotImplementedError:
self.log.info('Unable to call save_group with current result backend.')
raise
return promise.id
@app.task(name='swh.lister.github.tasks.ping',
bind=True)
def ping(self):
self.log.debug(self.name)
@app.task(name=__name__ + '.ping')
def ping():
return 'OK'

28
swh/lister/gitlab/tasks.py
View file

@ -5,7 +5,6 @@
import random
from celery import group
from swh.scheduler.celery_backend.config import app
from .. import utils
@ -22,34 +21,23 @@ def new_lister(api_baseurl='https://gitlab.com/api/v4',
per_page=per_page)
@app.task(name='swh.lister.gitlab.tasks.IncrementalGitLabLister',
bind=True)
def incremental_gitlab_lister(self, **lister_args):
self.log.debug('%s, lister_args=%s' % (
self.name, lister_args))
@app.task(name=__name__ + '.IncrementalGitLabLister')
def incremental_gitlab_lister(**lister_args):
lister_args['sort'] = 'desc'
lister = new_lister(**lister_args)
total_pages = lister.get_pages_information()[1]
# stopping as soon as existing origins for that instance are detected
lister.run(min_bound=1, max_bound=total_pages, check_existence=True)
self.log.debug('%s OK' % (self.name))
@app.task(name='swh.lister.gitlab.tasks.RangeGitLabLister',
bind=True)
def range_gitlab_lister(self, start, end, **lister_args):
self.log.debug('%s(start=%s, end=%d), lister_args=%s' % (
self.name, start, end, lister_args))
@app.task(name=__name__ + '.RangeGitLabLister')
def range_gitlab_lister(start, end, **lister_args):
lister = new_lister(**lister_args)
lister.run(min_bound=start, max_bound=end)
self.log.debug('%s OK' % (self.name))
@app.task(name='swh.lister.gitlab.tasks.FullGitLabRelister',
bind=True)
@app.task(name=__name__ + '.FullGitLabRelister', bind=True)
def full_gitlab_relister(self, **lister_args):
self.log.debug('%s, lister_args=%s' % (
self.name, lister_args))
lister = new_lister(**lister_args)
_, total_pages, _ = lister.get_pages_information()
ranges = list(utils.split_range(total_pages, NBPAGES))
@ -64,8 +52,6 @@ def full_gitlab_relister(self, **lister_args):
return promise.id
@app.task(name='swh.lister.gitlab.tasks.ping',
bind=True)
def ping(self):
self.log.debug(self.name)
@app.task(name=__name__ + '.ping')
def ping():
return 'OK'

22
swh/lister/npm/tasks.py
View file

@ -40,31 +40,21 @@ def get_last_update_seq(lister):
return row[0]
@app.task(name='swh.lister.npm.tasks.NpmListerTask',
bind=True)
def npm_lister(self, **lister_args):
self.log.debug('%s, lister_args=%s' % (
self.name, lister_args))
@app.task(name=__name__ + '.NpmListerTask')
def npm_lister(**lister_args):
lister = NpmLister(**lister_args)
with save_registry_state(lister):
lister.run()
self.log.debug('%s OK' % (self.name))
@app.task(name='swh.lister.npm.tasks.NpmIncrementalListerTask',
bind=True)
def npm_incremental_lister(self, **lister_args):
self.log.debug('%s, lister_args=%s' % (
self.name, lister_args))
@app.task(name=__name__ + '.NpmIncrementalListerTask')
def npm_incremental_lister(**lister_args):
lister = NpmIncrementalLister(**lister_args)
update_seq_start = get_last_update_seq(lister)
with save_registry_state(lister):
lister.run(min_bound=update_seq_start)
self.log.debug('%s OK' % (self.name))
@app.task(name='swh.lister.npm.tasks.ping',
bind=True)
def ping(self):
self.log.debug(self.name)
@app.task(name=__name__ + '.ping')
def ping():
return 'OK'

14
swh/lister/pypi/tasks.py
View file

@ -7,17 +7,11 @@ from swh.scheduler.celery_backend.config import app
from .lister import PyPILister
@app.task(name='swh.lister.pypi.tasks.PyPIListerTask',
bind=True)
def pypi_lister(self, **lister_args):
self.log.debug('%s(), lister_args=%s' % (
self.name, lister_args))
@app.task(name=__name__ + '.PyPIListerTask')
def pypi_lister(**lister_args):
PyPILister(**lister_args).run()
self.log.debug('%s OK' % (self.name))
@app.task(name='swh.lister.pypi.tasks.ping',
bind=True)
def ping(self):
self.log.debug(self.name)
@app.task(name=__name__ + '.ping')
def ping():
return 'OK'