lister/tasks: Standardize return statements

The following commit adapts the return statements from both lister and their
associated tasks. This standardizes on what other modules (e.g. both dvcs and
package loaders) do.

swh/lister/bitbucket/tasks.py

@@ -14,13 +14,13 @@ GROUP_SPLIT = 10000
 def list_bitbucket_incremental(**lister_args):
     '''Incremental update of the BitBucket forge'''
     lister = BitBucketLister(**lister_args)
-    lister.run(min_bound=lister.db_last_index(), max_bound=None)
+    return lister.run(min_bound=lister.db_last_index(), max_bound=None)
 
 
 @shared_task(name=__name__ + '.RangeBitBucketLister')
 def _range_bitbucket_lister(start, end, **lister_args):
     lister = BitBucketLister(**lister_args)
-    lister.run(min_bound=start, max_bound=end)
+    return lister.run(min_bound=start, max_bound=end)
 
 
 @shared_task(name=__name__ + '.FullBitBucketRelister', bind=True)
@@ -44,6 +44,7 @@ def list_bitbucket_full(self, split=None, **lister_args):
         promise.save()  # so that we can restore the GroupResult in tests
     except (NotImplementedError, AttributeError):
         self.log.info('Unable to call save_group with current result backend.')
+    # FIXME: what to do in terms of return here?
     return promise.id
 
 

swh/lister/cgit/lister.py

@@ -78,6 +78,7 @@ class CGitLister(ListerBase):
         }
 
     def run(self):
+        status = 'uneventful'
         total = 0
         for repos in grouper(self.get_repos(), 10):
             models = list(filter(None, (self.build_model(repo)
@@ -87,6 +88,9 @@ class CGitLister(ListerBase):
             self.db_session.commit()
             total += len(injected_repos)
             logger.debug('Scheduled %s tasks for %s', total, self.url)
+            status = 'eventful'
+
+        return {'status': status}
 
     def get_repos(self):
         """Generate git 'project' URLs found on the current CGit server

swh/lister/cgit/tasks.py

@@ -10,7 +10,7 @@ from .lister import CGitLister
 @shared_task(name=__name__ + '.CGitListerTask')
 def list_cgit(**lister_args):
     '''Lister task for CGit instances'''
-    CGitLister(**lister_args).run()
+    return CGitLister(**lister_args).run()
 
 
 @shared_task(name=__name__ + '.ping')

swh/lister/core/indexing_lister.py

@@ -213,6 +213,7 @@ class IndexingLister(ListerBase):
         Returns:
             nothing
         """
+        status = 'uneventful'
         self.min_index = min_bound
         self.max_index = max_bound
 
@@ -243,9 +244,11 @@ class IndexingLister(ListerBase):
                 logger.debug('Flushing updates at index %s', i)
                 self.db_session.commit()
                 self.db_session = self.mk_session()
+                status = 'eventful'
 
         self.db_session.commit()
         self.db_session = self.mk_session()
+        return {'status': status}
 
 
 class IndexingHttpLister(ListerHttpTransport, IndexingLister):

swh/lister/core/page_by_page_lister.py

@@ -110,6 +110,7 @@ class PageByPageLister(ListerBase):
             nothing
 
         """
+        status = 'uneventful'
         page = min_bound or 0
         loop_count = 0
 
@@ -127,6 +128,7 @@ class PageByPageLister(ListerBase):
             elif not injected_repos:
                 logging.info('Repositories already seen, stopping')
                 break
+            status = 'eventful'
 
             next_page = self.get_next_target_from_response(response)
 
@@ -149,6 +151,8 @@ class PageByPageLister(ListerBase):
         self.db_session.commit()
         self.db_session = self.mk_session()
 
+        return {'status': status}
+
 
 class PageByPageHttpLister(ListerHttpTransport, PageByPageLister):
     """Convenience class for ensuring right lookup and init order when

swh/lister/core/simple_lister.py

@@ -89,3 +89,8 @@ class SimpleLister(ListerBase):
         response, injected_repos = self.ingest_data(dump_not_used_identifier)
         if not response and not injected_repos:
             logging.info('No response from api server, stopping')
+            status = 'uneventful'
+        else:
+            status = 'eventful'
+
+        return {'status': status}

swh/lister/cran/tasks.py

@@ -10,7 +10,7 @@ from swh.lister.cran.lister import CRANLister
 @shared_task(name=__name__ + '.CRANListerTask')
 def list_cran(**lister_args):
     '''Lister task for the CRAN registry'''
-    CRANLister(**lister_args).run()
+    return CRANLister(**lister_args).run()
 
 
 @shared_task(name=__name__ + '.ping')

swh/lister/debian/lister.py

@@ -216,12 +216,14 @@ class DebianLister(ListerHttpTransport, ListerBase):
                            .one_or_none()
 
         if not distribution:
-            raise ValueError("Distribution %s is not registered" %
-                             self.distribution)
+            logger.error("Distribution %s is not registered" %
+                         self.distribution)
+            return {'status': 'failed'}
 
         if not distribution.type == 'deb':
-            raise ValueError("Distribution %s is not a Debian derivative" %
-                             distribution)
+            logger.error("Distribution %s is not a Debian derivative" %
+                         distribution)
+            return {'status': 'failed'}
 
         date = self.date
 
@@ -250,4 +252,4 @@ class DebianLister(ListerHttpTransport, ListerBase):
 
         self.db_session.commit()
 
-        return True
+        return {'status': 'eventful'}

swh/lister/debian/tasks.py

@@ -10,7 +10,7 @@ from .lister import DebianLister
 @shared_task(name=__name__ + '.DebianListerTask')
 def list_debian_distribution(distribution, **lister_args):
     '''List a Debian distribution'''
-    DebianLister(distribution=distribution, **lister_args).run()
+    return DebianLister(distribution=distribution, **lister_args).run()
 
 
 @shared_task(name=__name__ + '.ping')

swh/lister/github/tasks.py

@@ -15,13 +15,13 @@ GROUP_SPLIT = 10000
 def list_github_incremental(**lister_args):
     'Incremental update of GitHub'
     lister = GitHubLister(**lister_args)
-    lister.run(min_bound=lister.db_last_index(), max_bound=None)
+    return lister.run(min_bound=lister.db_last_index(), max_bound=None)
 
 
 @shared_task(name=__name__ + '.RangeGitHubLister')
 def _range_github_lister(start, end, **lister_args):
     lister = GitHubLister(**lister_args)
-    lister.run(min_bound=start, max_bound=end)
+    return lister.run(min_bound=start, max_bound=end)
 
 
 @shared_task(name=__name__ + '.FullGitHubRelister', bind=True)
@@ -44,6 +44,7 @@ def list_github_full(self, split=None, **lister_args):
         promise.save()  # so that we can restore the GroupResult in tests
     except (NotImplementedError, AttributeError):
         self.log.info('Unable to call save_group with current result backend.')
+    # FIXME: what to do in terms of return here?
     return promise.id
 
 

swh/lister/gitlab/tasks.py

@@ -20,13 +20,13 @@ def list_gitlab_incremental(**lister_args):
     lister = GitLabLister(**lister_args)
     total_pages = lister.get_pages_information()[1]
     # stopping as soon as existing origins for that instance are detected
-    lister.run(min_bound=1, max_bound=total_pages, check_existence=True)
+    return lister.run(min_bound=1, max_bound=total_pages, check_existence=True)
 
 
 @shared_task(name=__name__ + '.RangeGitLabLister')
 def _range_gitlab_lister(start, end, **lister_args):
     lister = GitLabLister(**lister_args)
-    lister.run(min_bound=start, max_bound=end)
+    return lister.run(min_bound=start, max_bound=end)
 
 
 @shared_task(name=__name__ + '.FullGitLabRelister', bind=True)
@@ -43,6 +43,7 @@ def list_gitlab_full(self, **lister_args):
         promise.save()
     except (NotImplementedError, AttributeError):
         self.log.info('Unable to call save_group with current result backend.')
+    # FIXME: what to do in terms of return here?
     return promise.id
 
 

swh/lister/gnu/tasks.py

@@ -9,8 +9,8 @@ from .lister import GNULister
 
 @shared_task(name=__name__ + '.GNUListerTask')
 def list_gnu_full(**lister_args):
-    'List lister for the GNU source code archive'
-    GNULister(**lister_args).run()
+    """List lister for the GNU source code archive"""
+    return GNULister(**lister_args).run()
 
 
 @shared_task(name=__name__ + '.ping')

swh/lister/npm/tasks.py

@@ -45,7 +45,7 @@ def list_npm_full(**lister_args):
     'Full lister for the npm (javascript) registry'
     lister = NpmLister(**lister_args)
     with save_registry_state(lister):
-        lister.run()
+        return lister.run()
 
 
 @shared_task(name=__name__ + '.NpmIncrementalListerTask')
@@ -54,7 +54,7 @@ def list_npm_incremental(**lister_args):
     lister = NpmIncrementalLister(**lister_args)
     update_seq_start = get_last_update_seq(lister)
     with save_registry_state(lister):
-        lister.run(min_bound=update_seq_start)
+        return lister.run(min_bound=update_seq_start)
 
 
 @shared_task(name=__name__ + '.ping')

swh/lister/phabricator/tasks.py

@@ -9,8 +9,8 @@ from swh.lister.phabricator.lister import PhabricatorLister
 
 @shared_task(name=__name__ + '.FullPhabricatorLister')
 def list_phabricator_full(**lister_args):
-    'Full update of a Phabricator instance'
-    PhabricatorLister(**lister_args).run()
+    """Full update of a Phabricator instance"""
+    return PhabricatorLister(**lister_args).run()
 
 
 @shared_task(name=__name__ + '.ping')

swh/lister/pypi/tasks.py

@@ -10,7 +10,7 @@ from .lister import PyPILister
 @shared_task(name=__name__ + '.PyPIListerTask')
 def list_pypi(**lister_args):
     'Full update of the PyPI (python) registry'
-    PyPILister(**lister_args).run()
+    return PyPILister(**lister_args).run()
 
 
 @shared_task(name=__name__ + '.ping')