Release version 0.25.0

Issue 119: Change retrieve_data to be a generator

CHANGES.rst

@@ -1,9 +1,16 @@
 Changelog
 =========
 
-0.24.0 (2019-06-27)
+0.25.0 (2019-07-03)
 -------------------
 ------------------------
+- Issue 119: Change retrieve_data to be a generator. [2a]
+
+  See issue #119.
+
+
+0.24.0 (2019-06-27)
+-------------------
 - QKT-45: include assets - update readme. [Ethan Timm]
 
   update readme with flag information for including assets alongside their respective releases

bin/github-backup

@@ -406,12 +406,11 @@ def get_github_repo_url(args, repository):
     return repo_url
 
 
-def retrieve_data(args, template, query_args=None, single_request=False):
+def retrieve_data_gen(args, template, query_args=None, single_request=False):
     auth = get_auth(args)
     query_args = get_query_args(query_args)
     per_page = 100
     page = 0
-    data = []
 
     while True:
         page = page + 1
@@ -438,11 +437,12 @@ def retrieve_data(args, template, query_args=None, single_request=False):
         response = json.loads(r.read().decode('utf-8'))
         if len(errors) == 0:
             if type(response) == list:
-                data.extend(response)
+                for resp in response:
+                    yield resp
                 if len(response) < per_page:
                     break
             elif type(response) == dict and single_request:
-                data.append(response)
+                yield response
 
         if len(errors) > 0:
             log_error(errors)
@@ -450,8 +450,8 @@ def retrieve_data(args, template, query_args=None, single_request=False):
         if single_request:
             break
 
-    return data
+def retrieve_data(args, template, query_args=None, single_request=False):
-
+    return list(retrieve_data_gen(args, template, query_args, single_request))
 
 def get_query_args(query_args=None):
     if not query_args:
@@ -836,18 +836,21 @@ def backup_pulls(args, repo_cwd, repository, repos_template):
         pull_states = ['open', 'closed']
         for pull_state in pull_states:
             query_args['state'] = pull_state
-            # It'd be nice to be able to apply the args.since filter here...
+            _pulls = retrieve_data_gen(args,
-            _pulls = retrieve_data(args,
                                    _pulls_template,
                                    query_args=query_args)
             for pull in _pulls:
+                if args.since and pull['updated_at'] < args.since:
+                    break
                 if not args.since or pull['updated_at'] >= args.since:
                     pulls[pull['number']] = pull
     else:
-        _pulls = retrieve_data(args,
+        _pulls = retrieve_data_gen(args,
                                _pulls_template,
                                query_args=query_args)
         for pull in _pulls:
+            if args.since and pull['updated_at'] < args.since:
+                break
             if not args.since or pull['updated_at'] >= args.since:
                 pulls[pull['number']] = retrieve_data(
                     args,

github_backup/__init__.py

@@ -1 +1 @@
-__version__ = '0.24.0'
+__version__ = '0.25.0'