Wavyzz/python-github-backup
2
0
Fork 0
mirror of https://github.com/josegonzalez/python-github-backup.git synced 2025-12-05 08:08:02 +01:00
Code Issues Packages Projects Releases Wiki Activity

Issue 119: Change retrieve_data to be a generator

Browse Source 
See issue #119.
This commit is contained in:
2a
2019-07-03 23:01:00 +02:00
committed by GitHub
parent 6db5bd731b
commit e74765ba7f
1 changed files with 12 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
bin/github-backup
View File

@@ -406,12 +406,11 @@ def get_github_repo_url(args, repository):
return repo_url
def retrieve_data(args, template, query_args=None, single_request=False):
def retrieve_data_gen(args, template, query_args=None, single_request=False):
auth = get_auth(args)
query_args = get_query_args(query_args)
per_page = 100
page = 0
data = []
while True:
page = page + 1
@@ -438,11 +437,12 @@ def retrieve_data(args, template, query_args=None, single_request=False):
response = json.loads(r.read().decode('utf-8'))
if len(errors) == 0:
if type(response) == list:
data.extend(response)
for resp in response:
yield resp
if len(response) < per_page:
break
elif type(response) == dict and single_request:
data.append(response)
yield response
if len(errors) > 0:
log_error(errors)
@@ -450,8 +450,8 @@ def retrieve_data(args, template, query_args=None, single_request=False):
if single_request:
break
return data
def retrieve_data(args, template, query_args=None, single_request=False):
return list(retrieve_data_gen(args, template, query_args, single_request))
def get_query_args(query_args=None):
if not query_args:
@@ -836,18 +836,21 @@ def backup_pulls(args, repo_cwd, repository, repos_template):
pull_states = ['open', 'closed']
for pull_state in pull_states:
query_args['state'] = pull_state
# It'd be nice to be able to apply the args.since filter here...
_pulls = retrieve_data(args,
_pulls = retrieve_data_gen(args,
_pulls_template,
query_args=query_args)
for pull in _pulls:
if args.since and pull['updated_at'] < args.since:
break
if not args.since or pull['updated_at'] >= args.since:
pulls[pull['number']] = pull
else:
_pulls = retrieve_data(args,
_pulls = retrieve_data_gen(args,
_pulls_template,
query_args=query_args)
for pull in _pulls:
if args.since and pull['updated_at'] < args.since:
break
if not args.since or pull['updated_at'] >= args.since:
pulls[pull['number']] = retrieve_data(
args,