From 78098aae23ef3c439e96e876c6931c59b17fe4bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enrico=20Tr=C3=B6ger?= Date: Mon, 13 Apr 2020 23:06:09 +0200 Subject: [PATCH] Add basic API request throttling A simple approach to throttle API requests and so keep within the rate limits of the API. Can be enabled with "--throttle-limit" to specify when throttling should start. "--throttle-pause" defines the time to sleep between further API requests. --- github_backup/github_backup.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/github_backup/github_backup.py b/github_backup/github_backup.py index 8798c85..178da62 100644 --- a/github_backup/github_backup.py +++ b/github_backup/github_backup.py @@ -331,6 +331,16 @@ def parse_args(): action='store_true', dest='include_assets', help='include assets alongside release information; only applies if including releases') + parser.add_argument('--throttle-limit', + dest='throttle_limit', + type=int, + default=0, + help='start throttling of GitHub API requests after this amount of API requests remain') + parser.add_argument('--throttle-pause', + dest='throttle_pause', + type=float, + default=30.0, + help='wait this amount of seconds when API request throttling is active (default: 30.0, requires --throttle-limit to be set)') return parser.parse_args() @@ -439,6 +449,14 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False): r, errors = _get_response(request, auth, template) status_code = int(r.getcode()) + # be gentle with API request limit and throttle requests if remaining requests getting low + limit_remaining = int(r.headers.get('x-ratelimit-remaining', 0)) + if limit_remaining <= args.throttle_limit: + log_info( + 'API request limit hit: {} requests left, pausing further requests for {}s'.format( + limit_remaining, + args.throttle_pause)) + time.sleep(args.throttle_pause) retries = 0 while retries < 3 and status_code == 502: