Release version 0.33.1

Add basic API request throttling

CHANGES.rst

@@ -1,9 +1,20 @@
 Changelog
 =========
 
+0.33.1 (2020-05-28)
+-------------------
+-------------------
+- Add basic API request throttling. [Enrico Tröger]
+
+  A simple approach to throttle API requests and so keep within the rate
+  limits of the API. Can be enabled with "--throttle-limit" to specify
+  when throttling should start.
+  "--throttle-pause" defines the time to sleep between further API
+  requests.
+
+
 0.32.0 (2020-04-13)
 -------------------
-------------------------
 - Add timestamp to log messages. [Enrico Tröger]
 
 

README.rst

@@ -41,7 +41,8 @@ CLI Usage is as follows::
                   [-P] [-F] [--prefer-ssh] [-v]
                   [--keychain-name OSX_KEYCHAIN_ITEM_NAME]
                   [--keychain-account OSX_KEYCHAIN_ITEM_ACCOUNT]
-                  [--releases] [--assets]
+                  [--releases] [--assets] [--throttle-limit THROTTLE_LIMIT]
+                  [--throttle-pause THROTTLE_PAUSE]
                   USER
 
     Backup a github account
@@ -111,6 +112,13 @@ CLI Usage is as follows::
                             binaries
       --assets              include assets alongside release information; only
                             applies if including releases
+      --throttle-limit THROTTLE_LIMIT
+                            start throttling of GitHub API requests after this
+                            amount of API requests remain
+      --throttle-pause THROTTLE_PAUSE
+                            wait this amount of seconds when API request
+                            throttling is active (default: 30.0, requires
+                            --throttle-limit to be set)
 
 
 The package can be used to backup an *entire* organization or repository, including issues and wikis in the most appropriate format (clones for wikis, json files for issues).

github_backup/__init__.py

@@ -1 +1 @@
-__version__ = '0.32.0'
+__version__ = '0.33.1'

github_backup/github_backup.py

@@ -331,6 +331,16 @@ def parse_args():
                         action='store_true',
                         dest='include_assets',
                         help='include assets alongside release information; only applies if including releases')
+    parser.add_argument('--throttle-limit',
+                        dest='throttle_limit',
+                        type=int,
+                        default=0,
+                        help='start throttling of GitHub API requests after this amount of API requests remain')
+    parser.add_argument('--throttle-pause',
+                        dest='throttle_pause',
+                        type=float,
+                        default=30.0,
+                        help='wait this amount of seconds when API request throttling is active (default: 30.0, requires --throttle-limit to be set)')
     return parser.parse_args()
 
 
@@ -439,6 +449,14 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False):
         r, errors = _get_response(request, auth, template)
 
         status_code = int(r.getcode())
+        # be gentle with API request limit and throttle requests if remaining requests getting low
+        limit_remaining = int(r.headers.get('x-ratelimit-remaining', 0))
+        if limit_remaining <= args.throttle_limit:
+            log_info(
+                'API request limit hit: {} requests left, pausing further requests for {}s'.format(
+                    limit_remaining,
+                    args.throttle_pause))
+            time.sleep(args.throttle_pause)
 
         retries = 0
         while retries < 3 and status_code == 502: