Release version 0.35.0

Make API request throttling optional

CHANGES.rst

@@ -1,9 +1,35 @@
 Changelog
 =========
 
-0.31.0 (2020-02-25)
+0.35.0 (2020-08-05)
 -------------------
 ------------------------
+- Make API request throttling optional. [Samantha Baldwin]
+
+
+0.34.0 (2020-07-24)
+-------------------
+- Add logic for transforming gist repository urls to ssh. [Matt Fields]
+
+
+0.33.0 (2020-04-13)
+-------------------
+- Add basic API request throttling. [Enrico Tröger]
+
+  A simple approach to throttle API requests and so keep within the rate
+  limits of the API. Can be enabled with "--throttle-limit" to specify
+  when throttling should start.
+  "--throttle-pause" defines the time to sleep between further API
+  requests.
+
+
+0.32.0 (2020-04-13)
+-------------------
+- Add timestamp to log messages. [Enrico Tröger]
+
+
+0.31.0 (2020-02-25)
+-------------------
 - #123 update: changed --as-app 'help' description. [ethan]
 - #123: Support Authenticating As Github Application. [ethan]
 

README.rst

@@ -29,19 +29,20 @@ Usage
 
 CLI Usage is as follows::
 
-    github-backup [-h] [-u USERNAME] [-p PASSWORD] [-t TOKEN]
+    github-backup [-h] [-u USERNAME] [-p PASSWORD] [-t TOKEN] [--as-app]
                   [-o OUTPUT_DIRECTORY] [-i] [--starred] [--all-starred]
                   [--watched] [--followers] [--following] [--all]
                   [--issues] [--issue-comments] [--issue-events] [--pulls]
-                  [--pull-comments] [--pull-commits] [--labels] [--hooks]
-                  [--milestones] [--repositories] [--releases] [--assets]
+                  [--pull-comments] [--pull-commits] [--pull-details]
+                  [--labels] [--hooks] [--milestones] [--repositories]
                   [--bare] [--lfs] [--wikis] [--gists] [--starred-gists]
-                  [--skip-existing]
-                  [-L [LANGUAGES [LANGUAGES ...]]] [-N NAME_REGEX]
-                  [-H GITHUB_HOST] [-O] [-R REPOSITORY] [-P] [-F]
-                  [--prefer-ssh] [-v]
+                  [--skip-existing] [-L [LANGUAGES [LANGUAGES ...]]]
+                  [-N NAME_REGEX] [-H GITHUB_HOST] [-O] [-R REPOSITORY]
+                  [-P] [-F] [--prefer-ssh] [-v]
                   [--keychain-name OSX_KEYCHAIN_ITEM_NAME]
                   [--keychain-account OSX_KEYCHAIN_ITEM_ACCOUNT]
+                  [--releases] [--assets] [--throttle-limit THROTTLE_LIMIT]
+                  [--throttle-pause THROTTLE_PAUSE]
                   USER
 
     Backup a github account
@@ -57,36 +58,36 @@ CLI Usage is as follows::
                             password for basic auth. If a username is given but
                             not a password, the password will be prompted for.
       -t TOKEN, --token TOKEN
-                            personal access or OAuth token, or path to token
-                            (file://...)
+                            personal access, OAuth, or JSON Web token, or path to
+                            token (file://...)
+      --as-app              authenticate as github app instead of as a user.
       -o OUTPUT_DIRECTORY, --output-directory OUTPUT_DIRECTORY
                             directory at which to backup the repositories
       -i, --incremental     incremental backup
       --starred             include JSON output of starred repositories in backup
-      --all-starred         include starred repositories in backup
-      --watched             include watched repositories in backup
+      --all-starred         include starred repositories in backup [*]
+      --watched             include JSON output of watched repositories in backup
       --followers           include JSON output of followers in backup
       --following           include JSON output of following users in backup
-      --all                 include everything in backup
+      --all                 include everything in backup (not including [*])
       --issues              include issues in backup
       --issue-comments      include issue comments in backup
       --issue-events        include issue events in backup
       --pulls               include pull requests in backup
       --pull-comments       include pull request review comments in backup
       --pull-commits        include pull request commits in backup
+      --pull-details        include more pull request details in backup [*]
       --labels              include labels in backup
       --hooks               include hooks in backup (works only when
                             authenticated)
       --milestones          include milestones in backup
       --repositories        include repository clone in backup
-      --releases            include repository releases' information without assets or binaries
-      --assets              include assets alongside release information; only applies if including releases
       --bare                clone bare repositories
       --lfs                 clone LFS repositories (requires Git LFS to be
-                            installed, https://git-lfs.github.com)
+                            installed, https://git-lfs.github.com) [*]
       --wikis               include wiki clone in backup
-      --gists               include gists in backup
-      --starred-gists       include starred gists in backup
+      --gists               include gists in backup [*]
+      --starred-gists       include starred gists in backup [*]
       --skip-existing       skip project if a backup directory exists
       -L [LANGUAGES [LANGUAGES ...]], --languages [LANGUAGES [LANGUAGES ...]]
                             only allow these languages
@@ -97,8 +98,8 @@ CLI Usage is as follows::
       -O, --organization    whether or not this is an organization user
       -R REPOSITORY, --repository REPOSITORY
                             name of repository to limit backup to
-      -P, --private         include private repositories
-      -F, --fork            include forked repositories
+      -P, --private         include private repositories [*]
+      -F, --fork            include forked repositories [*]
       --prefer-ssh          Clone repositories using SSH instead of HTTPS
       -v, --version         show program's version number and exit
       --keychain-name OSX_KEYCHAIN_ITEM_NAME
@@ -107,6 +108,17 @@ CLI Usage is as follows::
       --keychain-account OSX_KEYCHAIN_ITEM_ACCOUNT
                             OSX ONLY: account field of password item in OSX
                             keychain that holds the personal access or OAuth token
+      --releases            include release information, not including assets or
+                            binaries
+      --assets              include assets alongside release information; only
+                            applies if including releases
+      --throttle-limit THROTTLE_LIMIT
+                            start throttling of GitHub API requests after this
+                            amount of API requests remain
+      --throttle-pause THROTTLE_PAUSE
+                            wait this amount of seconds when API request
+                            throttling is active (default: 30.0, requires
+                            --throttle-limit to be set)
 
 
 The package can be used to backup an *entire* organization or repository, including issues and wikis in the most appropriate format (clones for wikis, json files for issues).

github_backup/__init__.py

@@ -1 +1 @@
-__version__ = '0.31.0'
+__version__ = '0.35.0'

github_backup/github_backup.py

@@ -7,6 +7,7 @@ import argparse
 import base64
 import calendar
 import codecs
+import datetime
 import errno
 import getpass
 import json
@@ -50,6 +51,10 @@ except ImportError:
 FNULL = open(os.devnull, 'w')
 
 
+def _get_log_date():
+    return datetime.datetime.isoformat(datetime.datetime.now())
+
+
 def log_error(message):
     """
     Log message (str) or messages (List[str]) to stderr and exit with status 1
@@ -66,7 +71,7 @@ def log_info(message):
         message = [message]
 
     for msg in message:
-        sys.stdout.write("{0}\n".format(msg))
+        sys.stdout.write("{0}: {1}\n".format(_get_log_date(), msg))
 
 
 def log_warning(message):
@@ -77,7 +82,7 @@ def log_warning(message):
         message = [message]
 
     for msg in message:
-        sys.stderr.write("{0}\n".format(msg))
+        sys.stderr.write("{0}: {1}\n".format(_get_log_date(), msg))
 
 
 def logging_subprocess(popenargs,
@@ -326,6 +331,16 @@ def parse_args():
                         action='store_true',
                         dest='include_assets',
                         help='include assets alongside release information; only applies if including releases')
+    parser.add_argument('--throttle-limit',
+                        dest='throttle_limit',
+                        type=int,
+                        default=0,
+                        help='start throttling of GitHub API requests after this amount of API requests remain')
+    parser.add_argument('--throttle-pause',
+                        dest='throttle_pause',
+                        type=float,
+                        default=30.0,
+                        help='wait this amount of seconds when API request throttling is active (default: 30.0, requires --throttle-limit to be set)')
     return parser.parse_args()
 
 
@@ -404,7 +419,13 @@ def get_github_host(args):
 
 def get_github_repo_url(args, repository):
     if repository.get('is_gist'):
-        return repository['git_pull_url']
+        if args.prefer_ssh:
+            # The git_pull_url value is always https for gists, so we need to transform it to ssh form
+            repo_url = re.sub('^https?:\/\/(.+)\/(.+)\.git$', r'git@\1:\2.git', repository['git_pull_url'])
+            repo_url = re.sub('^git@gist\.', 'git@', repo_url) # strip gist subdomain for better hostkey compatibility
+        else:
+            repo_url = repository['git_pull_url']
+        return repo_url
 
     if args.prefer_ssh:
         return repository['ssh_url']
@@ -434,10 +455,18 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False):
         r, errors = _get_response(request, auth, template)
 
         status_code = int(r.getcode())
+        # be gentle with API request limit and throttle requests if remaining requests getting low
+        limit_remaining = int(r.headers.get('x-ratelimit-remaining', 0))
+        if args.throttle_limit and limit_remaining <= args.throttle_limit:
+            log_info(
+                'API request limit hit: {} requests left, pausing further requests for {}s'.format(
+                    limit_remaining,
+                    args.throttle_pause))
+            time.sleep(args.throttle_pause)
 
         retries = 0
         while retries < 3 and status_code == 502:
-            print('API request returned HTTP 502: Bad Gateway. Retrying in 5 seconds')
+            log_warning('API request returned HTTP 502: Bad Gateway. Retrying in 5 seconds')
             retries += 1
             time.sleep(5)
             request = _construct_request(per_page, page, query_args, template, auth, as_app=args.as_app)  # noqa
@@ -544,12 +573,10 @@ def _request_http_error(exc, auth, errors):
         delta = max(10, reset - gm_now)
 
         limit = headers.get('x-ratelimit-limit')
-        print('Exceeded rate limit of {} requests; waiting {} seconds to reset'.format(limit, delta),  # noqa
-              file=sys.stderr)
+        log_warning('Exceeded rate limit of {} requests; waiting {} seconds to reset'.format(limit, delta))  # noqa
 
         if auth is None:
-            print('Hint: Authenticate to raise your GitHub rate limit',
-                  file=sys.stderr)
+            log_info('Hint: Authenticate to raise your GitHub rate limit')
 
         time.sleep(delta)
         should_continue = True