Release version 0.42.0

Add option to exclude repositories

README.rst

@@ -41,8 +41,8 @@ CLI Usage is as follows::
                   [-P] [-F] [--prefer-ssh] [-v]
                   [--keychain-name OSX_KEYCHAIN_ITEM_NAME]
                   [--keychain-account OSX_KEYCHAIN_ITEM_ACCOUNT]
-                  [--releases] [--assets] [--throttle-limit THROTTLE_LIMIT]
-                  [--throttle-pause THROTTLE_PAUSE]
+                  [--releases] [--assets] [--exclude [REPOSITORY [REPOSITORY ...]]
+                  [--throttle-limit THROTTLE_LIMIT] [--throttle-pause THROTTLE_PAUSE]
                   USER
 
     Backup a github account
@@ -112,6 +112,8 @@ CLI Usage is as follows::
                             binaries
       --assets              include assets alongside release information; only
                             applies if including releases
+      --exclude [REPOSITORY [REPOSITORY ...]]
+                            names of repositories to exclude from backup.
       --throttle-limit THROTTLE_LIMIT
                             start throttling of GitHub API requests after this
                             amount of API requests remain

github_backup/__init__.py

@@ -1 +1 @@
-__version__ = '0.40.1'
+__version__ = '0.42.0'

github_backup/github_backup.py

@@ -11,7 +11,6 @@ import datetime
 import errno
 import getpass
 import json
-import logging
 import os
 import re
 import select
@@ -42,6 +41,7 @@ FNULL = open(os.devnull, 'w')
 def _get_log_date():
     return datetime.datetime.isoformat(datetime.datetime.now())
 
+
 def log_info(message):
     """
     Log message (str) or messages (List[str]) to stdout
@@ -133,7 +133,7 @@ def mask_password(url, secret='*****'):
     return url.replace(parsed.password, secret)
 
 
-def parse_args(args = None):
+def parse_args(args=None):
     parser = argparse.ArgumentParser(description='Backup a github account')
     parser.add_argument('user',
                         metavar='USER',
@@ -239,6 +239,10 @@ def parse_args(args = None):
                         action='store_true',
                         dest='bare_clone',
                         help='clone bare repositories')
+    parser.add_argument('--no-prune',
+                        action='store_true',
+                        dest='no_prune',
+                        help='disable prune option for git fetch')
     parser.add_argument('--lfs',
                         action='store_true',
                         dest='lfs_clone',
@@ -324,6 +328,10 @@ def parse_args(args = None):
                         type=float,
                         default=30.0,
                         help='wait this amount of seconds when API request throttling is active (default: 30.0, requires --throttle-limit to be set)')
+    parser.add_argument('--exclude',
+                        dest='exclude',
+                        help='names of repositories to exclude',
+                        nargs="*")
     return parser.parse_args(args)
 
 
@@ -533,12 +541,12 @@ def _get_response(request, auth, template):
             r = exc
         except URLError as e:
             log_warning(e.reason)
-            should_continue = _request_url_error(template, retry_timeout)
+            should_continue, retry_timeout = _request_url_error(template, retry_timeout)
             if not should_continue:
                 raise
         except socket.error as e:
             log_warning(e.strerror)
-            should_continue = _request_url_error(template, retry_timeout)
+            should_continue, retry_timeout = _request_url_error(template, retry_timeout)
             if not should_continue:
                 raise
 
@@ -598,16 +606,15 @@ def _request_http_error(exc, auth, errors):
 
 
 def _request_url_error(template, retry_timeout):
-    # Incase of a connection timing out, we can retry a few time
+    # In case of a connection timing out, we can retry a few time
     # But we won't crash and not back-up the rest now
     log_info('{} timed out'.format(template))
     retry_timeout -= 1
 
     if retry_timeout >= 0:
-        return True
+        return True, retry_timeout
 
     raise Exception('{} timed out to much, skipping!')
-    return False
 
 
 class S3HTTPRedirectHandler(HTTPRedirectHandler):
@@ -750,6 +757,8 @@ def filter_repositories(args, unfiltered_repositories):
         repositories = [r for r in repositories if name_regex.match(r['name'])]
     if args.skip_archived:
         repositories = [r for r in repositories if not r.get('archived')]
+    if args.exclude:
+        repositories = [r for r in repositories if r['name'] not in args.exclude]
 
     return repositories
 
@@ -790,7 +799,8 @@ def backup_repositories(args, output_directory, repositories):
                              repo_dir,
                              skip_existing=args.skip_existing,
                              bare_clone=args.bare_clone,
-                             lfs_clone=args.lfs_clone)
+                             lfs_clone=args.lfs_clone,
+                             no_prune=args.no_prune)
 
             if repository.get('is_gist'):
                 # dump gist information to a file as well
@@ -807,8 +817,9 @@ def backup_repositories(args, output_directory, repositories):
                              os.path.join(repo_cwd, 'wiki'),
                              skip_existing=args.skip_existing,
                              bare_clone=args.bare_clone,
-                             lfs_clone=args.lfs_clone)
-
+                             lfs_clone=args.lfs_clone,
+                             no_prune=args.no_prune
+                             )
         if args.include_issues or args.include_everything:
             backup_issues(args, repo_cwd, repository, repos_template)
 
@@ -902,6 +913,8 @@ def backup_pulls(args, repo_cwd, repository, repos_template):
     pulls = {}
     _pulls_template = '{0}/{1}/pulls'.format(repos_template,
                                              repository['full_name'])
+    _issue_template = '{0}/{1}/issues'.format(repos_template,
+                                              repository['full_name'])
     query_args = {
         'filter': 'all',
         'state': 'all',
@@ -941,10 +954,17 @@ def backup_pulls(args, repo_cwd, repository, repos_template):
 
     log_info('Saving {0} pull requests to disk'.format(
         len(list(pulls.keys()))))
+    # Comments from pulls API are only _review_ comments
+    # regular comments need to be fetched via issue API.
+    # For backwards compatibility with versions <= 0.41.0
+    # keep name "comment_data" for review comments
+    comments_regular_template = _issue_template + '/{0}/comments'
     comments_template = _pulls_template + '/{0}/comments'
     commits_template = _pulls_template + '/{0}/commits'
     for number, pull in list(pulls.items()):
         if args.include_pull_comments or args.include_everything:
+            template = comments_regular_template.format(number)
+            pulls[number]['comment_regular_data'] = retrieve_data(args, template)
             template = comments_template.format(number)
             pulls[number]['comment_data'] = retrieve_data(args, template)
         if args.include_pull_commits or args.include_everything:
@@ -1053,7 +1073,8 @@ def fetch_repository(name,
                      local_dir,
                      skip_existing=False,
                      bare_clone=False,
-                     lfs_clone=False):
+                     lfs_clone=False,
+                     no_prune=False):
     if bare_clone:
         if os.path.exists(local_dir):
             clone_exists = subprocess.check_output(['git',
@@ -1099,6 +1120,8 @@ def fetch_repository(name,
             git_command = ['git', 'lfs', 'fetch', '--all', '--prune']
         else:
             git_command = ['git', 'fetch', '--all', '--force', '--tags', '--prune']
+        if no_prune:
+            git_command.pop()
         logging_subprocess(git_command, None, cwd=local_dir)
     else:
         log_info('Cloning {0} repository from {1} to {2}'.format(
@@ -1106,16 +1129,19 @@ def fetch_repository(name,
             masked_remote_url,
             local_dir))
         if bare_clone:
+            git_command = ['git', 'clone', '--mirror', remote_url, local_dir]
+            logging_subprocess(git_command, None)
             if lfs_clone:
-                git_command = ['git', 'lfs', 'clone', '--mirror', remote_url, local_dir]
-            else:
-                git_command = ['git', 'clone', '--mirror', remote_url, local_dir]
+                git_command = ['git', 'lfs', 'fetch', '--all', '--prune']
+                if no_prune:
+                    git_command.pop()
+                logging_subprocess(git_command, None, cwd=local_dir)
         else:
             if lfs_clone:
                 git_command = ['git', 'lfs', 'clone', remote_url, local_dir]
             else:
                 git_command = ['git', 'clone', remote_url, local_dir]
-        logging_subprocess(git_command, None)
+            logging_subprocess(git_command, None)
 
 
 def backup_account(args, output_directory):