Release version 0.19.2

Pull Details

CHANGES.rst

@@ -1,6 +1,40 @@
 Changelog
 =========
 
+0.19.2 (2018-03-24)
+-------------------
+
+Fix
+~~~
+
+- Cleanup pep8 violations. [Jose Diaz-Gonzalez]
+
+0.19.0 (2018-03-24)
+-------------------
+
+- Add additional output for the current request. [Robin Gloster]
+
+  This is useful to have some progress indication for huge repositories.
+
+
+- Add option to backup additional PR details. [Robin Gloster]
+
+  Some payload is only included when requesting a single pull request
+
+
+- Mark string as binary in comparison for skip_existing. [Johannes
+  Bornhold]
+
+  Found out that the flag "--skip-existing" did not work out as expected on Python
+  3.6. Tracked it down to the comparison which has to be against a string of bytes
+  in Python3.
+
+
+0.18.0 (2018-02-22)
+-------------------
+
+- Add option to fetch followers/following JSON data. [Stephen Greene]
+
 0.17.0 (2018-02-20)
 -------------------
 

README.rst

@@ -28,12 +28,12 @@ Usage
 CLI Usage is as follows::
 
     github-backup [-h] [-u USERNAME] [-p PASSWORD] [-t TOKEN]
-                  [-o OUTPUT_DIRECTORY] [-i] [--starred] [--watched]
+                  [-o OUTPUT_DIRECTORY] [-i] [--starred] [--all-starred]
-                  [--all] [--issues] [--issue-comments] [--issue-events]
+                  [--watched] [--followers] [--following] [--all]
-                  [--pulls] [--pull-comments] [--pull-commits] [--labels]
+                  [--issues] [--issue-comments] [--issue-events] [--pulls]
-                  [--hooks] [--milestones] [--repositories] [--bare] [--lfs]
+                  [--pull-comments] [--pull-commits] [--labels] [--hooks]
-                  [--wikis] [--skip-existing] [--all-starred]
+                  [--milestones] [--repositories] [--bare] [--lfs]
-                  [--gists] [--starred-gists]
+                  [--wikis] [--gists] [--starred-gists] [--skip-existing]
                   [-L [LANGUAGES [LANGUAGES ...]]] [-N NAME_REGEX]
                   [-H GITHUB_HOST] [-O] [-R REPOSITORY] [-P] [-F]
                   [--prefer-ssh] [-v]
@@ -54,12 +54,16 @@ CLI Usage is as follows::
                             password for basic auth. If a username is given but
                             not a password, the password will be prompted for.
       -t TOKEN, --token TOKEN
-                            personal access or OAuth token
+                            personal access or OAuth token, or path to token
+                            (file://...)
       -o OUTPUT_DIRECTORY, --output-directory OUTPUT_DIRECTORY
                             directory at which to backup the repositories
       -i, --incremental     incremental backup
       --starred             include JSON output of starred repositories in backup
+      --all-starred         include starred repositories in backup
       --watched             include watched repositories in backup
+      --followers           include JSON output of followers in backup
+      --following           include JSON output of following users in backup
       --all                 include everything in backup
       --issues              include issues in backup
       --issue-comments      include issue comments in backup
@@ -73,12 +77,12 @@ CLI Usage is as follows::
       --milestones          include milestones in backup
       --repositories        include repository clone in backup
       --bare                clone bare repositories
-      --lfs                 clone LFS repositories (requires Git LFS to be installed, https://git-lfs.github.com)
+      --lfs                 clone LFS repositories (requires Git LFS to be
+                            installed, https://git-lfs.github.com)
       --wikis               include wiki clone in backup
-      --skip-existing       skip project if a backup directory exists
-      --all-starred         include starred repositories in backup
       --gists               include gists in backup
       --starred-gists       include starred gists in backup
+      --skip-existing       skip project if a backup directory exists
       -L [LANGUAGES [LANGUAGES ...]], --languages [LANGUAGES [LANGUAGES ...]]
                             only allow these languages
       -N NAME_REGEX, --name-regex NAME_REGEX

bin/github-backup

@@ -168,6 +168,14 @@ def parse_args():
                         action='store_true',
                         dest='include_watched',
                         help='include watched repositories in backup')
+    parser.add_argument('--followers',
+                        action='store_true',
+                        dest='include_followers',
+                        help='include JSON output of followers in backup')
+    parser.add_argument('--following',
+                        action='store_true',
+                        dest='include_following',
+                        help='include JSON output of following users in backup')
     parser.add_argument('--all',
                         action='store_true',
                         dest='include_everything',
@@ -196,6 +204,10 @@ def parse_args():
                         action='store_true',
                         dest='include_pull_commits',
                         help='include pull request commits in backup')
+    parser.add_argument('--pull-details',
+                        action='store_true',
+                        dest='include_pull_details',
+                        help='include more pull request details in backup')
     parser.add_argument('--labels',
                         action='store_true',
                         dest='include_labels',
@@ -291,12 +303,12 @@ def get_auth(args, encode=True):
             if platform.system() != 'Darwin':
                 log_error("Keychain arguments are only supported on Mac OSX")
             try:
-                with open(os.devnull,'w') as devnull:
+                with open(os.devnull, 'w') as devnull:
                     token = (subprocess.check_output([
-                        'security','find-generic-password',
+                        'security', 'find-generic-password',
-                        '-s',args.osx_keychain_item_name,
+                        '-s', args.osx_keychain_item_name,
-                        '-a',args.osx_keychain_item_account,
+                        '-a', args.osx_keychain_item_account,
-                        '-w' ], stderr=devnull).strip())
+                        '-w'], stderr=devnull).strip())
                 auth = token + ':' + 'x-oauth-basic'
             except:
                 log_error('No password item matching the provided name and account could be found in the osx keychain.')
@@ -442,6 +454,7 @@ def _construct_request(per_page, page, query_args, template, auth):
     request = Request(template + '?' + querystring)
     if auth is not None:
         request.add_header('Authorization', 'Basic '.encode('ascii') + auth)
+    log_info('Requesting {}?{}'.format(template, querystring))
     return request
 
 
@@ -717,23 +730,35 @@ def backup_pulls(args, repo_cwd, repository, repos_template):
     pulls = {}
     _pulls_template = '{0}/{1}/pulls'.format(repos_template,
                                              repository['full_name'])
+    query_args = {
+        'filter': 'all',
+        'state': 'all',
+        'sort': 'updated',
+        'direction': 'desc',
+    }
 
-    pull_states = ['open', 'closed']
+    if not args.include_pull_details:
-    for pull_state in pull_states:
+        pull_states = ['open', 'closed']
-        query_args = {
+        for pull_state in pull_states:
-            'filter': 'all',
+            query_args['state'] = pull_state
-            'state': pull_state,
+            # It'd be nice to be able to apply the args.since filter here...
-            'sort': 'updated',
+            _pulls = retrieve_data(args,
-            'direction': 'desc',
+                                   _pulls_template,
-        }
+                                   query_args=query_args)
-
+            for pull in _pulls:
-        # It'd be nice to be able to apply the args.since filter here...
+                if not args.since or pull['updated_at'] >= args.since:
+                    pulls[pull['number']] = pull
+    else:
         _pulls = retrieve_data(args,
                                _pulls_template,
                                query_args=query_args)
         for pull in _pulls:
             if not args.since or pull['updated_at'] >= args.since:
-                pulls[pull['number']] = pull
+                pulls[pull['number']] = retrieve_data(
+                    args,
+                    _pulls_template + '/{}'.format(pull['number']),
+                    single_request=True
+                )
 
     log_info('Saving {0} pull requests to disk'.format(
         len(list(pulls.keys()))))
@@ -823,7 +848,7 @@ def fetch_repository(name,
             clone_exists = subprocess.check_output(['git',
                                                     'rev-parse',
                                                     '--is-bare-repository'],
-                                                   cwd=local_dir) == "true\n"
+                                                   cwd=local_dir) == b"true\n"
         else:
             clone_exists = False
     else:
@@ -886,21 +911,37 @@ def backup_account(args, output_directory):
     account_cwd = os.path.join(output_directory, 'account')
 
     if args.include_starred or args.include_everything:
-        output_file = '{0}/starred.json'.format(account_cwd)
+        output_file = "{0}/starred.json".format(account_cwd)
-        template = "https://{0}/users/{1}/starred"
+        template = "https://{0}/users/{1}/starred".format(get_github_api_host(args), args.user)
-        template = template.format(get_github_api_host(args), args.user)
         _backup_data(args,
-                     'starred repositories',
+                     "starred repositories",
                      template,
                      output_file,
                      account_cwd)
 
     if args.include_watched or args.include_everything:
-        output_file = '{0}/watched.json'.format(account_cwd)
+        output_file = "{0}/watched.json".format(account_cwd)
-        template = "https://{0}/users/{1}/subscriptions"
+        template = "https://{0}/users/{1}/subscriptions".format(get_github_api_host(args), args.user)
-        template = template.format(get_github_api_host(args), args.user)
         _backup_data(args,
-                     'watched repositories',
+                     "watched repositories",
+                     template,
+                     output_file,
+                     account_cwd)
+
+    if args.include_followers or args.include_everything:
+        output_file = "{0}/followers.json".format(account_cwd)
+        template = "https://{0}/users/{1}/followers".format(get_github_api_host(args), args.user)
+        _backup_data(args,
+                     "followers",
+                     template,
+                     output_file,
+                     account_cwd)
+
+    if args.include_following or args.include_everything:
+        output_file = "{0}/following.json".format(account_cwd)
+        template = "https://{0}/users/{1}/following".format(get_github_api_host(args), args.user)
+        _backup_data(args,
+                     "following",
                      template,
                      output_file,
                      account_cwd)

github_backup/__init__.py

@@ -1 +1 @@
-__version__ = '0.17.0'
+__version__ = '0.19.2'