Release version 0.19.1

Pull Details

CHANGES.rst

@@ -1,6 +1,41 @@
 Changelog
 =========
 
+0.19.1 (2018-03-24)
+-------------------
+
+- Add additional output for the current request. [Robin Gloster]
+
+  This is useful to have some progress indication for huge repositories.
+
+
+- Add option to backup additional PR details. [Robin Gloster]
+
+  Some payload is only included when requesting a single pull request
+
+
+- Mark string as binary in comparison for skip_existing. [Johannes
+  Bornhold]
+
+  Found out that the flag "--skip-existing" did not work out as expected on Python
+  3.6. Tracked it down to the comparison which has to be against a string of bytes
+  in Python3.
+
+
+0.18.0 (2018-02-22)
+-------------------
+
+- Add option to fetch followers/following JSON data. [Stephen Greene]
+
+0.17.0 (2018-02-20)
+-------------------
+
+- Short circuit gists backup process. [W. Harrison Wright]
+
+- Formatting. [W. Harrison Wright]
+
+- Add ability to backup gists. [W. Harrison Wright]
+
 0.16.0 (2018-01-22)
 -------------------
 

README.rst

@@ -28,11 +28,12 @@ Usage
 CLI Usage is as follows::
 
     github-backup [-h] [-u USERNAME] [-p PASSWORD] [-t TOKEN]
-                  [-o OUTPUT_DIRECTORY] [-i] [--starred] [--watched]
+                  [-o OUTPUT_DIRECTORY] [-i] [--starred] [--all-starred]
-                  [--all] [--issues] [--issue-comments] [--issue-events]
+                  [--watched] [--followers] [--following] [--all]
-                  [--pulls] [--pull-comments] [--pull-commits] [--labels]
+                  [--issues] [--issue-comments] [--issue-events] [--pulls]
-                  [--hooks] [--milestones] [--repositories] [--bare] [--lfs]
+                  [--pull-comments] [--pull-commits] [--labels] [--hooks]
-                  [--wikis] [--skip-existing] [--all-starred]
+                  [--milestones] [--repositories] [--bare] [--lfs]
+                  [--wikis] [--gists] [--starred-gists] [--skip-existing]
                   [-L [LANGUAGES [LANGUAGES ...]]] [-N NAME_REGEX]
                   [-H GITHUB_HOST] [-O] [-R REPOSITORY] [-P] [-F]
                   [--prefer-ssh] [-v]
@@ -53,12 +54,16 @@ CLI Usage is as follows::
                             password for basic auth. If a username is given but
                             not a password, the password will be prompted for.
       -t TOKEN, --token TOKEN
-                            personal access or OAuth token
+                            personal access or OAuth token, or path to token
+                            (file://...)
       -o OUTPUT_DIRECTORY, --output-directory OUTPUT_DIRECTORY
                             directory at which to backup the repositories
       -i, --incremental     incremental backup
       --starred             include JSON output of starred repositories in backup
+      --all-starred         include starred repositories in backup
       --watched             include watched repositories in backup
+      --followers           include JSON output of followers in backup
+      --following           include JSON output of following users in backup
       --all                 include everything in backup
       --issues              include issues in backup
       --issue-comments      include issue comments in backup
@@ -72,10 +77,12 @@ CLI Usage is as follows::
       --milestones          include milestones in backup
       --repositories        include repository clone in backup
       --bare                clone bare repositories
-      --lfs                 clone LFS repositories (requires Git LFS to be installed, https://git-lfs.github.com)
+      --lfs                 clone LFS repositories (requires Git LFS to be
+                            installed, https://git-lfs.github.com)
       --wikis               include wiki clone in backup
+      --gists               include gists in backup
+      --starred-gists       include starred gists in backup
       --skip-existing       skip project if a backup directory exists
-      --all-starred         include starred repositories in backup
       -L [LANGUAGES [LANGUAGES ...]], --languages [LANGUAGES [LANGUAGES ...]]
                             only allow these languages
       -N NAME_REGEX, --name-regex NAME_REGEX

bin/github-backup

@@ -168,6 +168,14 @@ def parse_args():
                         action='store_true',
                         dest='include_watched',
                         help='include watched repositories in backup')
+    parser.add_argument('--followers',
+                        action='store_true',
+                        dest='include_followers',
+                        help='include JSON output of followers in backup')
+    parser.add_argument('--following',
+                        action='store_true',
+                        dest='include_following',
+                        help='include JSON output of following users in backup')
     parser.add_argument('--all',
                         action='store_true',
                         dest='include_everything',
@@ -196,6 +204,10 @@ def parse_args():
                         action='store_true',
                         dest='include_pull_commits',
                         help='include pull request commits in backup')
+    parser.add_argument('--pull-details',
+                        action='store_true',
+                        dest='include_pull_details',
+                        help='include more pull request details in backup')
     parser.add_argument('--labels',
                         action='store_true',
                         dest='include_labels',
@@ -224,6 +236,14 @@ def parse_args():
                         action='store_true',
                         dest='include_wiki',
                         help='include wiki clone in backup')
+    parser.add_argument('--gists',
+                        action='store_true',
+                        dest='include_gists',
+                        help='include gists in backup')
+    parser.add_argument('--starred-gists',
+                        action='store_true',
+                        dest='include_starred_gists',
+                        help='include starred gists in backup')
     parser.add_argument('--skip-existing',
                         action='store_true',
                         dest='skip_existing',
@@ -342,6 +362,9 @@ def get_github_repo_url(args, repository):
     if args.prefer_ssh:
         return repository['ssh_url']
 
+    if repository.get('is_gist'):
+        return repository['git_pull_url']
+
     auth = get_auth(args, False)
     if auth:
         repo_url = 'https://{0}@{1}/{2}/{3}.git'.format(
@@ -431,6 +454,7 @@ def _construct_request(per_page, page, query_args, template, auth):
     request = Request(template + '?' + querystring)
     if auth is not None:
         request.add_header('Authorization', 'Basic '.encode('ascii') + auth)
+    log_info('Requesting {}?{}'.format(template, querystring))
     return request
 
 
@@ -506,15 +530,30 @@ def retrieve_repositories(args):
     repos = retrieve_data(args, template, single_request=single_request)
 
     if args.all_starred:
-        starred_template = 'https://{0}/user/starred'.format(
+        starred_template = 'https://{0}/user/starred'.format(get_github_api_host(args))
-            get_github_api_host(args))
         starred_repos = retrieve_data(args, starred_template, single_request=False)
-        # we need to be able to determine this repo was retrieved as a starred repo
+        # flag each repo as starred for downstream processing
-        # later, so add a flag to each item
         for item in starred_repos:
             item.update({'is_starred': True})
         repos.extend(starred_repos)
 
+    if args.include_gists:
+        gists_template = 'https://{0}/gists'.format(get_github_api_host(args))
+        gists = retrieve_data(args, gists_template, single_request=False)
+        # flag each repo as a gist for downstream processing
+        for item in gists:
+            item.update({'is_gist': True})
+        repos.extend(gists)
+
+    if args.include_starred_gists:
+        starred_gists_template = 'https://{0}/gists/starred'.format(get_github_api_host(args))
+        starred_gists = retrieve_data(args, starred_gists_template, single_request=False)
+        # flag each repo as a starred gist for downstream processing
+        for item in starred_gists:
+            item.update({'is_gist': True,
+                         'is_starred': True})
+        repos.extend(starred_gists)
+
     return repos
 
 
@@ -523,7 +562,8 @@ def filter_repositories(args, unfiltered_repositories):
 
     repositories = []
     for r in unfiltered_repositories:
-        if r['owner']['login'] == args.user or r.get('is_starred'):
+        # gists can be anonymous, so need to safely check owner
+        if r.get('owner', {}).get('login') == args.user or r.get('is_starred'):
             repositories.append(r)
 
     name_regex = None
@@ -535,11 +575,11 @@ def filter_repositories(args, unfiltered_repositories):
         languages = [x.lower() for x in args.languages]
 
     if not args.fork:
-        repositories = [r for r in repositories if not r['fork']]
+        repositories = [r for r in repositories if not r.get('fork')]
     if not args.private:
-        repositories = [r for r in repositories if not r['private']]
+        repositories = [r for r in repositories if not r.get('private') or r.get('public')]
     if languages:
-        repositories = [r for r in repositories if r['language'] and r['language'].lower() in languages]  # noqa
+        repositories = [r for r in repositories if r.get('language') and r.get('language').lower() in languages]  # noqa
     if name_regex:
         repositories = [r for r in repositories if name_regex.match(r['name'])]
 
@@ -561,27 +601,37 @@ def backup_repositories(args, output_directory, repositories):
         args.since = None
 
     for repository in repositories:
-        backup_cwd = os.path.join(output_directory, 'repositories')
+        if repository.get('is_gist'):
-        repo_cwd = os.path.join(backup_cwd, repository['name'])
+            repo_cwd = os.path.join(output_directory, 'gists', repository['id'])
-
+        elif repository.get('is_starred'):
-        # put starred repos in -o/starred/${owner}/${repo} to prevent collision of
+            # put starred repos in -o/starred/${owner}/${repo} to prevent collision of
-        # any repositories with the same name
+            # any repositories with the same name
-        if repository.get('is_starred'):
+            repo_cwd = os.path.join(output_directory, 'starred', repository['owner']['login'], repository['name'])
-            backup_cwd = os.path.join(output_directory, 'starred')
+        else:
-            repo_cwd = os.path.join(backup_cwd, repository['owner']['login'],
+            repo_cwd = os.path.join(output_directory, 'repositories', repository['name'])
-                                    repository['name'])
 
         repo_dir = os.path.join(repo_cwd, 'repository')
         repo_url = get_github_repo_url(args, repository)
 
-        if args.include_repository or args.include_everything:
+        include_gists = (args.include_gists or args.include_starred_gists)
-            fetch_repository(repository['name'],
+        if (args.include_repository or args.include_everything) \
+                or (include_gists and repository.get('is_gist')):
+            repo_name = repository.get('name') if not repository.get('is_gist') else repository.get('id')
+            fetch_repository(repo_name,
                              repo_url,
                              repo_dir,
                              skip_existing=args.skip_existing,
                              bare_clone=args.bare_clone,
                              lfs_clone=args.lfs_clone)
 
+            if repository.get('is_gist'):
+                # dump gist information to a file as well
+                output_file = '{0}/gist.json'.format(repo_cwd)
+                with codecs.open(output_file, 'w', encoding='utf-8') as f:
+                    json_dump(repository, f)
+
+                continue  # don't try to back anything else for a gist; it doesn't exist
+
         download_wiki = (args.include_wiki or args.include_everything)
         if repository['has_wiki'] and download_wiki:
             fetch_repository(repository['name'],
@@ -680,23 +730,35 @@ def backup_pulls(args, repo_cwd, repository, repos_template):
     pulls = {}
     _pulls_template = '{0}/{1}/pulls'.format(repos_template,
                                              repository['full_name'])
+    query_args = {
+        'filter': 'all',
+        'state': 'all',
+        'sort': 'updated',
+        'direction': 'desc',
+    }
 
-    pull_states = ['open', 'closed']
+    if not args.include_pull_details:
-    for pull_state in pull_states:
+        pull_states = ['open', 'closed']
-        query_args = {
+        for pull_state in pull_states:
-            'filter': 'all',
+            query_args['state'] = pull_state
-            'state': pull_state,
+            # It'd be nice to be able to apply the args.since filter here...
-            'sort': 'updated',
+            _pulls = retrieve_data(args,
-            'direction': 'desc',
+                                   _pulls_template,
-        }
+                                   query_args=query_args)
-
+            for pull in _pulls:
-        # It'd be nice to be able to apply the args.since filter here...
+                if not args.since or pull['updated_at'] >= args.since:
+                    pulls[pull['number']] = pull
+    else:
         _pulls = retrieve_data(args,
                                _pulls_template,
                                query_args=query_args)
         for pull in _pulls:
             if not args.since or pull['updated_at'] >= args.since:
-                pulls[pull['number']] = pull
+                pulls[pull['number']] = retrieve_data(
+                    args,
+                    _pulls_template + '/{}'.format(pull['number']),
+                    single_request=True
+                )
 
     log_info('Saving {0} pull requests to disk'.format(
         len(list(pulls.keys()))))
@@ -786,7 +848,7 @@ def fetch_repository(name,
             clone_exists = subprocess.check_output(['git',
                                                     'rev-parse',
                                                     '--is-bare-repository'],
-                                                   cwd=local_dir) == "true\n"
+                                                   cwd=local_dir) == b"true\n"
         else:
             clone_exists = False
     else:
@@ -849,25 +911,41 @@ def backup_account(args, output_directory):
     account_cwd = os.path.join(output_directory, 'account')
 
     if args.include_starred or args.include_everything:
-        output_file = '{0}/starred.json'.format(account_cwd)
+        output_file = "{0}/starred.json".format(account_cwd)
-        template = "https://{0}/users/{1}/starred"
+        template = "https://{0}/users/{1}/starred".format(get_github_api_host(args), args.user)
-        template = template.format(get_github_api_host(args), args.user)
         _backup_data(args,
-                     'starred repositories',
+                     "starred repositories",
                      template,
                      output_file,
                      account_cwd)
 
     if args.include_watched or args.include_everything:
-        output_file = '{0}/watched.json'.format(account_cwd)
+        output_file = "{0}/watched.json".format(account_cwd)
-        template = "https://{0}/users/{1}/subscriptions"
+        template = "https://{0}/users/{1}/subscriptions".format(get_github_api_host(args), args.user)
-        template = template.format(get_github_api_host(args), args.user)
         _backup_data(args,
-                     'watched repositories',
+                     "watched repositories",
                      template,
                      output_file,
                      account_cwd)
 
+    if args.include_followers or args.include_everything:
+        output_file = "{0}/followers.json".format(account_cwd)
+        template = "https://{0}/users/{1}/followers".format(get_github_api_host(args), args.user)
+        _backup_data(args,
+                    "followers",
+                    template,
+                    output_file,
+                    account_cwd)
+
+    if args.include_following or args.include_everything:
+        output_file = "{0}/following.json".format(account_cwd)
+        template = "https://{0}/users/{1}/following".format(get_github_api_host(args), args.user)
+        _backup_data(args,
+                    "following",
+                    template,
+                    output_file,
+                    account_cwd)
+
 
 def _backup_data(args, name, template, output_file, output_directory):
     skip_existing = args.skip_existing

github_backup/__init__.py

@@ -1 +1 @@
-__version__ = '0.16.0'
+__version__ = '0.19.1'