Release version 0.21.1

Mark options which are not included in --all

CHANGES.rst

@@ -1,6 +1,78 @@
 Changelog
 =========
 
+0.21.1 (2018-12-25)
+-------------------
+
+- Mark options which are not included in --all. [Bernd]
+
+  As discussed in Issue #100
+
+0.21.0 (2018-11-28)
+-------------------
+
+- Correctly download repos when user arg != authenticated user. [W.
+  Harrison Wright]
+
+0.20.1 (2018-09-29)
+-------------------
+
+- Clone the specified user's gists, not the authenticated user. [W.
+  Harrison Wright]
+
+- Clone the specified user's starred repos, not the authenticated user.
+  [W. Harrison Wright]
+
+0.20.0 (2018-03-24)
+-------------------
+
+- Chore: drop Python 2.6. [Jose Diaz-Gonzalez]
+
+- Feat: simplify release script. [Jose Diaz-Gonzalez]
+
+0.19.2 (2018-03-24)
+-------------------
+
+Fix
+~~~
+
+- Cleanup pep8 violations. [Jose Diaz-Gonzalez]
+
+0.19.0 (2018-03-24)
+-------------------
+
+- Add additional output for the current request. [Robin Gloster]
+
+  This is useful to have some progress indication for huge repositories.
+
+
+- Add option to backup additional PR details. [Robin Gloster]
+
+  Some payload is only included when requesting a single pull request
+
+
+- Mark string as binary in comparison for skip_existing. [Johannes
+  Bornhold]
+
+  Found out that the flag "--skip-existing" did not work out as expected on Python
+  3.6. Tracked it down to the comparison which has to be against a string of bytes
+  in Python3.
+
+
+0.18.0 (2018-02-22)
+-------------------
+
+- Add option to fetch followers/following JSON data. [Stephen Greene]
+
+0.17.0 (2018-02-20)
+-------------------
+
+- Short circuit gists backup process. [W. Harrison Wright]
+
+- Formatting. [W. Harrison Wright]
+
+- Add ability to backup gists. [W. Harrison Wright]
+
 0.16.0 (2018-01-22)
 -------------------
 

README.rst

@@ -28,11 +28,12 @@ Usage
 CLI Usage is as follows::
 
     github-backup [-h] [-u USERNAME] [-p PASSWORD] [-t TOKEN]
-                  [-o OUTPUT_DIRECTORY] [-i] [--starred] [--watched]
+                  [-o OUTPUT_DIRECTORY] [-i] [--starred] [--all-starred]
-                  [--all] [--issues] [--issue-comments] [--issue-events]
+                  [--watched] [--followers] [--following] [--all]
-                  [--pulls] [--pull-comments] [--pull-commits] [--labels]
+                  [--issues] [--issue-comments] [--issue-events] [--pulls]
-                  [--hooks] [--milestones] [--repositories] [--bare] [--lfs]
+                  [--pull-comments] [--pull-commits] [--labels] [--hooks]
-                  [--wikis] [--skip-existing] [--all-starred]
+                  [--milestones] [--repositories] [--bare] [--lfs]
+                  [--wikis] [--gists] [--starred-gists] [--skip-existing]
                   [-L [LANGUAGES [LANGUAGES ...]]] [-N NAME_REGEX]
                   [-H GITHUB_HOST] [-O] [-R REPOSITORY] [-P] [-F]
                   [--prefer-ssh] [-v]
@@ -53,12 +54,16 @@ CLI Usage is as follows::
                             password for basic auth. If a username is given but
                             not a password, the password will be prompted for.
       -t TOKEN, --token TOKEN
-                            personal access or OAuth token
+                            personal access or OAuth token, or path to token
+                            (file://...)
       -o OUTPUT_DIRECTORY, --output-directory OUTPUT_DIRECTORY
                             directory at which to backup the repositories
       -i, --incremental     incremental backup
       --starred             include JSON output of starred repositories in backup
+      --all-starred         include starred repositories in backup
       --watched             include watched repositories in backup
+      --followers           include JSON output of followers in backup
+      --following           include JSON output of following users in backup
       --all                 include everything in backup
       --issues              include issues in backup
       --issue-comments      include issue comments in backup
@@ -72,10 +77,12 @@ CLI Usage is as follows::
       --milestones          include milestones in backup
       --repositories        include repository clone in backup
       --bare                clone bare repositories
-      --lfs                 clone LFS repositories (requires Git LFS to be installed, https://git-lfs.github.com)
+      --lfs                 clone LFS repositories (requires Git LFS to be
+                            installed, https://git-lfs.github.com)
       --wikis               include wiki clone in backup
+      --gists               include gists in backup
+      --starred-gists       include starred gists in backup
       --skip-existing       skip project if a backup directory exists
-      --all-starred         include starred repositories in backup
       -L [LANGUAGES [LANGUAGES ...]], --languages [LANGUAGES [LANGUAGES ...]]
                             only allow these languages
       -N NAME_REGEX, --name-regex NAME_REGEX

bin/github-backup

@@ -163,15 +163,23 @@ def parse_args():
     parser.add_argument('--all-starred',
                         action='store_true',
                         dest='all_starred',
-                        help='include starred repositories in backup')
+                        help='include starred repositories in backup [*]')
     parser.add_argument('--watched',
                         action='store_true',
                         dest='include_watched',
-                        help='include watched repositories in backup')
+                        help='include JSON output of watched repositories in backup')
+    parser.add_argument('--followers',
+                        action='store_true',
+                        dest='include_followers',
+                        help='include JSON output of followers in backup')
+    parser.add_argument('--following',
+                        action='store_true',
+                        dest='include_following',
+                        help='include JSON output of following users in backup')
     parser.add_argument('--all',
                         action='store_true',
                         dest='include_everything',
-                        help='include everything in backup')
+                        help='include everything in backup (not including [*])')
     parser.add_argument('--issues',
                         action='store_true',
                         dest='include_issues',
@@ -196,6 +204,10 @@ def parse_args():
                         action='store_true',
                         dest='include_pull_commits',
                         help='include pull request commits in backup')
+    parser.add_argument('--pull-details',
+                        action='store_true',
+                        dest='include_pull_details',
+                        help='include more pull request details in backup [*]')
     parser.add_argument('--labels',
                         action='store_true',
                         dest='include_labels',
@@ -219,11 +231,19 @@ def parse_args():
     parser.add_argument('--lfs',
                         action='store_true',
                         dest='lfs_clone',
-                        help='clone LFS repositories (requires Git LFS to be installed, https://git-lfs.github.com)')
+                        help='clone LFS repositories (requires Git LFS to be installed, https://git-lfs.github.com) [*]')
     parser.add_argument('--wikis',
                         action='store_true',
                         dest='include_wiki',
                         help='include wiki clone in backup')
+    parser.add_argument('--gists',
+                        action='store_true',
+                        dest='include_gists',
+                        help='include gists in backup [*]')
+    parser.add_argument('--starred-gists',
+                        action='store_true',
+                        dest='include_starred_gists',
+                        help='include starred gists in backup [*]')
     parser.add_argument('--skip-existing',
                         action='store_true',
                         dest='skip_existing',
@@ -253,11 +273,11 @@ def parse_args():
     parser.add_argument('-P', '--private',
                         action='store_true',
                         dest='private',
-                        help='include private repositories')
+                        help='include private repositories [*]')
     parser.add_argument('-F', '--fork',
                         action='store_true',
                         dest='fork',
-                        help='include forked repositories')
+                        help='include forked repositories [*]')
     parser.add_argument('--prefer-ssh',
                         action='store_true',
                         help='Clone repositories using SSH instead of HTTPS')
@@ -283,12 +303,12 @@ def get_auth(args, encode=True):
             if platform.system() != 'Darwin':
                 log_error("Keychain arguments are only supported on Mac OSX")
             try:
-                with open(os.devnull,'w') as devnull:
+                with open(os.devnull, 'w') as devnull:
                     token = (subprocess.check_output([
-                        'security','find-generic-password',
+                        'security', 'find-generic-password',
-                        '-s',args.osx_keychain_item_name,
+                        '-s', args.osx_keychain_item_name,
-                        '-a',args.osx_keychain_item_account,
+                        '-a', args.osx_keychain_item_account,
-                        '-w' ], stderr=devnull).strip())
+                        '-w'], stderr=devnull).strip())
                 auth = token + ':' + 'x-oauth-basic'
             except:
                 log_error('No password item matching the provided name and account could be found in the osx keychain.')
@@ -342,6 +362,9 @@ def get_github_repo_url(args, repository):
     if args.prefer_ssh:
         return repository['ssh_url']
 
+    if repository.get('is_gist'):
+        return repository['git_pull_url']
+
     auth = get_auth(args, False)
     if auth:
         repo_url = 'https://{0}@{1}/{2}/{3}.git'.format(
@@ -431,6 +454,7 @@ def _construct_request(per_page, page, query_args, template, auth):
     request = Request(template + '?' + querystring)
     if auth is not None:
         request.add_header('Authorization', 'Basic '.encode('ascii') + auth)
+    log_info('Requesting {}?{}'.format(template, querystring))
     return request
 
 
@@ -479,6 +503,12 @@ def _request_url_error(template, retry_timeout):
     return False
 
 
+def get_authenticated_user(args):
+    template = 'https://{0}/user'.format(get_github_api_host(args))
+    data = retrieve_data(args, template, single_request=True)
+    return data[0]
+
+
 def check_git_lfs_install():
     exit_code = subprocess.call(['git', 'lfs', 'version'])
     if exit_code != 0:
@@ -486,11 +516,20 @@ def check_git_lfs_install():
         sys.exit(1)
 
 
-def retrieve_repositories(args):
+def retrieve_repositories(args, authenticated_user):
     log_info('Retrieving repositories')
     single_request = False
-    template = 'https://{0}/user/repos'.format(
+    if args.user == authenticated_user['login']:
-        get_github_api_host(args))
+        # we must use the /user/repos API to be able to access private repos
+        template = 'https://{0}/user/repos'.format(
+            get_github_api_host(args))
+    else:
+        if args.private:
+            log_error('Authenticated user is different from user being backed up, thus private repositories cannot be accessed')
+        template = 'https://{0}/users/{1}/repos'.format(
+            get_github_api_host(args),
+            args.user)
+
     if args.organization:
         template = 'https://{0}/orgs/{1}/repos'.format(
             get_github_api_host(args),
@@ -506,15 +545,30 @@ def retrieve_repositories(args):
     repos = retrieve_data(args, template, single_request=single_request)
 
     if args.all_starred:
-        starred_template = 'https://{0}/user/starred'.format(
+        starred_template = 'https://{0}/users/{1}/starred'.format(get_github_api_host(args), args.user)
-            get_github_api_host(args))
         starred_repos = retrieve_data(args, starred_template, single_request=False)
-        # we need to be able to determine this repo was retrieved as a starred repo
+        # flag each repo as starred for downstream processing
-        # later, so add a flag to each item
         for item in starred_repos:
             item.update({'is_starred': True})
         repos.extend(starred_repos)
 
+    if args.include_gists:
+        gists_template = 'https://{0}/users/{1}/gists'.format(get_github_api_host(args), args.user)
+        gists = retrieve_data(args, gists_template, single_request=False)
+        # flag each repo as a gist for downstream processing
+        for item in gists:
+            item.update({'is_gist': True})
+        repos.extend(gists)
+
+    if args.include_starred_gists:
+        starred_gists_template = 'https://{0}/gists/starred'.format(get_github_api_host(args))
+        starred_gists = retrieve_data(args, starred_gists_template, single_request=False)
+        # flag each repo as a starred gist for downstream processing
+        for item in starred_gists:
+            item.update({'is_gist': True,
+                         'is_starred': True})
+        repos.extend(starred_gists)
+
     return repos
 
 
@@ -523,7 +577,8 @@ def filter_repositories(args, unfiltered_repositories):
 
     repositories = []
     for r in unfiltered_repositories:
-        if r['owner']['login'] == args.user or r.get('is_starred'):
+        # gists can be anonymous, so need to safely check owner
+        if r.get('owner', {}).get('login') == args.user or r.get('is_starred'):
             repositories.append(r)
 
     name_regex = None
@@ -535,11 +590,11 @@ def filter_repositories(args, unfiltered_repositories):
         languages = [x.lower() for x in args.languages]
 
     if not args.fork:
-        repositories = [r for r in repositories if not r['fork']]
+        repositories = [r for r in repositories if not r.get('fork')]
     if not args.private:
-        repositories = [r for r in repositories if not r['private']]
+        repositories = [r for r in repositories if not r.get('private') or r.get('public')]
     if languages:
-        repositories = [r for r in repositories if r['language'] and r['language'].lower() in languages]  # noqa
+        repositories = [r for r in repositories if r.get('language') and r.get('language').lower() in languages]  # noqa
     if name_regex:
         repositories = [r for r in repositories if name_regex.match(r['name'])]
 
@@ -561,27 +616,37 @@ def backup_repositories(args, output_directory, repositories):
         args.since = None
 
     for repository in repositories:
-        backup_cwd = os.path.join(output_directory, 'repositories')
+        if repository.get('is_gist'):
-        repo_cwd = os.path.join(backup_cwd, repository['name'])
+            repo_cwd = os.path.join(output_directory, 'gists', repository['id'])
-
+        elif repository.get('is_starred'):
-        # put starred repos in -o/starred/${owner}/${repo} to prevent collision of
+            # put starred repos in -o/starred/${owner}/${repo} to prevent collision of
-        # any repositories with the same name
+            # any repositories with the same name
-        if repository.get('is_starred'):
+            repo_cwd = os.path.join(output_directory, 'starred', repository['owner']['login'], repository['name'])
-            backup_cwd = os.path.join(output_directory, 'starred')
+        else:
-            repo_cwd = os.path.join(backup_cwd, repository['owner']['login'],
+            repo_cwd = os.path.join(output_directory, 'repositories', repository['name'])
-                                    repository['name'])
 
         repo_dir = os.path.join(repo_cwd, 'repository')
         repo_url = get_github_repo_url(args, repository)
 
-        if args.include_repository or args.include_everything:
+        include_gists = (args.include_gists or args.include_starred_gists)
-            fetch_repository(repository['name'],
+        if (args.include_repository or args.include_everything) \
+                or (include_gists and repository.get('is_gist')):
+            repo_name = repository.get('name') if not repository.get('is_gist') else repository.get('id')
+            fetch_repository(repo_name,
                              repo_url,
                              repo_dir,
                              skip_existing=args.skip_existing,
                              bare_clone=args.bare_clone,
                              lfs_clone=args.lfs_clone)
 
+            if repository.get('is_gist'):
+                # dump gist information to a file as well
+                output_file = '{0}/gist.json'.format(repo_cwd)
+                with codecs.open(output_file, 'w', encoding='utf-8') as f:
+                    json_dump(repository, f)
+
+                continue  # don't try to back anything else for a gist; it doesn't exist
+
         download_wiki = (args.include_wiki or args.include_everything)
         if repository['has_wiki'] and download_wiki:
             fetch_repository(repository['name'],
@@ -680,23 +745,35 @@ def backup_pulls(args, repo_cwd, repository, repos_template):
     pulls = {}
     _pulls_template = '{0}/{1}/pulls'.format(repos_template,
                                              repository['full_name'])
+    query_args = {
+        'filter': 'all',
+        'state': 'all',
+        'sort': 'updated',
+        'direction': 'desc',
+    }
 
-    pull_states = ['open', 'closed']
+    if not args.include_pull_details:
-    for pull_state in pull_states:
+        pull_states = ['open', 'closed']
-        query_args = {
+        for pull_state in pull_states:
-            'filter': 'all',
+            query_args['state'] = pull_state
-            'state': pull_state,
+            # It'd be nice to be able to apply the args.since filter here...
-            'sort': 'updated',
+            _pulls = retrieve_data(args,
-            'direction': 'desc',
+                                   _pulls_template,
-        }
+                                   query_args=query_args)
-
+            for pull in _pulls:
-        # It'd be nice to be able to apply the args.since filter here...
+                if not args.since or pull['updated_at'] >= args.since:
+                    pulls[pull['number']] = pull
+    else:
         _pulls = retrieve_data(args,
                                _pulls_template,
                                query_args=query_args)
         for pull in _pulls:
             if not args.since or pull['updated_at'] >= args.since:
-                pulls[pull['number']] = pull
+                pulls[pull['number']] = retrieve_data(
+                    args,
+                    _pulls_template + '/{}'.format(pull['number']),
+                    single_request=True
+                )
 
     log_info('Saving {0} pull requests to disk'.format(
         len(list(pulls.keys()))))
@@ -786,7 +863,7 @@ def fetch_repository(name,
             clone_exists = subprocess.check_output(['git',
                                                     'rev-parse',
                                                     '--is-bare-repository'],
-                                                   cwd=local_dir) == "true\n"
+                                                   cwd=local_dir) == b"true\n"
         else:
             clone_exists = False
     else:
@@ -849,21 +926,37 @@ def backup_account(args, output_directory):
     account_cwd = os.path.join(output_directory, 'account')
 
     if args.include_starred or args.include_everything:
-        output_file = '{0}/starred.json'.format(account_cwd)
+        output_file = "{0}/starred.json".format(account_cwd)
-        template = "https://{0}/users/{1}/starred"
+        template = "https://{0}/users/{1}/starred".format(get_github_api_host(args), args.user)
-        template = template.format(get_github_api_host(args), args.user)
         _backup_data(args,
-                     'starred repositories',
+                     "starred repositories",
                      template,
                      output_file,
                      account_cwd)
 
     if args.include_watched or args.include_everything:
-        output_file = '{0}/watched.json'.format(account_cwd)
+        output_file = "{0}/watched.json".format(account_cwd)
-        template = "https://{0}/users/{1}/subscriptions"
+        template = "https://{0}/users/{1}/subscriptions".format(get_github_api_host(args), args.user)
-        template = template.format(get_github_api_host(args), args.user)
         _backup_data(args,
-                     'watched repositories',
+                     "watched repositories",
+                     template,
+                     output_file,
+                     account_cwd)
+
+    if args.include_followers or args.include_everything:
+        output_file = "{0}/followers.json".format(account_cwd)
+        template = "https://{0}/users/{1}/followers".format(get_github_api_host(args), args.user)
+        _backup_data(args,
+                     "followers",
+                     template,
+                     output_file,
+                     account_cwd)
+
+    if args.include_following or args.include_everything:
+        output_file = "{0}/following.json".format(account_cwd)
+        template = "https://{0}/users/{1}/following".format(get_github_api_host(args), args.user)
+        _backup_data(args,
+                     "following",
                      template,
                      output_file,
                      account_cwd)
@@ -903,7 +996,8 @@ def main():
 
     log_info('Backing up user {0} to {1}'.format(args.user, output_directory))
 
-    repositories = retrieve_repositories(args)
+    authenticated_user = get_authenticated_user(args)
+    repositories = retrieve_repositories(args, authenticated_user)
     repositories = filter_repositories(args, repositories)
     backup_repositories(args, output_directory, repositories)
     backup_account(args, output_directory)

github_backup/__init__.py

@@ -1 +1 @@
-__version__ = '0.16.0'
+__version__ = '0.21.1'

release

@@ -1,8 +1,13 @@
 #!/usr/bin/env bash
 set -eo pipefail; [[ $RELEASE_TRACE ]] && set -x
 
-PACKAGE_NAME='github-backup'
+if [[ ! -f setup.py ]]; then
-INIT_PACKAGE_NAME='github_backup'
+  echo -e "${RED}WARNING: Missing setup.py${COLOR_OFF}\n"
+  exit 1
+fi
+
+PACKAGE_NAME="$(cat setup.py | grep "name='" | head | cut -d "'" -f2)"
+INIT_PACKAGE_NAME="$(echo "${PACKAGE_NAME//-/_}")"
 PUBLIC="true"
 
 # Colors

setup.py

@@ -37,7 +37,6 @@ setup(
         'Development Status :: 5 - Production/Stable',
         'Topic :: System :: Archiving :: Backup',
         'License :: OSI Approved :: MIT License',
-        'Programming Language :: Python :: 2.6',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',