Release version 0.17.0

Add ability to back up gists

CHANGES.rst

@@ -1,7 +1,30 @@
 Changelog
 =========
 
-0.15.0 (2017-12-11)
+0.17.0 (2018-02-20)
+-------------------
+
+- Short circuit gists backup process. [W. Harrison Wright]
+
+- Formatting. [W. Harrison Wright]
+
+- Add ability to backup gists. [W. Harrison Wright]
+
+0.16.0 (2018-01-22)
+-------------------
+
+- Change option to --all-starred. [W. Harrison Wright]
+
+- JK don't update documentation. [W. Harrison Wright]
+
+- Put starred clone repoistories under a new option. [W. Harrison
+  Wright]
+
+- Add comment. [W. Harrison Wright]
+
+- Add ability to clone starred repos. [W. Harrison Wright]
+
+0.14.1 (2017-10-11)
 -------------------
 
 - Fix arg not defined error. [Edward Pfremmer]

README.rst

@@ -32,7 +32,8 @@ CLI Usage is as follows::
                   [--all] [--issues] [--issue-comments] [--issue-events]
                   [--pulls] [--pull-comments] [--pull-commits] [--labels]
                   [--hooks] [--milestones] [--repositories] [--bare] [--lfs]
-                  [--wikis] [--skip-existing]
+                  [--wikis] [--skip-existing] [--all-starred]
+                  [--gists] [--starred-gists]
                   [-L [LANGUAGES [LANGUAGES ...]]] [-N NAME_REGEX]
                   [-H GITHUB_HOST] [-O] [-R REPOSITORY] [-P] [-F]
                   [--prefer-ssh] [-v]
@@ -57,7 +58,7 @@ CLI Usage is as follows::
       -o OUTPUT_DIRECTORY, --output-directory OUTPUT_DIRECTORY
                             directory at which to backup the repositories
       -i, --incremental     incremental backup
-      --starred             include starred repositories in backup
+      --starred             include JSON output of starred repositories in backup
       --watched             include watched repositories in backup
       --all                 include everything in backup
       --issues              include issues in backup
@@ -75,6 +76,9 @@ CLI Usage is as follows::
       --lfs                 clone LFS repositories (requires Git LFS to be installed, https://git-lfs.github.com)
       --wikis               include wiki clone in backup
       --skip-existing       skip project if a backup directory exists
+      --all-starred         include starred repositories in backup
+      --gists               include gists in backup
+      --starred-gists       include starred gists in backup
       -L [LANGUAGES [LANGUAGES ...]], --languages [LANGUAGES [LANGUAGES ...]]
                             only allow these languages
       -N NAME_REGEX, --name-regex NAME_REGEX

bin/github-backup

@@ -159,6 +159,10 @@ def parse_args():
     parser.add_argument('--starred',
                         action='store_true',
                         dest='include_starred',
+                        help='include JSON output of starred repositories in backup')
+    parser.add_argument('--all-starred',
+                        action='store_true',
+                        dest='all_starred',
                         help='include starred repositories in backup')
     parser.add_argument('--watched',
                         action='store_true',
@@ -220,6 +224,14 @@ def parse_args():
                         action='store_true',
                         dest='include_wiki',
                         help='include wiki clone in backup')
+    parser.add_argument('--gists',
+                        action='store_true',
+                        dest='include_gists',
+                        help='include gists in backup')
+    parser.add_argument('--starred-gists',
+                        action='store_true',
+                        dest='include_starred_gists',
+                        help='include starred gists in backup')
     parser.add_argument('--skip-existing',
                         action='store_true',
                         dest='skip_existing',
@@ -338,12 +350,15 @@ def get_github_repo_url(args, repository):
     if args.prefer_ssh:
         return repository['ssh_url']
 
+    if repository.get('is_gist'):
+        return repository['git_pull_url']
+
     auth = get_auth(args, False)
     if auth:
         repo_url = 'https://{0}@{1}/{2}/{3}.git'.format(
             auth,
             get_github_host(args),
-            args.user,
+            repository['owner']['login'],
             repository['name'])
     else:
         repo_url = repository['clone_url']
@@ -499,7 +514,34 @@ def retrieve_repositories(args):
             args.user,
             args.repository)
 
-    return retrieve_data(args, template, single_request=single_request)
+    repos = retrieve_data(args, template, single_request=single_request)
+
+    if args.all_starred:
+        starred_template = 'https://{0}/user/starred'.format(get_github_api_host(args))
+        starred_repos = retrieve_data(args, starred_template, single_request=False)
+        # flag each repo as starred for downstream processing
+        for item in starred_repos:
+            item.update({'is_starred': True})
+        repos.extend(starred_repos)
+
+    if args.include_gists:
+        gists_template = 'https://{0}/gists'.format(get_github_api_host(args))
+        gists = retrieve_data(args, gists_template, single_request=False)
+        # flag each repo as a gist for downstream processing
+        for item in gists:
+            item.update({'is_gist': True})
+        repos.extend(gists)
+
+    if args.include_starred_gists:
+        starred_gists_template = 'https://{0}/gists/starred'.format(get_github_api_host(args))
+        starred_gists = retrieve_data(args, starred_gists_template, single_request=False)
+        # flag each repo as a starred gist for downstream processing
+        for item in starred_gists:
+            item.update({'is_gist': True,
+                         'is_starred': True})
+        repos.extend(starred_gists)
+
+    return repos
 
 
 def filter_repositories(args, unfiltered_repositories):
@@ -507,7 +549,8 @@ def filter_repositories(args, unfiltered_repositories):
 
     repositories = []
     for r in unfiltered_repositories:
-        if r['owner']['login'] == args.user:
+        # gists can be anonymous, so need to safely check owner
+        if r.get('owner', {}).get('login') == args.user or r.get('is_starred'):
             repositories.append(r)
 
     name_regex = None
@@ -519,11 +562,11 @@ def filter_repositories(args, unfiltered_repositories):
         languages = [x.lower() for x in args.languages]
 
     if not args.fork:
-        repositories = [r for r in repositories if not r['fork']]
+        repositories = [r for r in repositories if not r.get('fork')]
     if not args.private:
-        repositories = [r for r in repositories if not r['private']]
+        repositories = [r for r in repositories if not r.get('private') or r.get('public')]
     if languages:
-        repositories = [r for r in repositories if r['language'] and r['language'].lower() in languages]  # noqa
+        repositories = [r for r in repositories if r.get('language') and r.get('language').lower() in languages]  # noqa
     if name_regex:
         repositories = [r for r in repositories if name_regex.match(r['name'])]
 
@@ -545,19 +588,37 @@ def backup_repositories(args, output_directory, repositories):
         args.since = None
 
     for repository in repositories:
-        backup_cwd = os.path.join(output_directory, 'repositories')
+        if repository.get('is_gist'):
-        repo_cwd = os.path.join(backup_cwd, repository['name'])
+            repo_cwd = os.path.join(output_directory, 'gists', repository['id'])
+        elif repository.get('is_starred'):
+            # put starred repos in -o/starred/${owner}/${repo} to prevent collision of
+            # any repositories with the same name
+            repo_cwd = os.path.join(output_directory, 'starred', repository['owner']['login'], repository['name'])
+        else:
+            repo_cwd = os.path.join(output_directory, 'repositories', repository['name'])
+
         repo_dir = os.path.join(repo_cwd, 'repository')
         repo_url = get_github_repo_url(args, repository)
 
-        if args.include_repository or args.include_everything:
+        include_gists = (args.include_gists or args.include_starred_gists)
-            fetch_repository(repository['name'],
+        if (args.include_repository or args.include_everything) \
+                or (include_gists and repository.get('is_gist')):
+            repo_name = repository.get('name') if not repository.get('is_gist') else repository.get('id')
+            fetch_repository(repo_name,
                              repo_url,
                              repo_dir,
                              skip_existing=args.skip_existing,
                              bare_clone=args.bare_clone,
                              lfs_clone=args.lfs_clone)
 
+            if repository.get('is_gist'):
+                # dump gist information to a file as well
+                output_file = '{0}/gist.json'.format(repo_cwd)
+                with codecs.open(output_file, 'w', encoding='utf-8') as f:
+                    json_dump(repository, f)
+
+                continue  # don't try to back anything else for a gist; it doesn't exist
+
         download_wiki = (args.include_wiki or args.include_everything)
         if repository['has_wiki'] and download_wiki:
             fetch_repository(repository['name'],

github_backup/__init__.py

@@ -1 +1 @@
-__version__ = '0.15.0'
+__version__ = '0.17.0'