Merge pull request #78 from whwright/clone-starred-repos

Clone starred repos

README.rst

@@ -32,7 +32,7 @@ CLI Usage is as follows::
                   [--all] [--issues] [--issue-comments] [--issue-events]
                   [--pulls] [--pull-comments] [--pull-commits] [--labels]
                   [--hooks] [--milestones] [--repositories] [--bare] [--lfs]
-                  [--wikis] [--skip-existing]
+                  [--wikis] [--skip-existing] [--all-starred]
                   [-L [LANGUAGES [LANGUAGES ...]]] [-N NAME_REGEX]
                   [-H GITHUB_HOST] [-O] [-R REPOSITORY] [-P] [-F]
                   [--prefer-ssh] [-v]
@@ -57,7 +57,7 @@ CLI Usage is as follows::
       -o OUTPUT_DIRECTORY, --output-directory OUTPUT_DIRECTORY
                             directory at which to backup the repositories
       -i, --incremental     incremental backup
-      --starred             include starred repositories in backup
+      --starred             include JSON output of starred repositories in backup
       --watched             include watched repositories in backup
       --all                 include everything in backup
       --issues              include issues in backup
@@ -75,6 +75,7 @@ CLI Usage is as follows::
       --lfs                 clone LFS repositories (requires Git LFS to be installed, https://git-lfs.github.com)
       --wikis               include wiki clone in backup
       --skip-existing       skip project if a backup directory exists
+      --all-starred         include starred repositories in backup
       -L [LANGUAGES [LANGUAGES ...]], --languages [LANGUAGES [LANGUAGES ...]]
                             only allow these languages
       -N NAME_REGEX, --name-regex NAME_REGEX

bin/github-backup

@@ -159,6 +159,10 @@ def parse_args():
     parser.add_argument('--starred',
                         action='store_true',
                         dest='include_starred',
+                        help='include JSON output of starred repositories in backup')
+    parser.add_argument('--all-starred',
+                        action='store_true',
+                        dest='all_starred',
                         help='include starred repositories in backup')
     parser.add_argument('--watched',
                         action='store_true',
@@ -343,7 +347,7 @@ def get_github_repo_url(args, repository):
         repo_url = 'https://{0}@{1}/{2}/{3}.git'.format(
             auth,
             get_github_host(args),
-            args.user,
+            repository['owner']['login'],
             repository['name'])
     else:
         repo_url = repository['clone_url']
@@ -499,7 +503,19 @@ def retrieve_repositories(args):
             args.user,
             args.repository)
 
-    return retrieve_data(args, template, single_request=single_request)
+    repos = retrieve_data(args, template, single_request=single_request)
+
+    if args.all_starred:
+        starred_template = 'https://{0}/user/starred'.format(
+            get_github_api_host(args))
+        starred_repos = retrieve_data(args, starred_template, single_request=False)
+        # we need to be able to determine this repo was retrieved as a starred repo
+        # later, so add a flag to each item
+        for item in starred_repos:
+            item.update({'is_starred': True})
+        repos.extend(starred_repos)
+
+    return repos
 
 
 def filter_repositories(args, unfiltered_repositories):
@@ -507,7 +523,7 @@ def filter_repositories(args, unfiltered_repositories):
 
     repositories = []
     for r in unfiltered_repositories:
-        if r['owner']['login'] == args.user:
+        if r['owner']['login'] == args.user or r.get('is_starred'):
             repositories.append(r)
 
     name_regex = None
@@ -547,6 +563,14 @@ def backup_repositories(args, output_directory, repositories):
     for repository in repositories:
         backup_cwd = os.path.join(output_directory, 'repositories')
         repo_cwd = os.path.join(backup_cwd, repository['name'])
+
+        # put starred repos in -o/starred/${owner}/${repo} to prevent collision of
+        # any repositories with the same name
+        if repository.get('is_starred'):
+            backup_cwd = os.path.join(output_directory, 'starred')
+            repo_cwd = os.path.join(backup_cwd, repository['owner']['login'],
+                                    repository['name'])
+
         repo_dir = os.path.join(repo_cwd, 'repository')
         repo_url = get_github_repo_url(args, repository)