From 8b95f187ad0b07179d27219156ffdc6ae2251af9 Mon Sep 17 00:00:00 2001 From: "W. Harrison Wright" Date: Sat, 13 Jan 2018 14:08:36 -0600 Subject: [PATCH 1/8] Add ability to clone starred repos --- bin/github-backup | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/bin/github-backup b/bin/github-backup index 1f4c73e..d921fbe 100755 --- a/bin/github-backup +++ b/bin/github-backup @@ -343,7 +343,7 @@ def get_github_repo_url(args, repository): repo_url = 'https://{0}@{1}/{2}/{3}.git'.format( auth, get_github_host(args), - args.user, + repository['owner']['login'], repository['name']) else: repo_url = repository['clone_url'] @@ -499,7 +499,17 @@ def retrieve_repositories(args): args.user, args.repository) - return retrieve_data(args, template, single_request=single_request) + repos = retrieve_data(args, template, single_request=single_request) + + if args.include_starred or args.include_everything: + starred_template = 'https://{0}/user/starred'.format( + get_github_api_host(args)) + starred_repos = retrieve_data(args, starred_template, single_request=False) + for item in starred_repos: + item.update({'is_starred': True}) + repos.extend(starred_repos) + + return repos def filter_repositories(args, unfiltered_repositories): @@ -507,7 +517,7 @@ def filter_repositories(args, unfiltered_repositories): repositories = [] for r in unfiltered_repositories: - if r['owner']['login'] == args.user: + if r['owner']['login'] == args.user or is_starred_repo(args, r): repositories.append(r) name_regex = None @@ -547,6 +557,14 @@ def backup_repositories(args, output_directory, repositories): for repository in repositories: backup_cwd = os.path.join(output_directory, 'repositories') repo_cwd = os.path.join(backup_cwd, repository['name']) + + # put starred repos in -o/starred/${owner}/${repo} to prevent collision of + # any repositories with the same name + if is_starred_repo(args, repository): + backup_cwd = os.path.join(output_directory, 'starred') + repo_cwd = os.path.join(backup_cwd, repository['owner']['login'], + repository['name']) + repo_dir = os.path.join(repo_cwd, 'repository') repo_url = get_github_repo_url(args, repository) @@ -866,6 +884,10 @@ def json_dump(data, output_file): separators=(',', ': ')) +def is_starred_repo(args, repo): + return (args.include_starred or args.include_everything) and repo.get('is_starred') + + def main(): args = parse_args() From 6e3cbe841ae50e9927a9d4d203379794da8146a3 Mon Sep 17 00:00:00 2001 From: "W. Harrison Wright" Date: Sat, 13 Jan 2018 14:12:26 -0600 Subject: [PATCH 2/8] Add comment --- bin/github-backup | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/github-backup b/bin/github-backup index d921fbe..b382e10 100755 --- a/bin/github-backup +++ b/bin/github-backup @@ -505,6 +505,8 @@ def retrieve_repositories(args): starred_template = 'https://{0}/user/starred'.format( get_github_api_host(args)) starred_repos = retrieve_data(args, starred_template, single_request=False) + # we need to be able to determine this repo was retrieved as a starred repo + # later, so add a flag to each item for item in starred_repos: item.update({'is_starred': True}) repos.extend(starred_repos) From bd346de8987bfbdccc36f4269185720f0edcf299 Mon Sep 17 00:00:00 2001 From: "W. Harrison Wright" Date: Sat, 13 Jan 2018 17:43:00 -0600 Subject: [PATCH 3/8] Put starred clone repoistories under a new option --- bin/github-backup | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/bin/github-backup b/bin/github-backup index b382e10..2d39875 100755 --- a/bin/github-backup +++ b/bin/github-backup @@ -160,6 +160,10 @@ def parse_args(): action='store_true', dest='include_starred', help='include starred repositories in backup') + parser.add_argument('--clone-starred', + action='store_true', + dest='clone_starred', + help='clone starred repositories in backup') parser.add_argument('--watched', action='store_true', dest='include_watched', @@ -501,7 +505,7 @@ def retrieve_repositories(args): repos = retrieve_data(args, template, single_request=single_request) - if args.include_starred or args.include_everything: + if args.clone_starred: starred_template = 'https://{0}/user/starred'.format( get_github_api_host(args)) starred_repos = retrieve_data(args, starred_template, single_request=False) @@ -519,7 +523,7 @@ def filter_repositories(args, unfiltered_repositories): repositories = [] for r in unfiltered_repositories: - if r['owner']['login'] == args.user or is_starred_repo(args, r): + if r['owner']['login'] == args.user or r.get('is_starred'): repositories.append(r) name_regex = None @@ -562,7 +566,7 @@ def backup_repositories(args, output_directory, repositories): # put starred repos in -o/starred/${owner}/${repo} to prevent collision of # any repositories with the same name - if is_starred_repo(args, repository): + if repository.get('is_starred'): backup_cwd = os.path.join(output_directory, 'starred') repo_cwd = os.path.join(backup_cwd, repository['owner']['login'], repository['name']) @@ -886,10 +890,6 @@ def json_dump(data, output_file): separators=(',', ': ')) -def is_starred_repo(args, repo): - return (args.include_starred or args.include_everything) and repo.get('is_starred') - - def main(): args = parse_args() From cd2372183e728a666263307b1139bca208796c25 Mon Sep 17 00:00:00 2001 From: "W. Harrison Wright" Date: Sat, 13 Jan 2018 17:44:09 -0600 Subject: [PATCH 4/8] Update documentation --- README.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 4ae7d9c..af382f4 100644 --- a/README.rst +++ b/README.rst @@ -32,7 +32,7 @@ CLI Usage is as follows:: [--all] [--issues] [--issue-comments] [--issue-events] [--pulls] [--pull-comments] [--pull-commits] [--labels] [--hooks] [--milestones] [--repositories] [--bare] [--lfs] - [--wikis] [--skip-existing] + [--wikis] [--skip-existing] [--clone-starred] [-L [LANGUAGES [LANGUAGES ...]]] [-N NAME_REGEX] [-H GITHUB_HOST] [-O] [-R REPOSITORY] [-P] [-F] [--prefer-ssh] [-v] @@ -75,6 +75,7 @@ CLI Usage is as follows:: --lfs clone LFS repositories (requires Git LFS to be installed, https://git-lfs.github.com) --wikis include wiki clone in backup --skip-existing skip project if a backup directory exists + --clone-starred clone starred repositories -L [LANGUAGES [LANGUAGES ...]], --languages [LANGUAGES [LANGUAGES ...]] only allow these languages -N NAME_REGEX, --name-regex NAME_REGEX From 9a539b1d6b4bca2d6e3dd91dd960272b1e45ea11 Mon Sep 17 00:00:00 2001 From: "W. Harrison Wright" Date: Sun, 14 Jan 2018 10:18:51 -0600 Subject: [PATCH 5/8] JK don't update documentation --- README.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/README.rst b/README.rst index af382f4..1e61958 100644 --- a/README.rst +++ b/README.rst @@ -75,7 +75,6 @@ CLI Usage is as follows:: --lfs clone LFS repositories (requires Git LFS to be installed, https://git-lfs.github.com) --wikis include wiki clone in backup --skip-existing skip project if a backup directory exists - --clone-starred clone starred repositories -L [LANGUAGES [LANGUAGES ...]], --languages [LANGUAGES [LANGUAGES ...]] only allow these languages -N NAME_REGEX, --name-regex NAME_REGEX From 7cccd42ec9ea4a4d2bcf7d3da85397371a1194ce Mon Sep 17 00:00:00 2001 From: "W. Harrison Wright" Date: Sun, 14 Jan 2018 10:21:15 -0600 Subject: [PATCH 6/8] Change option to --all-starred --- README.rst | 2 +- bin/github-backup | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index 1e61958..4ae7d9c 100644 --- a/README.rst +++ b/README.rst @@ -32,7 +32,7 @@ CLI Usage is as follows:: [--all] [--issues] [--issue-comments] [--issue-events] [--pulls] [--pull-comments] [--pull-commits] [--labels] [--hooks] [--milestones] [--repositories] [--bare] [--lfs] - [--wikis] [--skip-existing] [--clone-starred] + [--wikis] [--skip-existing] [-L [LANGUAGES [LANGUAGES ...]]] [-N NAME_REGEX] [-H GITHUB_HOST] [-O] [-R REPOSITORY] [-P] [-F] [--prefer-ssh] [-v] diff --git a/bin/github-backup b/bin/github-backup index 2d39875..4d4b485 100755 --- a/bin/github-backup +++ b/bin/github-backup @@ -159,11 +159,11 @@ def parse_args(): parser.add_argument('--starred', action='store_true', dest='include_starred', - help='include starred repositories in backup') - parser.add_argument('--clone-starred', + help='include JSON output of starred repositories in backup') + parser.add_argument('--all-starred', action='store_true', - dest='clone_starred', - help='clone starred repositories in backup') + dest='all_starred', + help='include starred repositories in backup') parser.add_argument('--watched', action='store_true', dest='include_watched', @@ -505,7 +505,7 @@ def retrieve_repositories(args): repos = retrieve_data(args, template, single_request=single_request) - if args.clone_starred: + if args.all_starred: starred_template = 'https://{0}/user/starred'.format( get_github_api_host(args)) starred_repos = retrieve_data(args, starred_template, single_request=False) From c142707a903ed6722a3b2c200b6aacfc70cd0b49 Mon Sep 17 00:00:00 2001 From: "W. Harrison Wright" Date: Mon, 22 Jan 2018 11:34:27 -0600 Subject: [PATCH 7/8] Update documentation --- README.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 4ae7d9c..a45d4d7 100644 --- a/README.rst +++ b/README.rst @@ -32,7 +32,7 @@ CLI Usage is as follows:: [--all] [--issues] [--issue-comments] [--issue-events] [--pulls] [--pull-comments] [--pull-commits] [--labels] [--hooks] [--milestones] [--repositories] [--bare] [--lfs] - [--wikis] [--skip-existing] + [--wikis] [--skip-existing] [--all-starred] [-L [LANGUAGES [LANGUAGES ...]]] [-N NAME_REGEX] [-H GITHUB_HOST] [-O] [-R REPOSITORY] [-P] [-F] [--prefer-ssh] [-v] @@ -57,7 +57,7 @@ CLI Usage is as follows:: -o OUTPUT_DIRECTORY, --output-directory OUTPUT_DIRECTORY directory at which to backup the repositories -i, --incremental incremental backup - --starred include starred repositories in backup + --starred include list of starred repositories in backup --watched include watched repositories in backup --all include everything in backup --issues include issues in backup @@ -75,6 +75,7 @@ CLI Usage is as follows:: --lfs clone LFS repositories (requires Git LFS to be installed, https://git-lfs.github.com) --wikis include wiki clone in backup --skip-existing skip project if a backup directory exists + --all-starred include everything from starred repositories in backup -L [LANGUAGES [LANGUAGES ...]], --languages [LANGUAGES [LANGUAGES ...]] only allow these languages -N NAME_REGEX, --name-regex NAME_REGEX From cb054c26316857eba81341efe3d388a246d3ea6d Mon Sep 17 00:00:00 2001 From: Jose Diaz-Gonzalez Date: Mon, 22 Jan 2018 12:36:32 -0500 Subject: [PATCH 8/8] Update README.rst --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index a45d4d7..29ac8aa 100644 --- a/README.rst +++ b/README.rst @@ -57,7 +57,7 @@ CLI Usage is as follows:: -o OUTPUT_DIRECTORY, --output-directory OUTPUT_DIRECTORY directory at which to backup the repositories -i, --incremental incremental backup - --starred include list of starred repositories in backup + --starred include JSON output of starred repositories in backup --watched include watched repositories in backup --all include everything in backup --issues include issues in backup @@ -75,7 +75,7 @@ CLI Usage is as follows:: --lfs clone LFS repositories (requires Git LFS to be installed, https://git-lfs.github.com) --wikis include wiki clone in backup --skip-existing skip project if a backup directory exists - --all-starred include everything from starred repositories in backup + --all-starred include starred repositories in backup -L [LANGUAGES [LANGUAGES ...]], --languages [LANGUAGES [LANGUAGES ...]] only allow these languages -N NAME_REGEX, --name-regex NAME_REGEX