From 040516325aec1a5dc179041710eb5777b0a24952 Mon Sep 17 00:00:00 2001 From: Chris Adams Date: Fri, 13 Mar 2015 15:39:35 -0400 Subject: [PATCH 1/3] Switch to using ssh_url The previous commit used the wrong URL for a private repo. This was masked by the lack of error loging in logging_subprocess (which will be in a separate branch) --- bin/github-backup | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bin/github-backup b/bin/github-backup index e740f51..f566ef5 100755 --- a/bin/github-backup +++ b/bin/github-backup @@ -249,9 +249,7 @@ def backup_repositories(args, output_directory, repositories): repo_url = repository['clone_url'] if args.prefer_ssh: - git_url = repository.get('git_url') - if git_url: - repo_url = git_url + repo_url = repository['ssh_url'] git_command = ['git', 'clone', repo_url, 'repository'] logging_subprocess(git_command, logger=None, cwd=repo_cwd) From c81bf986274689b9075f2e07716b46f606f0661f Mon Sep 17 00:00:00 2001 From: Chris Adams Date: Fri, 13 Mar 2015 15:45:15 -0400 Subject: [PATCH 2/3] logging_subprocess: always log when a command fails Previously git clones could fail without any indication unless you edited the source to change `logger=None` to use a configured logger. Now a non-zero return code will always output a message to stderr and will display the executed command so it can be rerun for troubleshooting. --- bin/github-backup | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/bin/github-backup b/bin/github-backup index f566ef5..ab293a6 100755 --- a/bin/github-backup +++ b/bin/github-backup @@ -1,5 +1,7 @@ #!/usr/bin/env python +from __future__ import print_function + import argparse import base64 import errno @@ -61,7 +63,13 @@ def logging_subprocess(popenargs, logger, stdout_log_level=logging.DEBUG, stderr check_io() # check again to catch anything after the process exits - return child.wait() + rc = child.wait() + + if rc != 0: + print(u'{} returned {}:'.format(popenargs[0], rc), file=sys.stderr) + print('\t', u' '.join(popenargs), file=sys.stderr) + + return rc def mkdir_p(*args): From 5612e51153a6a33f66f4cea3f3d3f3a1c77f0541 Mon Sep 17 00:00:00 2001 From: Chris Adams Date: Fri, 13 Mar 2015 15:49:49 -0400 Subject: [PATCH 3/3] Update repository back up handling for wikis MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Now wikis will follow the same logic as the main repo checkout for --prefer-ssh. * The regular repository and wiki paths both use the same function to handle either cloning or updating a local copy of the remote repo * All git updates will now use “git fetch --all --tags” to ensure that tags and branches other than master will also be backed up --- bin/github-backup | 60 ++++++++++++++++++++++------------------------- 1 file changed, 28 insertions(+), 32 deletions(-) diff --git a/bin/github-backup b/bin/github-backup index ab293a6..f34ddfc 100755 --- a/bin/github-backup +++ b/bin/github-backup @@ -234,48 +234,28 @@ def filter_repositories(args, repositories): def backup_repositories(args, output_directory, repositories): log_info('Backing up repositories') repos_template = 'https://{0}/repos'.format(get_github_api_host(args)) - wiki_template = "git@{0}:{1}.wiki.git" issue_states = ['open', 'closed'] pull_states = ['open', 'closed'] + for repository in repositories: backup_cwd = os.path.join(output_directory, 'repositories') repo_cwd = os.path.join(backup_cwd, repository['name']) + repo_dir = os.path.join(repo_cwd, 'repository') + + if args.prefer_ssh: + repo_url = repository['ssh_url'] + else: + repo_url = repository['git_url'] if args.include_repository or args.include_everything: - mkdir_p(backup_cwd, repo_cwd) - exists = os.path.isdir('{0}/repository/.git'.format(repo_cwd)) - if args.skip_existing and exists: - continue - - if exists: - log_info('Updating {0} repository'.format(repository['full_name'])) - git_command = ["git", "pull", 'origin', 'master'] - logging_subprocess(git_command, logger=None, cwd=os.path.join(repo_cwd, 'repository')) - else: - log_info('Cloning {0} repository'.format(repository['full_name'])) - - repo_url = repository['clone_url'] - if args.prefer_ssh: - repo_url = repository['ssh_url'] - - git_command = ['git', 'clone', repo_url, 'repository'] - logging_subprocess(git_command, logger=None, cwd=repo_cwd) + fetch_repository(repository['name'], repo_url, repo_dir, skip_existing=args.skip_existing) if repository['has_wiki'] and (args.include_wiki or args.include_everything): - mkdir_p(backup_cwd, repo_cwd) - exists = os.path.isdir('{0}/wiki/.git'.format(repo_cwd)) - if args.skip_existing and exists: - continue - - if exists: - log_info('Updating {0} wiki'.format(repository['full_name'])) - git_command = ["git", "pull", 'origin', 'master'] - logging_subprocess(git_command, logger=None, cwd=os.path.join(repo_cwd, 'wiki')) - else: - log_info('Cloning {0} wiki'.format(repository['full_name'])) - git_command = ["git", "clone", wiki_template.format(get_github_ssh_host(args), repository['full_name']), 'wiki'] - logging_subprocess(git_command, logger=None, cwd=repo_cwd) + fetch_repository(repository['name'], + repo_url.replace('.git', '.wiki.git'), + os.path.join(repo_cwd, 'wiki'), + skip_existing=args.skip_existing) if args.include_issues or args.include_everything: if args.skip_existing and os.path.isdir('{0}/issues/.git'.format(repo_cwd)): @@ -344,6 +324,22 @@ def backup_repositories(args, output_directory, repositories): json.dump(pull, pull_file, sort_keys=True, indent=4, separators=(',', ': ')) +def fetch_repository(name, remote_url, local_dir, skip_existing=False): + clone_exists = os.path.exists(os.path.join(local_dir, '.git')) + + if clone_exists and skip_existing: + return + + if clone_exists: + log_info('Updating {} in {}'.format(name, local_dir)) + git_command = ['git', 'fetch', '--all', '--tags', '--prune'] + logging_subprocess(git_command, None, cwd=local_dir) + else: + log_info('Cloning {} repository from {} to {}'.format(name, remote_url, local_dir)) + git_command = ['git', 'clone', remote_url, local_dir] + logging_subprocess(git_command, None) + + def backup_account(args, output_directory): account_cwd = os.path.join(output_directory, 'account') if args.include_starred or args.include_everything: