Release version 0.6.0

Add backup hooks

CHANGES.rst

@@ -1,6 +1,41 @@
 Changelog
 =========
 
+0.6.0 (2015-11-10)
+------------------
+
+- Force proper remote url. [Jose Diaz-Gonzalez]
+
+- Improve error handling in case of HTTP errors. [Enrico Tröger]
+
+  In case of a HTTP status code 404, the returned 'r' was never assigned.
+  In case of URL errors which are not timeouts, we probably should bail
+  out.
+
+
+- Add --hooks to also include web hooks into the backup. [Enrico Tröger]
+
+- Create the user specified output directory if it does not exist.
+  [Enrico Tröger]
+
+  Fixes #17.
+
+
+- Add missing auth argument to _get_response() [Enrico Tröger]
+
+  When running unauthenticated and Github starts rate-limiting the client,
+  github-backup crashes because the used auth variable in _get_response()
+  was not available. This change should fix it.
+
+
+- Add repository URL to error message for non-existing repositories.
+  [Enrico Tröger]
+
+  This makes it easier for the user to identify which repository does not
+  exist or is not initialised, i.e. whether it is the main repository or
+  the wiki repository and which clone URL was used to check.
+
+
 0.5.0 (2015-10-10)
 ------------------
 

README.rst

@@ -24,7 +24,7 @@ CLI Usage is as follows::
                      [-o OUTPUT_DIRECTORY] [--starred] [--watched] [--all]
                      [--issues] [--issue-comments] [--issue-events] [--pulls]
                      [--pull-comments] [--pull-commits] [--repositories]
-                     [--wikis] [--skip-existing]
+                     [--wikis] [--labels] [--hooks] [--skip-existing]
                      [-L [LANGUAGES [LANGUAGES ...]]] [-N NAME_REGEX]
                      [-H GITHUB_HOST] [-O] [-R REPOSITORY] [-P] [-F]
                      [--prefer-ssh] [-v]
@@ -57,6 +57,8 @@ CLI Usage is as follows::
       --pull-commits        include pull request commits in backup
       --repositories        include repository clone in backup
       --wikis               include wiki clone in backup
+      --labels              include labels in backup
+      --hooks               include web hooks in backup (works only when authenticated)
       --skip-existing       skip project if a backup directory exists
       -L [LANGUAGES [LANGUAGES ...]], --languages [LANGUAGES [LANGUAGES ...]]
                             only allow these languages
@@ -71,6 +73,6 @@ CLI Usage is as follows::
       -F, --fork            include forked repositories
       --prefer-ssh          Clone repositories using SSH instead of HTTPS
       -v, --version         show program's version number and exit
-      
+
 
 The package can be used to backup an *entire* organization or repository, including issues and wikis in the most appropriate format (clones for wikis, json files for issues).

bin/github-backup

@@ -159,6 +159,10 @@ def parse_args():
                         action='store_true',
                         dest='include_labels',
                         help='include labels in backup')
+    parser.add_argument('--hooks',
+                        action='store_true',
+                        dest='include_hooks',
+                        help='include hooks in backup (works only when authenticated)')
     parser.add_argument('--milestones',
                         action='store_true',
                         dest='include_milestones',
@@ -256,7 +260,7 @@ def retrieve_data(args, template, query_args=None, single_request=False):
     while True:
         page = page + 1
         request = _construct_request(per_page, page, query_args, template, auth)  # noqa
-        r, errors = _get_response(request, template)
+        r, errors = _get_response(request, auth, template)
 
         status_code = int(r.getcode())
 
@@ -289,7 +293,7 @@ def get_query_args(query_args=None):
     return query_args
 
 
-def _get_response(request, template):
+def _get_response(request, auth, template):
     retry_timeout = 3
     errors = []
     # We'll make requests in a loop so we can
@@ -300,8 +304,11 @@ def _get_response(request, template):
             r = urllib2.urlopen(request)
         except urllib2.HTTPError as exc:
             errors, should_continue = _request_http_error(exc, auth, errors)  # noqa
+            r = exc
         except urllib2.URLError:
             should_continue = _request_url_error(template, retry_timeout)
+            if not should_continue:
+                raise
 
         if should_continue:
             continue
@@ -450,6 +457,9 @@ def backup_repositories(args, output_directory, repositories):
         if args.include_labels or args.include_everything:
             backup_labels(args, repo_cwd, repository, repos_template)
 
+        if args.include_hooks or args.include_everything:
+            backup_hooks(args, repo_cwd, repository, repos_template)
+
 
 def backup_issues(args, repo_cwd, repository, repos_template):
     has_issues_dir = os.path.isdir('{0}/issues/.git'.format(repo_cwd))
@@ -575,6 +585,22 @@ def backup_labels(args, repo_cwd, repository, repos_template):
                  label_cwd)
 
 
+def backup_hooks(args, repo_cwd, repository, repos_template):
+    auth = get_auth(args)
+    if not auth:
+        log_info("Skipping hooks since no authentication provided")
+        return
+    hook_cwd = os.path.join(repo_cwd, 'hooks')
+    output_file = '{0}/hooks.json'.format(hook_cwd)
+    template = '{0}/{1}/hooks'.format(repos_template,
+                                       repository['full_name'])
+    _backup_data(args,
+                 'hooks',
+                 template,
+                 output_file,
+                 hook_cwd)
+
+
 def fetch_repository(name, remote_url, local_dir, skip_existing=False):
     clone_exists = os.path.exists(os.path.join(local_dir, '.git'))
 
@@ -586,11 +612,15 @@ def fetch_repository(name, remote_url, local_dir, skip_existing=False):
                                  stderr=FNULL,
                                  shell=True)
     if initalized == 128:
-        log_info("Skipping {0} since it's not initalized".format(name))
+        log_info("Skipping {0} ({1}) since it's not initalized".format(name, remote_url))
         return
 
     if clone_exists:
         log_info('Updating {0} in {1}'.format(name, local_dir))
+        git_command = ['git', 'remote', 'rm', 'origin']
+        logging_subprocess(git_command, None, cwd=local_dir)
+        git_command = ['git', 'remote', 'add', 'origin', remote_url]
+        logging_subprocess(git_command, None, cwd=local_dir)
         git_command = ['git', 'fetch', '--all', '--tags', '--prune']
         logging_subprocess(git_command, None, cwd=local_dir)
     else:
@@ -651,8 +681,8 @@ def main():
 
     output_directory = os.path.realpath(args.output_directory)
     if not os.path.isdir(output_directory):
-        log_error('Specified output directory is not a directory: {0}'.format(
-            output_directory))
+        log_info('Create output directory {0}'.format(output_directory))
+        mkdir_p(output_directory)
 
     log_info('Backing up user {0} to {1}'.format(args.user, output_directory))
 

github_backup/__init__.py

@@ -1 +1 @@
-__version__ = '0.5.0'
+__version__ = '0.6.0'