Release version 0.12.1

Avoid remote branch name churn

CHANGES.rst

@@ -1,7 +1,82 @@
 Changelog
 =========
 
-0.10.2 (2016-08-20)
+0.12.1 (2017-03-27)
+-------------------
+
+- Avoid remote branch name churn. [Chris Adams]
+
+  This avoids the backup output having lots of "[new branch]" messages
+  because removing the old remote name removed all of the existing branch
+  references.
+
+
+- Fix detection of bare git directories. [Andrzej Maczuga]
+
+0.12.0 (2016-11-22)
+-------------------
+
+Fix
+~~~
+
+- Properly import version from github_backup package. [Jose Diaz-
+  Gonzalez]
+
+- Support alternate git status output. [Jose Diaz-Gonzalez]
+
+Other
+~~~~~
+
+- Pep8: E501 line too long (83 > 79 characters) [Jose Diaz-Gonzalez]
+
+- Pep8: E128 continuation line under-indented for visual indent. [Jose
+  Diaz-Gonzalez]
+
+- Support archivization using bare git clones. [Andrzej Maczuga]
+
+- Fix typo, 3x. [Terrell Russell]
+
+0.11.0 (2016-10-26)
+-------------------
+
+- Support --token file:///home/user/token.txt (fixes gh-51) [Björn
+  Dahlgren]
+
+- Fix some linting. [Albert Wang]
+
+- Fix byte/string conversion for python 3. [Albert Wang]
+
+- Support python 3. [Albert Wang]
+
+- Encode special characters in password. [Remi Rampin]
+
+- Don't pretend program name is "Github Backup" [Remi Rampin]
+
+- Don't install over insecure connection. [Remi Rampin]
+
+  The git:// protocol is unauthenticated and unencrypted, and no longer advertised by GitHub. Using HTTPS shouldn't impact performance.
+
+0.10.3 (2016-08-21)
+-------------------
+
+- Fixes #29. [Jonas Michel]
+
+  Reporting an error when the user's rate limit is exceeded causes
+  the script to terminate after resuming execution from a rate limit
+  sleep. Instead of generating an explicit error we just want to
+  inform the user that the script is going to sleep until their rate
+  limit count resets.
+
+
+- Fixes #29. [Jonas Michel]
+
+  The errors list was not being cleared out after resuming a backup
+  from a rate limit sleep. When the backup was resumed, the non-empty
+  errors list caused the backup to quit after the next `retrieve_data`
+  request.
+
+
+0.10.2 (2016-08-21)
 -------------------
 
 - Add a note regarding git version requirement. [Jose Diaz-Gonzalez]

README.rst

@@ -18,21 +18,22 @@ Using PIP via PyPI::
 
 Using PIP via Github::
 
-    pip install git+git://github.com/josegonzalez/python-github-backup.git#egg=github-backup
+    pip install git+https://github.com/josegonzalez/python-github-backup.git#egg=github-backup
 
 Usage
 =====
 
 CLI Usage is as follows::
 
-    Github Backup [-h] [-u USERNAME] [-p PASSWORD] [-t TOKEN]
+    github-backup [-h] [-u USERNAME] [-p PASSWORD] [-t TOKEN]
-                     [-o OUTPUT_DIRECTORY] [--starred] [--watched] [--all]
+                  [-o OUTPUT_DIRECTORY] [-i] [--starred] [--watched]
-                     [--issues] [--issue-comments] [--issue-events] [--pulls]
+                  [--all] [--issues] [--issue-comments] [--issue-events]
-                     [--pull-comments] [--pull-commits] [--labels] [--hooks]
+                  [--pulls] [--pull-comments] [--pull-commits] [--labels]
-                     [--milestones] [--repositories] [--wikis]
+                  [--hooks] [--milestones] [--repositories] [--bare]
-                     [--skip-existing] [-L [LANGUAGES [LANGUAGES ...]]]
+                  [--wikis] [--skip-existing]
-                     [-N NAME_REGEX] [-H GITHUB_HOST] [-O] [-R REPOSITORY]
+                  [-L [LANGUAGES [LANGUAGES ...]]] [-N NAME_REGEX]
-                     [-P] [-F] [--prefer-ssh] [-v]
+                  [-H GITHUB_HOST] [-O] [-R REPOSITORY] [-P] [-F]
+                  [--prefer-ssh] [-v]
                   USER
 
     Backup a github account
@@ -51,6 +52,7 @@ CLI Usage is as follows::
                             personal access or OAuth token
       -o OUTPUT_DIRECTORY, --output-directory OUTPUT_DIRECTORY
                             directory at which to backup the repositories
+      -i, --incremental     incremental backup
       --starred             include starred repositories in backup
       --watched             include watched repositories in backup
       --all                 include everything in backup
@@ -65,6 +67,7 @@ CLI Usage is as follows::
                             authenticated)
       --milestones          include milestones in backup
       --repositories        include repository clone in backup
+      --bare                clone bare repositories
       --wikis               include wiki clone in backup
       --skip-existing       skip project if a backup directory exists
       -L [LANGUAGES [LANGUAGES ...]], --languages [LANGUAGES [LANGUAGES ...]]

bin/github-backup

@@ -16,9 +16,22 @@ import select
 import subprocess
 import sys
 import time
-import urlparse
+try:
-import urllib
+    # python 3
-import urllib2
+    from urllib.parse import urlparse
+    from urllib.parse import quote as urlquote
+    from urllib.parse import urlencode
+    from urllib.error import HTTPError, URLError
+    from urllib.request import urlopen
+    from urllib.request import Request
+except ImportError:
+    # python 2
+    from urlparse import urlparse
+    from urllib import quote as urlquote
+    from urllib import urlencode
+    from urllib2 import HTTPError, URLError
+    from urllib2 import urlopen
+    from urllib2 import Request
 
 from github_backup import __version__
 
@@ -80,8 +93,8 @@ def logging_subprocess(popenargs,
     rc = child.wait()
 
     if rc != 0:
-        print(u'{} returned {}:'.format(popenargs[0], rc), file=sys.stderr)
+        print('{} returned {}:'.format(popenargs[0], rc), file=sys.stderr)
-        print('\t', u' '.join(popenargs), file=sys.stderr)
+        print('\t', ' '.join(popenargs), file=sys.stderr)
 
     return rc
 
@@ -96,8 +109,9 @@ def mkdir_p(*args):
             else:
                 raise
 
+
 def mask_password(url, secret='*****'):
-    parsed = urlparse.urlparse(url)
+    parsed = urlparse(url)
 
     if not parsed.password:
         return url
@@ -106,9 +120,9 @@ def mask_password(url, secret='*****'):
 
     return url.replace(parsed.password, secret)
 
+
 def parse_args():
-    parser = argparse.ArgumentParser(description='Backup a github account',
+    parser = argparse.ArgumentParser(description='Backup a github account')
-                                     prog='Github Backup')
     parser.add_argument('user',
                         metavar='USER',
                         type=str,
@@ -126,7 +140,7 @@ def parse_args():
     parser.add_argument('-t',
                         '--token',
                         dest='token',
-                        help='personal access or OAuth token')
+                        help='personal access or OAuth token, or path to token (file://...)')  # noqa
     parser.add_argument('-o',
                         '--output-directory',
                         default='.',
@@ -180,7 +194,7 @@ def parse_args():
     parser.add_argument('--hooks',
                         action='store_true',
                         dest='include_hooks',
-                        help='include hooks in backup (works only when authenticated)')
+                        help='include hooks in backup (works only when authenticated)')  # noqa
     parser.add_argument('--milestones',
                         action='store_true',
                         dest='include_milestones',
@@ -189,6 +203,10 @@ def parse_args():
                         action='store_true',
                         dest='include_repository',
                         help='include repository clone in backup')
+    parser.add_argument('--bare',
+                        action='store_true',
+                        dest='bare_clone',
+                        help='clone bare repositories')
     parser.add_argument('--wikis',
                         action='store_true',
                         dest='include_wiki',
@@ -240,21 +258,29 @@ def get_auth(args, encode=True):
     auth = None
 
     if args.token:
+        _path_specifier = 'file://'
+        if args.token.startswith(_path_specifier):
+            args.token = open(args.token[len(_path_specifier):],
+                              'rt').readline().strip()
         auth = args.token + ':' + 'x-oauth-basic'
     elif args.username:
         if not args.password:
             args.password = getpass.getpass()
-        auth = args.username + ':' + args.password
+        if encode:
+            password = args.password
+        else:
+            password = urlquote(args.password)
+        auth = args.username + ':' + password
     elif args.password:
         log_error('You must specify a username for basic auth')
 
     if not auth:
         return None
 
-    if encode == False:
+    if not encode:
         return auth
 
-    return base64.b64encode(auth)
+    return base64.b64encode(auth.encode('ascii'))
 
 
 def get_github_api_host(args):
@@ -274,6 +300,7 @@ def get_github_host(args):
 
     return host
 
+
 def get_github_repo_url(args, repository):
     if args.prefer_ssh:
         return repository['ssh_url']
@@ -290,6 +317,7 @@ def get_github_repo_url(args, repository):
 
     return repo_url
 
+
 def retrieve_data(args, template, query_args=None, single_request=False):
     auth = get_auth(args)
     query_args = get_query_args(query_args)
@@ -309,7 +337,7 @@ def retrieve_data(args, template, query_args=None, single_request=False):
             errors.append(template.format(status_code, r.reason))
             log_error(errors)
 
-        response = json.loads(r.read())
+        response = json.loads(r.read().decode('utf-8'))
         if len(errors) == 0:
             if type(response) == list:
                 data.extend(response)
@@ -341,11 +369,11 @@ def _get_response(request, auth, template):
     while True:
         should_continue = False
         try:
-            r = urllib2.urlopen(request)
+            r = urlopen(request)
-        except urllib2.HTTPError as exc:
+        except HTTPError as exc:
             errors, should_continue = _request_http_error(exc, auth, errors)  # noqa
             r = exc
-        except urllib2.URLError:
+        except URLError:
             should_continue = _request_url_error(template, retry_timeout)
             if not should_continue:
                 raise
@@ -358,14 +386,14 @@ def _get_response(request, auth, template):
 
 
 def _construct_request(per_page, page, query_args, template, auth):
-    querystring = urllib.urlencode(dict({
+    querystring = urlencode(dict(list({
         'per_page': per_page,
         'page': page
-    }.items() + query_args.items()))
+    }.items()) + list(query_args.items())))
 
-    request = urllib2.Request(template + '?' + querystring)
+    request = Request(template + '?' + querystring)
     if auth is not None:
-        request.add_header('Authorization', 'Basic ' + auth)
+        request.add_header('Authorization', 'Basic '.encode('ascii') + auth)
     return request
 
 
@@ -392,10 +420,9 @@ def _request_http_error(exc, auth, errors):
         print('Exceeded rate limit of {} requests; waiting {} seconds to reset'.format(limit, delta),  # noqa
               file=sys.stderr)
 
-        ratelimit_error = 'No more requests remaining'
         if auth is None:
-            ratelimit_error += '; authenticate to raise your GitHub rate limit'  # noqa
+            print('Hint: Authenticate to raise your GitHub rate limit',
-        errors.append(ratelimit_error)
+                  file=sys.stderr)
 
         time.sleep(delta)
         should_continue = True
@@ -435,10 +462,13 @@ def retrieve_repositories(args):
     return retrieve_data(args, template, single_request=single_request)
 
 
-def filter_repositories(args, repositories):
+def filter_repositories(args, unfiltered_repositories):
     log_info('Filtering repositories')
 
-    repositories = [r for r in repositories if r['owner']['login'] == args.user]
+    repositories = []
+    for r in unfiltered_repositories:
+        if r['owner']['login'] == args.user:
+            repositories.append(r)
 
     name_regex = None
     if args.name_regex:
@@ -465,7 +495,7 @@ def backup_repositories(args, output_directory, repositories):
     repos_template = 'https://{0}/repos'.format(get_github_api_host(args))
 
     if args.incremental:
-        last_update = max(repository['updated_at'] for repository in repositories)
+        last_update = max(repository['updated_at'] for repository in repositories)  # noqa
         last_update_path = os.path.join(output_directory, 'last_update')
         if os.path.exists(last_update_path):
             args.since = open(last_update_path).read().strip()
@@ -484,14 +514,16 @@ def backup_repositories(args, output_directory, repositories):
             fetch_repository(repository['name'],
                              repo_url,
                              repo_dir,
-                             skip_existing=args.skip_existing)
+                             skip_existing=args.skip_existing,
+                             bare_clone=args.bare_clone)
 
         download_wiki = (args.include_wiki or args.include_everything)
         if repository['has_wiki'] and download_wiki:
             fetch_repository(repository['name'],
                              repo_url.replace('.git', '.wiki.git'),
                              os.path.join(repo_cwd, 'wiki'),
-                             skip_existing=args.skip_existing)
+                             skip_existing=args.skip_existing,
+                             bare_clone=args.bare_clone)
 
         if args.include_issues or args.include_everything:
             backup_issues(args, repo_cwd, repository, repos_template)
@@ -511,6 +543,7 @@ def backup_repositories(args, output_directory, repositories):
     if args.incremental:
         open(last_update_path, 'w').write(last_update)
 
+
 def backup_issues(args, repo_cwd, repository, repos_template):
     has_issues_dir = os.path.isdir('{0}/issues/.git'.format(repo_cwd))
     if args.skip_existing and has_issues_dir:
@@ -526,6 +559,7 @@ def backup_issues(args, repo_cwd, repository, repos_template):
     _issue_template = '{0}/{1}/issues'.format(repos_template,
                                               repository['full_name'])
 
+    should_include_pulls = args.include_pulls or args.include_everything
     issue_states = ['open', 'closed']
     for issue_state in issue_states:
         query_args = {
@@ -541,18 +575,21 @@ def backup_issues(args, repo_cwd, repository, repos_template):
         for issue in _issues:
             # skip pull requests which are also returned as issues
             # if retrieving pull requests is requested as well
-            if 'pull_request' in issue and (args.include_pulls or args.include_everything):
+            if 'pull_request' in issue and should_include_pulls:
                 issues_skipped += 1
                 continue
 
             issues[issue['number']] = issue
 
     if issues_skipped:
-        issues_skipped_message = ' (skipped {0} pull requests)'.format(issues_skipped)
+        issues_skipped_message = ' (skipped {0} pull requests)'.format(
-    log_info('Saving {0} issues to disk{1}'.format(len(issues.keys()), issues_skipped_message))
+            issues_skipped)
+
+    log_info('Saving {0} issues to disk{1}'.format(
+        len(list(issues.keys())), issues_skipped_message))
     comments_template = _issue_template + '/{0}/comments'
     events_template = _issue_template + '/{0}/events'
-    for number, issue in issues.iteritems():
+    for number, issue in list(issues.items()):
         if args.include_issue_comments or args.include_everything:
             template = comments_template.format(number)
             issues[number]['comment_data'] = retrieve_data(args, template)
@@ -595,10 +632,11 @@ def backup_pulls(args, repo_cwd, repository, repos_template):
             if not args.since or pull['updated_at'] >= args.since:
                 pulls[pull['number']] = pull
 
-    log_info('Saving {0} pull requests to disk'.format(len(pulls.keys())))
+    log_info('Saving {0} pull requests to disk'.format(
+        len(list(pulls.keys()))))
     comments_template = _pulls_template + '/{0}/comments'
     commits_template = _pulls_template + '/{0}/commits'
-    for number, pull in pulls.iteritems():
+    for number, pull in list(pulls.items()):
         if args.include_pull_comments or args.include_everything:
             template = comments_template.format(number)
             pulls[number]['comment_data'] = retrieve_data(args, template)
@@ -632,8 +670,9 @@ def backup_milestones(args, repo_cwd, repository, repos_template):
     for milestone in _milestones:
         milestones[milestone['number']] = milestone
 
-    log_info('Saving {0} milestones to disk'.format(len(milestones.keys())))
+    log_info('Saving {0} milestones to disk'.format(
-    for number, milestone in milestones.iteritems():
+        len(list(milestones.keys()))))
+    for number, milestone in list(milestones.items()):
         milestone_file = '{0}/{1}.json'.format(milestone_cwd, number)
         with codecs.open(milestone_file, 'w', encoding='utf-8') as f:
             json_dump(milestone, f)
@@ -670,7 +709,20 @@ def backup_hooks(args, repo_cwd, repository, repos_template):
         log_info("Unable to read hooks, skipping")
 
 
-def fetch_repository(name, remote_url, local_dir, skip_existing=False):
+def fetch_repository(name,
+                     remote_url,
+                     local_dir,
+                     skip_existing=False,
+                     bare_clone=False):
+    if bare_clone:
+        if os.path.exists(local_dir):
+            clone_exists = subprocess.check_output(['git',
+                                                    'rev-parse',
+                                                    '--is-bare-repository'],
+                                                   cwd=local_dir) == "true\n"
+        else:
+            clone_exists = False
+    else:
         clone_exists = os.path.exists(os.path.join(local_dir, '.git'))
 
     if clone_exists and skip_existing:
@@ -678,26 +730,41 @@ def fetch_repository(name, remote_url, local_dir, skip_existing=False):
 
     masked_remote_url = mask_password(remote_url)
 
-    initalized = subprocess.call('git ls-remote ' + remote_url,
+    initialized = subprocess.call('git ls-remote ' + remote_url,
                                   stdout=FNULL,
                                   stderr=FNULL,
                                   shell=True)
-    if initalized == 128:
+    if initialized == 128:
-        log_info("Skipping {0} ({1}) since it's not initalized".format(name, masked_remote_url))
+        log_info("Skipping {0} ({1}) since it's not initialized".format(
+            name, masked_remote_url))
         return
 
     if clone_exists:
         log_info('Updating {0} in {1}'.format(name, local_dir))
+
+        remotes = subprocess.check_output(['git', 'remote', 'show'],
+                                          cwd=local_dir)
+        remotes = [i.strip() for i in remotes.decode('utf-8')]
+
+        if 'origin' not in remotes:
             git_command = ['git', 'remote', 'rm', 'origin']
             logging_subprocess(git_command, None, cwd=local_dir)
             git_command = ['git', 'remote', 'add', 'origin', remote_url]
             logging_subprocess(git_command, None, cwd=local_dir)
-        git_command = ['git', 'fetch', '--all', '--tags', '--prune']
+        else:
+            git_command = ['git', 'remote', 'set-url', 'origin', remote_url]
+            logging_subprocess(git_command, None, cwd=local_dir)
+
+        git_command = ['git', 'fetch', '--all', '--force', '--tags', '--prune']
         logging_subprocess(git_command, None, cwd=local_dir)
     else:
-        log_info('Cloning {0} repository from {1} to {2}'.format(name,
+        log_info('Cloning {0} repository from {1} to {2}'.format(
+            name,
             masked_remote_url,
             local_dir))
+        if bare_clone:
+            git_command = ['git', 'clone', '--mirror', remote_url, local_dir]
+        else:
             git_command = ['git', 'clone', remote_url, local_dir]
         logging_subprocess(git_command, None)
 

github_backup/__init__.py

@@ -1 +1 @@
-__version__ = '0.10.2'
+__version__ = '0.12.1'

release

@@ -34,7 +34,7 @@ fi
 echo -e "\n${GREEN}STARTING RELEASE PROCESS${COLOR_OFF}\n"
 
 set +e;
-git status | grep "working directory clean" &> /dev/null
+git status | grep -Eo "working (directory|tree) clean" &> /dev/null
 if [ ! $? -eq 0 ]; then # working directory is NOT clean
     echo -e "${RED}WARNING: You have uncomitted changes, you may have forgotten something${COLOR_OFF}\n"
     exit 1