Release version 0.41.0

git lfs clone does not respect --mirror

.gitignore

@@ -34,3 +34,4 @@ doc/aws_hostname.1
 .vscode
 .atom
 
+README

CHANGES.rst

@@ -1,9 +1,92 @@
 Changelog
 =========
 
-0.37.0 (2021-01-01)
+0.39.0 (2021-03-18)
 -------------------
 ------------
+
+Fix
+~~~
+- Fix missing INFO logs. [Gallo Feliz]
+
+Other
+~~~~~
+- Merge pull request #173 from gallofeliz/make-compatible-python-call.
+  [Jose Diaz-Gonzalez]
+
+  Try to make compatible code with direct Python call ; reduce the hard link of the code with the cli
+- Try to make compatible code with direct Python call ; reduce the hard
+  link of the code with the cli. [Gallo Feliz]
+- Merge pull request #174 from atorrescogollo/master. [Jose Diaz-
+  Gonzalez]
+
+  Fixed release_name with slash bug
+- Fixed release_name with slash bug. [Álvaro Torres Cogollo]
+
+
+0.38.0 (2021-02-13)
+-------------------
+
+Fix
+~~~
+- Always clone with OAuth token when provided. [Samantha Baldwin]
+
+  Github Enterprise servers with 'Anonymous Git read access' disabled
+  cause `git ls-remote` to fail (128) for a repo's `clone_url`. Using the
+  OAuth token when provided allows cloning private AND public repos when
+  Anonymous Git read access is disabled.
+
+Other
+~~~~~
+- Release version 0.38.0. [Jose Diaz-Gonzalez]
+- Merge pull request #172 from samanthaq/always-use-oauth-when-provided.
+  [Jose Diaz-Gonzalez]
+
+  fix: Always clone with OAuth token when provided
+- Merge pull request #170 from Mindavi/bugfix/broken-url. [Jose Diaz-
+  Gonzalez]
+
+  Fix broken and incorrect link to github repository
+- Change broken link to a fork to a working link to upstream. [Rick van
+  Schijndel]
+
+
+0.37.2 (2021-01-02)
+-------------------
+
+Fix
+~~~
+- Use distutils.core on error. [Jose Diaz-Gonzalez]
+
+Other
+~~~~~
+- Release version 0.37.2. [Jose Diaz-Gonzalez]
+
+
+0.37.1 (2021-01-02)
+-------------------
+
+Fix
+~~~
+- Use twine for releases. [Jose Diaz-Gonzalez]
+
+  The old method of releasing to pypi broke for whatever reason and switching to a supported toolchain is easier than debugging the old one.
+
+  Additionally:
+
+  - Update gitchangelog
+  - Fix license entry
+  - Set long description type
+  - Gitignore the temporary readme file
+
+Other
+~~~~~
+- Release version 0.37.1. [Jose Diaz-Gonzalez]
+
+
+0.37.0 (2021-01-02)
+-------------------
+- Release version 0.37.0. [Jose Diaz-Gonzalez]
 - Merge pull request #158 from albertyw/python3. [Jose Diaz-Gonzalez]
 
   Remove support for python 2

README.rst

@@ -178,4 +178,4 @@ This project currently contains no unit tests.  To run linting::
 .. |PyPI| image:: https://img.shields.io/pypi/v/github-backup.svg
    :target: https://pypi.python.org/pypi/github-backup/
 .. |Python Versions| image:: https://img.shields.io/pypi/pyversions/github-backup.svg
-   :target: https://github.com/albertyw/github-backup
+   :target: https://github.com/josegonzalez/python-github-backup

bin/github-backup

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-import os
+import os, sys, logging
 
 from github_backup.github_backup import (
     backup_account,
@@ -9,11 +9,17 @@ from github_backup.github_backup import (
     filter_repositories,
     get_authenticated_user,
     log_info,
+    log_warning,
     mkdir_p,
     parse_args,
     retrieve_repositories,
 )
 
+logging.basicConfig(
+    format='%(asctime)s.%(msecs)03d: %(message)s',
+    datefmt='%Y-%m-%dT%H:%M:%S',
+    level=logging.INFO
+)
 
 def main():
     args = parse_args()
@@ -39,4 +45,8 @@ def main():
 
 
 if __name__ == '__main__':
+    try:
         main()
+    except Exception as e:
+        log_warning(str(e))
+        sys.exit(1)

github_backup/__init__.py

@@ -1 +1 @@
-__version__ = '0.37.0'
+__version__ = '0.41.0'

github_backup/github_backup.py

@@ -11,12 +11,12 @@ import datetime
 import errno
 import getpass
 import json
-import logging
 import os
 import re
 import select
 import subprocess
 import sys
+import logging
 import time
 import platform
 from urllib.parse import urlparse
@@ -27,6 +27,7 @@ from urllib.request import urlopen
 from urllib.request import Request
 from urllib.request import HTTPRedirectHandler
 from urllib.request import build_opener
+from http.client import IncompleteRead
 
 try:
     from . import __version__
@@ -41,14 +42,6 @@ def _get_log_date():
     return datetime.datetime.isoformat(datetime.datetime.now())
 
 
-def log_error(message):
-    """
-    Log message (str) or messages (List[str]) to stderr and exit with status 1
-    """
-    log_warning(message)
-    sys.exit(1)
-
-
 def log_info(message):
     """
     Log message (str) or messages (List[str]) to stdout
@@ -57,7 +50,7 @@ def log_info(message):
         message = [message]
 
     for msg in message:
-        sys.stdout.write("{0}: {1}\n".format(_get_log_date(), msg))
+        logging.info(msg)
 
 
 def log_warning(message):
@@ -68,7 +61,7 @@ def log_warning(message):
         message = [message]
 
     for msg in message:
-        sys.stderr.write("{0}: {1}\n".format(_get_log_date(), msg))
+        logging.warning(msg)
 
 
 def logging_subprocess(popenargs,
@@ -140,7 +133,7 @@ def mask_password(url, secret='*****'):
     return url.replace(parsed.password, secret)
 
 
-def parse_args():
+def parse_args(args=None):
     parser = argparse.ArgumentParser(description='Backup a github account')
     parser.add_argument('user',
                         metavar='USER',
@@ -331,7 +324,7 @@ def parse_args():
                         type=float,
                         default=30.0,
                         help='wait this amount of seconds when API request throttling is active (default: 30.0, requires --throttle-limit to be set)')
-    return parser.parse_args()
+    return parser.parse_args(args)
 
 
 def get_auth(args, encode=True, for_git_cli=False):
@@ -339,10 +332,10 @@ def get_auth(args, encode=True, for_git_cli=False):
 
     if args.osx_keychain_item_name:
         if not args.osx_keychain_item_account:
-            log_error('You must specify both name and account fields for osx keychain password items')
+            raise Exception('You must specify both name and account fields for osx keychain password items')
         else:
             if platform.system() != 'Darwin':
-                log_error("Keychain arguments are only supported on Mac OSX")
+                raise Exception("Keychain arguments are only supported on Mac OSX")
             try:
                 with open(os.devnull, 'w') as devnull:
                     token = (subprocess.check_output([
@@ -353,9 +346,9 @@ def get_auth(args, encode=True, for_git_cli=False):
                 token = token.decode('utf-8')
                 auth = token + ':' + 'x-oauth-basic'
             except subprocess.SubprocessError:
-                log_error('No password item matching the provided name and account could be found in the osx keychain.')
+                raise Exception('No password item matching the provided name and account could be found in the osx keychain.')
     elif args.osx_keychain_item_account:
-        log_error('You must specify both name and account fields for osx keychain password items')
+        raise Exception('You must specify both name and account fields for osx keychain password items')
     elif args.token:
         _path_specifier = 'file://'
         if args.token.startswith(_path_specifier):
@@ -377,7 +370,7 @@ def get_auth(args, encode=True, for_git_cli=False):
             password = urlquote(args.password)
         auth = args.username + ':' + password
     elif args.password:
-        log_error('You must specify a username for basic auth')
+        raise Exception('You must specify a username for basic auth')
 
     if not auth:
         return None
@@ -420,7 +413,7 @@ def get_github_repo_url(args, repository):
         return repository['ssh_url']
 
     auth = get_auth(args, encode=False, for_git_cli=True)
-    if auth and repository['private'] is True:
+    if auth:
         repo_url = 'https://{0}@{1}/{2}/{3}.git'.format(
             auth,
             get_github_host(args),
@@ -444,6 +437,21 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False):
         r, errors = _get_response(request, auth, template)
 
         status_code = int(r.getcode())
+        # Check if we got correct data
+        try:
+            response = json.loads(r.read().decode('utf-8'))
+        except IncompleteRead:
+            log_warning("Incomplete read error detected")
+            read_error = True
+        except json.decoder.JSONDecodeError:
+            log_warning("JSON decode error detected")
+            read_error = True
+        except TimeoutError:
+            log_warning("Tiemout error detected")
+            read_error = True
+        else:
+            read_error = False
+
         # be gentle with API request limit and throttle requests if remaining requests getting low
         limit_remaining = int(r.headers.get('x-ratelimit-remaining', 0))
         if args.throttle_limit and limit_remaining <= args.throttle_limit:
@@ -454,21 +462,37 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False):
             time.sleep(args.throttle_pause)
 
         retries = 0
-        while retries < 3 and status_code == 502:
+        while retries < 3 and (status_code == 502 or read_error):
-            log_warning('API request returned HTTP 502: Bad Gateway. Retrying in 5 seconds')
+            log_warning('API request failed. Retrying in 5 seconds')
             retries += 1
             time.sleep(5)
             request = _construct_request(per_page, page, query_args, template, auth, as_app=args.as_app)  # noqa
             r, errors = _get_response(request, auth, template)
 
             status_code = int(r.getcode())
+            try:
+                response = json.loads(r.read().decode('utf-8'))
+                read_error = False
+            except IncompleteRead:
+                log_warning("Incomplete read error detected")
+                read_error = True
+            except json.decoder.JSONDecodeError:
+                log_warning("JSON decode error detected")
+                read_error = True
+            except TimeoutError:
+                log_warning("Tiemout error detected")
+                read_error = True
 
         if status_code != 200:
             template = 'API request returned HTTP {0}: {1}'
             errors.append(template.format(status_code, r.reason))
-            log_error(errors)
+            raise Exception(', '.join(errors))
+
+        if read_error:
+            template = 'API request problem reading response for {0}'
+            errors.append(template.format(request))
+            raise Exception(', '.join(errors))
 
-        response = json.loads(r.read().decode('utf-8'))
         if len(errors) == 0:
             if type(response) == list:
                 for resp in response:
@@ -479,7 +503,7 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False):
                 yield response
 
         if len(errors) > 0:
-            log_error(errors)
+            raise Exception(', '.join(errors))
 
         if single_request:
             break
@@ -582,7 +606,7 @@ def _request_url_error(template, retry_timeout):
     if retry_timeout >= 0:
         return True
 
-    log_error('{} timed out to much, skipping!')
+    raise Exception('{} timed out to much, skipping!')
     return False
 
 
@@ -640,7 +664,7 @@ def get_authenticated_user(args):
 def check_git_lfs_install():
     exit_code = subprocess.call(['git', 'lfs', 'version'])
     if exit_code != 0:
-        log_error('The argument --lfs requires you to have Git LFS installed.\nYou can get it from https://git-lfs.github.com.')
+        raise Exception('The argument --lfs requires you to have Git LFS installed.\nYou can get it from https://git-lfs.github.com.')
 
 
 def retrieve_repositories(args, authenticated_user):
@@ -1009,7 +1033,8 @@ def backup_releases(args, repo_cwd, repository, repos_template, include_assets=F
     log_info('Saving {0} releases to disk'.format(len(releases)))
     for release in releases:
         release_name = release['tag_name']
-        output_filepath = os.path.join(release_cwd, '{0}.json'.format(release_name))
+        release_name_safe = release_name.replace('/', '__')
+        output_filepath = os.path.join(release_cwd, '{0}.json'.format(release_name_safe))
         with codecs.open(output_filepath, 'w+', encoding='utf-8') as f:
             json_dump(release, f)
 
@@ -1017,7 +1042,7 @@ def backup_releases(args, repo_cwd, repository, repos_template, include_assets=F
             assets = retrieve_data(args, release['assets_url'])
             if len(assets) > 0:
                 # give release asset files somewhere to live & download them (not including source archives)
-                release_assets_cwd = os.path.join(release_cwd, release_name)
+                release_assets_cwd = os.path.join(release_cwd, release_name_safe)
                 mkdir_p(release_assets_cwd)
                 for asset in assets:
                     download_file(asset['url'], os.path.join(release_assets_cwd, asset['name']), get_auth(args))
@@ -1081,10 +1106,11 @@ def fetch_repository(name,
             masked_remote_url,
             local_dir))
         if bare_clone:
-            if lfs_clone:
-                git_command = ['git', 'lfs', 'clone', '--mirror', remote_url, local_dir]
-            else:
             git_command = ['git', 'clone', '--mirror', remote_url, local_dir]
+            logging_subprocess(git_command, None)
+            if lfs_clone:
+                git_command = ['git', 'lfs', 'fetch', '--all', '--prune']
+                logging_subprocess(git_command, None, cwd=local_dir)
         else:
             if lfs_clone:
                 git_command = ['git', 'lfs', 'clone', remote_url, local_dir]

release

@@ -22,7 +22,7 @@ CYAN="\033[0;36m"     # cyan
 pip install wheel > /dev/null
 
 command -v gitchangelog >/dev/null 2>&1 || {
-    echo -e "${RED}WARNING: Missing gitchangelog binary, please run: pip install gitchangelog==2.2.0${COLOR_OFF}\n"
+    echo -e "${RED}WARNING: Missing gitchangelog binary, please run: pip install gitchangelog==3.0.4${COLOR_OFF}\n"
     exit 1
 }
 
@@ -31,6 +31,11 @@ command -v rst-lint > /dev/null || {
     exit 1
 }
 
+command -v twine > /dev/null || {
+    echo -e "${RED}WARNING: Missing twine binary, please run: pip install twine==3.2.0${COLOR_OFF}\n"
+    exit 1
+}
+
 if [[ "$@" != "major" ]] && [[ "$@" != "minor" ]] && [[ "$@" != "patch" ]]; then
     echo -e "${RED}WARNING: Invalid release type, must specify 'major', 'minor', or 'patch'${COLOR_OFF}\n"
     exit 1
@@ -125,7 +130,8 @@ git push -q origin master && git push -q --tags
 if [[ "$PUBLIC" == "true" ]]; then
     echo -e "${YELLOW}--->${COLOR_OFF} Creating python release"
     cp README.rst README
-    python setup.py sdist bdist_wheel upload > /dev/null
+    python setup.py sdist bdist_wheel > /dev/null
+    twine upload dist/*
     rm README
 fi
 

setup.py

@@ -1,10 +1,24 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 import os
-from setuptools import setup
-
 from github_backup import __version__
 
+try:
+    from setuptools import setup
+    setup  # workaround for pyflakes issue #13
+except ImportError:
+    from distutils.core import setup
+
+# Hack to prevent stupid TypeError: 'NoneType' object is not callable error on
+# exit of python setup.py test # in multiprocessing/util.py _exit_function when
+# running python setup.py test (see
+# http://www.eby-sarna.com/pipermail/peak/2010-May/003357.html)
+try:
+    import multiprocessing
+    multiprocessing
+except ImportError:
+    pass
+
 
 def open_file(fname):
     return open(os.path.join(os.path.dirname(__file__), fname))
@@ -18,7 +32,7 @@ setup(
     packages=['github_backup'],
     scripts=['bin/github-backup'],
     url='http://github.com/josegonzalez/python-github-backup',
-    license=open('LICENSE.txt').read(),
+    license='MIT',
     classifiers=[
         'Development Status :: 5 - Production/Stable',
         'Topic :: System :: Archiving :: Backup',
@@ -30,6 +44,7 @@ setup(
     ],
     description='backup a github user or organization',
     long_description=open_file('README.rst').read(),
+    long_description_content_type='text/x-rst',
     install_requires=open_file('requirements.txt').readlines(),
     zip_safe=True,
 )