Release version 0.41.0

git lfs clone does not respect --mirror

CHANGES.rst

@@ -1,10 +1,32 @@
 Changelog
 =========
 
-0.38.0 (2021-02-13)
+0.39.0 (2021-03-18)
 -------------------
 ------------
 
+Fix
+~~~
+- Fix missing INFO logs. [Gallo Feliz]
+
+Other
+~~~~~
+- Merge pull request #173 from gallofeliz/make-compatible-python-call.
+  [Jose Diaz-Gonzalez]
+
+  Try to make compatible code with direct Python call ; reduce the hard link of the code with the cli
+- Try to make compatible code with direct Python call ; reduce the hard
+  link of the code with the cli. [Gallo Feliz]
+- Merge pull request #174 from atorrescogollo/master. [Jose Diaz-
+  Gonzalez]
+
+  Fixed release_name with slash bug
+- Fixed release_name with slash bug. [Álvaro Torres Cogollo]
+
+
+0.38.0 (2021-02-13)
+-------------------
+
 Fix
 ~~~
 - Always clone with OAuth token when provided. [Samantha Baldwin]
@@ -16,6 +38,7 @@ Fix
 
 Other
 ~~~~~
+- Release version 0.38.0. [Jose Diaz-Gonzalez]
 - Merge pull request #172 from samanthaq/always-use-oauth-when-provided.
   [Jose Diaz-Gonzalez]
 

bin/github-backup

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-import os
+import os, sys, logging
 
 from github_backup.github_backup import (
     backup_account,
@@ -9,11 +9,17 @@ from github_backup.github_backup import (
     filter_repositories,
     get_authenticated_user,
     log_info,
+    log_warning,
     mkdir_p,
     parse_args,
     retrieve_repositories,
 )
 
+logging.basicConfig(
+    format='%(asctime)s.%(msecs)03d: %(message)s',
+    datefmt='%Y-%m-%dT%H:%M:%S',
+    level=logging.INFO
+)
 
 def main():
     args = parse_args()
@@ -39,4 +45,8 @@ def main():
 
 
 if __name__ == '__main__':
+    try:
         main()
+    except Exception as e:
+        log_warning(str(e))
+        sys.exit(1)

github_backup/__init__.py

@@ -1 +1 @@
-__version__ = '0.38.0'
+__version__ = '0.41.0'

github_backup/github_backup.py

@@ -11,12 +11,12 @@ import datetime
 import errno
 import getpass
 import json
-import logging
 import os
 import re
 import select
 import subprocess
 import sys
+import logging
 import time
 import platform
 from urllib.parse import urlparse
@@ -27,6 +27,7 @@ from urllib.request import urlopen
 from urllib.request import Request
 from urllib.request import HTTPRedirectHandler
 from urllib.request import build_opener
+from http.client import IncompleteRead
 
 try:
     from . import __version__
@@ -41,14 +42,6 @@ def _get_log_date():
     return datetime.datetime.isoformat(datetime.datetime.now())
 
 
-def log_error(message):
-    """
-    Log message (str) or messages (List[str]) to stderr and exit with status 1
-    """
-    log_warning(message)
-    sys.exit(1)
-
-
 def log_info(message):
     """
     Log message (str) or messages (List[str]) to stdout
@@ -57,7 +50,7 @@ def log_info(message):
         message = [message]
 
     for msg in message:
-        sys.stdout.write("{0}: {1}\n".format(_get_log_date(), msg))
+        logging.info(msg)
 
 
 def log_warning(message):
@@ -68,7 +61,7 @@ def log_warning(message):
         message = [message]
 
     for msg in message:
-        sys.stderr.write("{0}: {1}\n".format(_get_log_date(), msg))
+        logging.warning(msg)
 
 
 def logging_subprocess(popenargs,
@@ -140,7 +133,7 @@ def mask_password(url, secret='*****'):
     return url.replace(parsed.password, secret)
 
 
-def parse_args():
+def parse_args(args=None):
     parser = argparse.ArgumentParser(description='Backup a github account')
     parser.add_argument('user',
                         metavar='USER',
@@ -331,7 +324,7 @@ def parse_args():
                         type=float,
                         default=30.0,
                         help='wait this amount of seconds when API request throttling is active (default: 30.0, requires --throttle-limit to be set)')
-    return parser.parse_args()
+    return parser.parse_args(args)
 
 
 def get_auth(args, encode=True, for_git_cli=False):
@@ -339,10 +332,10 @@ def get_auth(args, encode=True, for_git_cli=False):
 
     if args.osx_keychain_item_name:
         if not args.osx_keychain_item_account:
-            log_error('You must specify both name and account fields for osx keychain password items')
+            raise Exception('You must specify both name and account fields for osx keychain password items')
         else:
             if platform.system() != 'Darwin':
-                log_error("Keychain arguments are only supported on Mac OSX")
+                raise Exception("Keychain arguments are only supported on Mac OSX")
             try:
                 with open(os.devnull, 'w') as devnull:
                     token = (subprocess.check_output([
@@ -353,9 +346,9 @@ def get_auth(args, encode=True, for_git_cli=False):
                 token = token.decode('utf-8')
                 auth = token + ':' + 'x-oauth-basic'
             except subprocess.SubprocessError:
-                log_error('No password item matching the provided name and account could be found in the osx keychain.')
+                raise Exception('No password item matching the provided name and account could be found in the osx keychain.')
     elif args.osx_keychain_item_account:
-        log_error('You must specify both name and account fields for osx keychain password items')
+        raise Exception('You must specify both name and account fields for osx keychain password items')
     elif args.token:
         _path_specifier = 'file://'
         if args.token.startswith(_path_specifier):
@@ -377,7 +370,7 @@ def get_auth(args, encode=True, for_git_cli=False):
             password = urlquote(args.password)
         auth = args.username + ':' + password
     elif args.password:
-        log_error('You must specify a username for basic auth')
+        raise Exception('You must specify a username for basic auth')
 
     if not auth:
         return None
@@ -444,6 +437,21 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False):
         r, errors = _get_response(request, auth, template)
 
         status_code = int(r.getcode())
+        # Check if we got correct data
+        try:
+            response = json.loads(r.read().decode('utf-8'))
+        except IncompleteRead:
+            log_warning("Incomplete read error detected")
+            read_error = True
+        except json.decoder.JSONDecodeError:
+            log_warning("JSON decode error detected")
+            read_error = True
+        except TimeoutError:
+            log_warning("Tiemout error detected")
+            read_error = True
+        else:
+            read_error = False
+
         # be gentle with API request limit and throttle requests if remaining requests getting low
         limit_remaining = int(r.headers.get('x-ratelimit-remaining', 0))
         if args.throttle_limit and limit_remaining <= args.throttle_limit:
@@ -454,21 +462,37 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False):
             time.sleep(args.throttle_pause)
 
         retries = 0
-        while retries < 3 and status_code == 502:
-            log_warning('API request returned HTTP 502: Bad Gateway. Retrying in 5 seconds')
+        while retries < 3 and (status_code == 502 or read_error):
+            log_warning('API request failed. Retrying in 5 seconds')
             retries += 1
             time.sleep(5)
             request = _construct_request(per_page, page, query_args, template, auth, as_app=args.as_app)  # noqa
             r, errors = _get_response(request, auth, template)
 
             status_code = int(r.getcode())
+            try:
+                response = json.loads(r.read().decode('utf-8'))
+                read_error = False
+            except IncompleteRead:
+                log_warning("Incomplete read error detected")
+                read_error = True
+            except json.decoder.JSONDecodeError:
+                log_warning("JSON decode error detected")
+                read_error = True
+            except TimeoutError:
+                log_warning("Tiemout error detected")
+                read_error = True
 
         if status_code != 200:
             template = 'API request returned HTTP {0}: {1}'
             errors.append(template.format(status_code, r.reason))
-            log_error(errors)
+            raise Exception(', '.join(errors))
+
+        if read_error:
+            template = 'API request problem reading response for {0}'
+            errors.append(template.format(request))
+            raise Exception(', '.join(errors))
 
-        response = json.loads(r.read().decode('utf-8'))
         if len(errors) == 0:
             if type(response) == list:
                 for resp in response:
@@ -479,7 +503,7 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False):
                 yield response
 
         if len(errors) > 0:
-            log_error(errors)
+            raise Exception(', '.join(errors))
 
         if single_request:
             break
@@ -582,7 +606,7 @@ def _request_url_error(template, retry_timeout):
     if retry_timeout >= 0:
         return True
 
-    log_error('{} timed out to much, skipping!')
+    raise Exception('{} timed out to much, skipping!')
     return False
 
 
@@ -640,7 +664,7 @@ def get_authenticated_user(args):
 def check_git_lfs_install():
     exit_code = subprocess.call(['git', 'lfs', 'version'])
     if exit_code != 0:
-        log_error('The argument --lfs requires you to have Git LFS installed.\nYou can get it from https://git-lfs.github.com.')
+        raise Exception('The argument --lfs requires you to have Git LFS installed.\nYou can get it from https://git-lfs.github.com.')
 
 
 def retrieve_repositories(args, authenticated_user):
@@ -1009,7 +1033,8 @@ def backup_releases(args, repo_cwd, repository, repos_template, include_assets=F
     log_info('Saving {0} releases to disk'.format(len(releases)))
     for release in releases:
         release_name = release['tag_name']
-        output_filepath = os.path.join(release_cwd, '{0}.json'.format(release_name))
+        release_name_safe = release_name.replace('/', '__')
+        output_filepath = os.path.join(release_cwd, '{0}.json'.format(release_name_safe))
         with codecs.open(output_filepath, 'w+', encoding='utf-8') as f:
             json_dump(release, f)
 
@@ -1017,7 +1042,7 @@ def backup_releases(args, repo_cwd, repository, repos_template, include_assets=F
             assets = retrieve_data(args, release['assets_url'])
             if len(assets) > 0:
                 # give release asset files somewhere to live & download them (not including source archives)
-                release_assets_cwd = os.path.join(release_cwd, release_name)
+                release_assets_cwd = os.path.join(release_cwd, release_name_safe)
                 mkdir_p(release_assets_cwd)
                 for asset in assets:
                     download_file(asset['url'], os.path.join(release_assets_cwd, asset['name']), get_auth(args))
@@ -1081,10 +1106,11 @@ def fetch_repository(name,
             masked_remote_url,
             local_dir))
         if bare_clone:
-            if lfs_clone:
-                git_command = ['git', 'lfs', 'clone', '--mirror', remote_url, local_dir]
-            else:
             git_command = ['git', 'clone', '--mirror', remote_url, local_dir]
+            logging_subprocess(git_command, None)
+            if lfs_clone:
+                git_command = ['git', 'lfs', 'fetch', '--all', '--prune']
+                logging_subprocess(git_command, None, cwd=local_dir)
         else:
             if lfs_clone:
                 git_command = ['git', 'lfs', 'clone', remote_url, local_dir]