Release version 0.43.0

Add --log-level command line argument

.circleci/config.yml

@@ -0,0 +1,23 @@
+version: 2.1
+
+orbs:
+  python: circleci/python@0.3.2
+
+jobs:
+  build-and-test:
+    executor: python/default
+    steps:
+      - checkout
+      - python/load-cache
+      - run:
+          command: pip install flake8
+          name: Install dependencies
+      - python/save-cache
+      - run:
+          command: flake8 --ignore=E501
+          name: Lint
+
+workflows:
+  main:
+    jobs:
+      - build-and-test

.gitchangelog.rc

@@ -0,0 +1,117 @@
+#
+# Format
+#
+#   ACTION: [AUDIENCE:] COMMIT_MSG [@TAG ...]
+#
+# Description
+#
+#   ACTION is one of 'chg', 'fix', 'new'
+#
+#       Is WHAT the change is about.
+#
+#       'chg' is for refactor, small improvement, cosmetic changes...
+#       'fix' is for bug fixes
+#       'new' is for new features, big improvement
+#
+#   SUBJECT is optional and one of 'dev', 'usr', 'pkg', 'test', 'doc'
+#
+#       Is WHO is concerned by the change.
+#
+#       'dev'  is for developpers (API changes, refactors...)
+#       'usr'  is for final users (UI changes)
+#       'pkg'  is for packagers   (packaging changes)
+#       'test' is for testers     (test only related changes)
+#       'doc'  is for doc guys    (doc only changes)
+#
+#   COMMIT_MSG is ... well ... the commit message itself.
+#
+#   TAGs are additionnal adjective as 'refactor' 'minor' 'cosmetic'
+#
+#       'refactor' is obviously for refactoring code only
+#       'minor' is for a very meaningless change (a typo, adding a comment)
+#       'cosmetic' is for cosmetic driven change (re-indentation, 80-col...)
+#
+# Example:
+#
+#   new: usr: support of bazaar implemented
+#   chg: re-indentend some lines @cosmetic
+#   new: dev: updated code to be compatible with last version of killer lib.
+#   fix: pkg: updated year of licence coverage.
+#   new: test: added a bunch of test around user usability of feature X.
+#   fix: typo in spelling my name in comment. @minor
+#
+#   Please note that multi-line commit message are supported, and only the
+#   first line will be considered as the "summary" of the commit message. So
+#   tags, and other rules only applies to the summary.  The body of the commit
+#   message will be displayed in the changelog with minor reformating.
+
+#
+# ``ignore_regexps`` is a line of regexps
+#
+# Any commit having its full commit message matching any regexp listed here
+# will be ignored and won't be reported in the changelog.
+#
+ignore_regexps = [
+    r'(?i)^(Merge pull request|Merge branch|Release|Update)',
+]
+
+
+#
+# ``replace_regexps`` is a dict associating a regexp pattern and its replacement
+#
+# It will be applied to get the summary line from the full commit message.
+#
+# Note that you can provide multiple replacement patterns, they will be all
+# tried. If None matches, the summary line will be the full commit message.
+#
+replace_regexps = {
+    # current format (ie: 'chg: dev: my commit msg @tag1 @tag2')
+
+    r'^([cC]hg|[fF]ix|[nN]ew)\s*:\s*((dev|use?r|pkg|test|doc)\s*:\s*)?([^\n@]*)(@[a-z]+\s+)*$':
+    r'\4',
+}
+
+
+# ``section_regexps`` is a list of 2-tuples associating a string label and a
+# list of regexp
+#
+# Commit messages will be classified in sections thanks to this. Section
+# titles are the label, and a commit is classified under this section if any
+# of the regexps associated is matching.
+#
+section_regexps = [
+    ('New', [
+        r'^[nN]ew\s*:\s*((dev|use?r|pkg|test|doc)\s*:\s*)?([^\n]*)$',
+    ]),
+    ('Changes', [
+        r'^[cC]hg\s*:\s*((dev|use?r|pkg|test|doc)\s*:\s*)?([^\n]*)$',
+    ]),
+    ('Fix', [
+        r'^[fF]ix\s*:\s*((dev|use?r|pkg|test|doc)\s*:\s*)?([^\n]*)$',
+    ]),
+    ('Other', None  # Match all lines
+     ),
+
+]
+
+# ``body_split_regexp`` is a regexp
+#
+# Commit message body (not the summary) if existing will be split
+# (new line) on this regexp
+#
+body_split_regexp = r'[\n-]'
+
+
+# ``tag_filter_regexp`` is a regexp
+#
+# Tags that will be used for the changelog must match this regexp.
+#
+# tag_filter_regexp = r'^[0-9]+$'
+tag_filter_regexp = r'^(?:[vV])?[0-9\.]+$'
+
+
+# ``unreleased_version_label`` is a string
+#
+# This label will be used as the changelog Title of the last set of changes
+# between last valid tag and HEAD if any.
+unreleased_version_label = "%%version%% (unreleased)"

.github/workflows/tagged-release.yml

@@ -0,0 +1,19 @@
+---
+name: "tagged-release"
+
+# yamllint disable-line rule:truthy
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  tagged-release:
+    name: tagged-release
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: "marvinpinto/action-automatic-releases@v1.2.1"
+        with:
+          repo_token: "${{ secrets.GITHUB_TOKEN }}"
+          prerelease: false

.gitignore

@@ -34,3 +34,4 @@ doc/aws_hostname.1
 .vscode
 .atom
 
+README

CHANGES.rst

@@ -1,9 +1,184 @@
 Changelog
 =========
 
-0.32.0 (2020-04-13)
+0.43.0 (2023-05-29)
 -------------------
 ------------------------
+
+Fix
+~~~
+- Do not update readme. [Jose Diaz-Gonzalez]
+- Adjust for black. [Jose Diaz-Gonzalez]
+- Adjust for black. [Jose Diaz-Gonzalez]
+- Adjust for black. [Jose Diaz-Gonzalez]
+
+Other
+~~~~~
+- Feat: commit gitchangelog.rc to repo so anyone can generate a
+  changelog. [Jose Diaz-Gonzalez]
+- Feat: add release tagging. [Jose Diaz-Gonzalez]
+- Chore: formatting. [Jose Diaz-Gonzalez]
+- Chore: run black. [Jose Diaz-Gonzalez]
+- Add --log-level command line argument. [Enrico Tröger]
+
+  Support changing the log level to the desired value easily.
+  For example, this is useful to suppress progress messages but
+  keep logging warnings and errors.
+- Check both updated_at and pushed_at properties. [Ken Bailey]
+
+  Check both updated_at and pushed_at dates to get the last_update to reduce data retrieved on incremental api calls using since.
+
+
+0.42.0 (2022-11-28)
+-------------------
+- Add option to exclude repositories. [npounder]
+- Backup regular pull request comments as well. [Oneric]
+
+  Before, only review comments were backed up;
+  regular comments need to be fetched via issue API.
+- Fix bug forever retry when request url error. [kornpisey]
+- Added --no-prune option to disable prune option when doing git fetch.
+  [kornpisey]
+
+
+0.41.0 (2022-03-02)
+-------------------
+- Git lfs clone doe snot respect --mirror. [Louis Parisot]
+
+
+0.40.2 (2021-12-29)
+-------------------
+- Fix lint issues raised by Flake8. [atinary-afoulon]
+
+  According to job:
+  [ https://app.circleci.com/pipelines/github/josegonzalez/python-github-backup/30/workflows/74eb93f2-2505-435d-b728-03b3cc04c14a/jobs/23 ]
+
+  Failed on the following checks:
+  ./github_backup/github_backup.py:20:1: F811 redefinition of unused 'logging' from line 14
+  ./github_backup/github_backup.py:45:1: E302 expected 2 blank lines, found 1
+  ./github_backup/github_backup.py:136:20: E251 unexpected spaces around keyword / parameter equals
+
+
+0.40.1 (2021-09-22)
+-------------------
+- Revert to fetch. [Harrison Wright]
+
+
+0.40.0 (2021-07-12)
+-------------------
+- Add retry on certain network errors. [Jacek Nykis]
+
+  This change includes certain network level errors in the retry logic.
+  It partially address #110 but I think more comprehensive fix would be useful.
+- Pull changes from remote. [Jonas]
+
+  use `git pull` to pull actual files from the remote instead of using `fetch` for only the metadata
+
+
+0.39.0 (2021-03-19)
+-------------------
+
+Fix
+~~~
+- Fix missing INFO logs. [Gallo Feliz]
+
+Other
+~~~~~
+- Try to make compatible code with direct Python call ; reduce the hard
+  link of the code with the cli. [Gallo Feliz]
+- Fixed release_name with slash bug. [Álvaro Torres Cogollo]
+
+
+0.38.0 (2021-02-13)
+-------------------
+
+Fix
+~~~
+- Always clone with OAuth token when provided. [Samantha Baldwin]
+
+  Github Enterprise servers with 'Anonymous Git read access' disabled
+  cause `git ls-remote` to fail (128) for a repo's `clone_url`. Using the
+  OAuth token when provided allows cloning private AND public repos when
+  Anonymous Git read access is disabled.
+
+Other
+~~~~~
+- Change broken link to a fork to a working link to upstream. [Rick van
+  Schijndel]
+
+
+0.37.2 (2021-01-02)
+-------------------
+
+Fix
+~~~
+- Use distutils.core on error. [Jose Diaz-Gonzalez]
+
+
+0.37.1 (2021-01-02)
+-------------------
+
+Fix
+~~~
+- Use twine for releases. [Jose Diaz-Gonzalez]
+
+  The old method of releasing to pypi broke for whatever reason and switching to a supported toolchain is easier than debugging the old one.
+
+  Additionally:
+
+  - Update gitchangelog
+  - Fix license entry
+  - Set long description type
+  - Gitignore the temporary readme file
+
+
+0.37.0 (2021-01-02)
+-------------------
+- Add support for python 3.7 and 3.8 in package classifiers. [Albert
+  Wang]
+- Remove support for python 2.7 in package classifiers. [Albert Wang]
+- Remove python 2 specific import logic. [Albert Wang]
+- Remove python 2 specific logic. [Albert Wang]
+- Add ability to skip archived repositories. [Gary Moon]
+
+
+0.36.0 (2020-08-29)
+-------------------
+- Add flake8 instructions to readme. [Albert Wang]
+- Fix regex string. [Albert Wang]
+- Fix whitespace issues. [Albert Wang]
+- Do not use bare excepts. [Albert Wang]
+- Add .circleci/config.yml. [Albert Wang]
+- Include --private flag in example. [wouter bolsterlee]
+
+  By default, private repositories are not included. This is surprising.
+  It took me a while to figure this out, and making that clear in the
+  example can help others to be aware of that.
+
+
+0.35.0 (2020-08-05)
+-------------------
+- Make API request throttling optional. [Samantha Baldwin]
+
+
+0.34.0 (2020-07-24)
+-------------------
+- Add logic for transforming gist repository urls to ssh. [Matt Fields]
+
+
+0.33.0 (2020-04-13)
+-------------------
+- Add basic API request throttling. [Enrico Tröger]
+
+  A simple approach to throttle API requests and so keep within the rate
+  limits of the API. Can be enabled with "--throttle-limit" to specify
+  when throttling should start.
+  "--throttle-pause" defines the time to sleep between further API
+  requests.
+
+
+0.32.0 (2020-04-13)
+-------------------
 - Add timestamp to log messages. [Enrico Tröger]
 
 

README.rst

@@ -30,18 +30,19 @@ Usage
 CLI Usage is as follows::
 
     github-backup [-h] [-u USERNAME] [-p PASSWORD] [-t TOKEN] [--as-app]
-                  [-o OUTPUT_DIRECTORY] [-i] [--starred] [--all-starred]
-                  [--watched] [--followers] [--following] [--all]
-                  [--issues] [--issue-comments] [--issue-events] [--pulls]
-                  [--pull-comments] [--pull-commits] [--pull-details]
-                  [--labels] [--hooks] [--milestones] [--repositories]
-                  [--bare] [--lfs] [--wikis] [--gists] [--starred-gists]
-                  [--skip-existing] [-L [LANGUAGES [LANGUAGES ...]]]
-                  [-N NAME_REGEX] [-H GITHUB_HOST] [-O] [-R REPOSITORY]
-                  [-P] [-F] [--prefer-ssh] [-v]
+                  [-o OUTPUT_DIRECTORY] [-l LOG_LEVEL] [-i] [--starred]
+                  [--all-starred] [--watched] [--followers] [--following]
+                  [--all] [--issues] [--issue-comments] [--issue-events]
+                  [--pulls] [--pull-comments] [--pull-commits]
+                  [--pull-details] [--labels] [--hooks] [--milestones]
+                  [--repositories] [--bare] [--lfs] [--wikis] [--gists]
+                  [--starred-gists] [--skip-archived] [--skip-existing]
+                  [-L [LANGUAGES ...]] [-N NAME_REGEX] [-H GITHUB_HOST]
+                  [-O] [-R REPOSITORY] [-P] [-F] [--prefer-ssh] [-v]
                   [--keychain-name OSX_KEYCHAIN_ITEM_NAME]
                   [--keychain-account OSX_KEYCHAIN_ITEM_ACCOUNT]
-                  [--releases] [--assets]
+                  [--releases] [--assets] [--exclude [REPOSITORY [REPOSITORY ...]]
+                  [--throttle-limit THROTTLE_LIMIT] [--throttle-pause THROTTLE_PAUSE]
                   USER
 
     Backup a github account
@@ -62,6 +63,9 @@ CLI Usage is as follows::
       --as-app              authenticate as github app instead of as a user.
       -o OUTPUT_DIRECTORY, --output-directory OUTPUT_DIRECTORY
                             directory at which to backup the repositories
+      -l LOG_LEVEL, --log-level LOG_LEVEL
+                            log level to use (default: info, possible levels:
+                            debug, info, warning, error, critical)
       -i, --incremental     incremental backup
       --starred             include JSON output of starred repositories in backup
       --all-starred         include starred repositories in backup [*]
@@ -111,6 +115,15 @@ CLI Usage is as follows::
                             binaries
       --assets              include assets alongside release information; only
                             applies if including releases
+      --exclude [REPOSITORY [REPOSITORY ...]]
+                            names of repositories to exclude from backup.
+      --throttle-limit THROTTLE_LIMIT
+                            start throttling of GitHub API requests after this
+                            amount of API requests remain
+      --throttle-pause THROTTLE_PAUSE
+                            wait this amount of seconds when API request
+                            throttling is active (default: 30.0, requires
+                            --throttle-limit to be set)
 
 
 The package can be used to backup an *entire* organization or repository, including issues and wikis in the most appropriate format (clones for wikis, json files for issues).
@@ -145,10 +158,10 @@ Instructions on how to do this can be found on https://git-lfs.github.com.
 Examples
 ========
 
-Backup all repositories::
+Backup all repositories, including private ones::
 
     export ACCESS_TOKEN=SOME-GITHUB-TOKEN
-    github-backup WhiteHouse --token $ACCESS_TOKEN --organization --output-directory /tmp/white-house --repositories
+    github-backup WhiteHouse --token $ACCESS_TOKEN --organization --output-directory /tmp/white-house --repositories --private
 
 Backup a single organization repository with everything else (wiki, pull requests, comments, issues etc)::
 
@@ -158,7 +171,16 @@ Backup a single organization repository with everything else (wiki, pull request
     # e.g. git@github.com:docker/cli.git
     github-backup $ORGANIZATION -P -t $ACCESS_TOKEN -o . --all -O -R $REPO
 
+Testing
+=======
+
+This project currently contains no unit tests.  To run linting::
+
+    pip install flake8
+    flake8 --ignore=E501
+
+
 .. |PyPI| image:: https://img.shields.io/pypi/v/github-backup.svg
    :target: https://pypi.python.org/pypi/github-backup/
 .. |Python Versions| image:: https://img.shields.io/pypi/pyversions/github-backup.svg
-   :target: https://github.com/albertyw/github-backup
+   :target: https://github.com/josegonzalez/python-github-backup

bin/github-backup

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-import os
+import os, sys, logging
 
 from github_backup.github_backup import (
     backup_account,
@@ -9,11 +9,17 @@ from github_backup.github_backup import (
     filter_repositories,
     get_authenticated_user,
     log_info,
+    log_warning,
     mkdir_p,
     parse_args,
     retrieve_repositories,
 )
 
+logging.basicConfig(
+    format='%(asctime)s.%(msecs)03d: %(message)s',
+    datefmt='%Y-%m-%dT%H:%M:%S',
+    level=logging.INFO
+)
 
 def main():
     args = parse_args()
@@ -26,6 +32,11 @@ def main():
     if args.lfs_clone:
         check_git_lfs_install()
 
+    if args.log_level:
+        log_level = logging.getLevelName(args.log_level.upper())
+        if isinstance(log_level, int):
+            logging.root.setLevel(log_level)
+
     if not args.as_app:
         log_info('Backing up user {0} to {1}'.format(args.user, output_directory))
         authenticated_user = get_authenticated_user(args)
@@ -39,4 +50,8 @@ def main():
 
 
 if __name__ == '__main__':
-    main()
+    try:
+        main()
+    except Exception as e:
+        log_warning(str(e))
+        sys.exit(1)

github_backup/__init__.py

@@ -1 +1 @@
-__version__ = '0.32.0'
+__version__ = "0.43.0"

release

@@ -6,7 +6,7 @@ if [[ ! -f setup.py ]]; then
   exit 1
 fi
 
-PACKAGE_NAME="$(cat setup.py | grep "name='" | head | cut -d "'" -f2)"
+PACKAGE_NAME="$(cat setup.py | grep 'name="' | head | cut -d '"' -f2)"
 INIT_PACKAGE_NAME="$(echo "${PACKAGE_NAME//-/_}")"
 PUBLIC="true"
 
@@ -22,7 +22,7 @@ CYAN="\033[0;36m"     # cyan
 pip install wheel > /dev/null
 
 command -v gitchangelog >/dev/null 2>&1 || {
-    echo -e "${RED}WARNING: Missing gitchangelog binary, please run: pip install gitchangelog==2.2.0${COLOR_OFF}\n"
+    echo -e "${RED}WARNING: Missing gitchangelog binary, please run: pip install gitchangelog==3.0.4${COLOR_OFF}\n"
     exit 1
 }
 
@@ -31,6 +31,11 @@ command -v rst-lint > /dev/null || {
     exit 1
 }
 
+command -v twine > /dev/null || {
+    echo -e "${RED}WARNING: Missing twine binary, please run: pip install twine==3.2.0${COLOR_OFF}\n"
+    exit 1
+}
+
 if [[ "$@" != "major" ]] && [[ "$@" != "minor" ]] && [[ "$@" != "patch" ]]; then
     echo -e "${RED}WARNING: Invalid release type, must specify 'major', 'minor', or 'patch'${COLOR_OFF}\n"
     exit 1
@@ -81,18 +86,12 @@ TMPFILE=$(mktemp /tmp/${tempfoo}.XXXXXX) || {
     exit 1
 }
 
-find_this="__version__ = '$current_version'"
-replace_with="__version__ = '$next_version'"
+find_this="__version__ = \"$current_version\""
+replace_with="__version__ = \"$next_version\""
 
 echo -e "${YELLOW}--->${COLOR_OFF} Updating ${INIT_PACKAGE_NAME}/__init__.py"
 sed "s/$find_this/$replace_with/" ${INIT_PACKAGE_NAME}/__init__.py > $TMPFILE && mv $TMPFILE ${INIT_PACKAGE_NAME}/__init__.py
 
-find_this="${PACKAGE_NAME}.git@$current_version"
-replace_with="${PACKAGE_NAME}.git@$next_version"
-
-echo -e "${YELLOW}--->${COLOR_OFF} Updating README.rst"
-sed "s/$find_this/$replace_with/" README.rst > $TMPFILE && mv $TMPFILE README.rst
-
 if [ -f docs/conf.py ]; then
     echo -e "${YELLOW}--->${COLOR_OFF} Updating docs"
     find_this="version = '${current_version}'"
@@ -125,7 +124,8 @@ git push -q origin master && git push -q --tags
 if [[ "$PUBLIC" == "true" ]]; then
     echo -e "${YELLOW}--->${COLOR_OFF} Creating python release"
     cp README.rst README
-    python setup.py sdist bdist_wheel upload > /dev/null
+    python setup.py sdist bdist_wheel > /dev/null
+    twine upload dist/*
     rm README
 fi
 

setup.py

@@ -5,6 +5,7 @@ from github_backup import __version__
 
 try:
     from setuptools import setup
+
     setup  # workaround for pyflakes issue #13
 except ImportError:
     from distutils.core import setup
@@ -15,6 +16,7 @@ except ImportError:
 # http://www.eby-sarna.com/pipermail/peak/2010-May/003357.html)
 try:
     import multiprocessing
+
     multiprocessing
 except ImportError:
     pass
@@ -25,24 +27,26 @@ def open_file(fname):
 
 
 setup(
-    name='github-backup',
+    name="github-backup",
     version=__version__,
-    author='Jose Diaz-Gonzalez',
-    author_email='github-backup@josediazgonzalez.com',
-    packages=['github_backup'],
-    scripts=['bin/github-backup'],
-    url='http://github.com/josegonzalez/python-github-backup',
-    license=open('LICENSE.txt').read(),
+    author="Jose Diaz-Gonzalez",
+    author_email="github-backup@josediazgonzalez.com",
+    packages=["github_backup"],
+    scripts=["bin/github-backup"],
+    url="http://github.com/josegonzalez/python-github-backup",
+    license="MIT",
     classifiers=[
-        'Development Status :: 5 - Production/Stable',
-        'Topic :: System :: Archiving :: Backup',
-        'License :: OSI Approved :: MIT License',
-        'Programming Language :: Python :: 2.7',
-        'Programming Language :: Python :: 3.5',
-        'Programming Language :: Python :: 3.6',
+        "Development Status :: 5 - Production/Stable",
+        "Topic :: System :: Archiving :: Backup",
+        "License :: OSI Approved :: MIT License",
+        "Programming Language :: Python :: 3.5",
+        "Programming Language :: Python :: 3.6",
+        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
     ],
-    description='backup a github user or organization',
-    long_description=open_file('README.rst').read(),
-    install_requires=open_file('requirements.txt').readlines(),
+    description="backup a github user or organization",
+    long_description=open_file("README.rst").read(),
+    long_description_content_type="text/x-rst",
+    install_requires=open_file("requirements.txt").readlines(),
     zip_safe=True,
 )