Release version 0.44.0

This is disabled on pypi.

.circleci/config.yml

@@ -0,0 +1,23 @@
+version: 2.1
+
+orbs:
+  python: circleci/python@0.3.2
+
+jobs:
+  build-and-test:
+    executor: python/default
+    steps:
+      - checkout
+      - python/load-cache
+      - run:
+          command: pip install flake8
+          name: Install dependencies
+      - python/save-cache
+      - run:
+          command: flake8 --ignore=E501
+          name: Lint
+
+workflows:
+  main:
+    jobs:
+      - build-and-test

.gitchangelog.rc

@@ -0,0 +1,117 @@
+#
+# Format
+#
+#   ACTION: [AUDIENCE:] COMMIT_MSG [@TAG ...]
+#
+# Description
+#
+#   ACTION is one of 'chg', 'fix', 'new'
+#
+#       Is WHAT the change is about.
+#
+#       'chg' is for refactor, small improvement, cosmetic changes...
+#       'fix' is for bug fixes
+#       'new' is for new features, big improvement
+#
+#   SUBJECT is optional and one of 'dev', 'usr', 'pkg', 'test', 'doc'
+#
+#       Is WHO is concerned by the change.
+#
+#       'dev'  is for developpers (API changes, refactors...)
+#       'usr'  is for final users (UI changes)
+#       'pkg'  is for packagers   (packaging changes)
+#       'test' is for testers     (test only related changes)
+#       'doc'  is for doc guys    (doc only changes)
+#
+#   COMMIT_MSG is ... well ... the commit message itself.
+#
+#   TAGs are additionnal adjective as 'refactor' 'minor' 'cosmetic'
+#
+#       'refactor' is obviously for refactoring code only
+#       'minor' is for a very meaningless change (a typo, adding a comment)
+#       'cosmetic' is for cosmetic driven change (re-indentation, 80-col...)
+#
+# Example:
+#
+#   new: usr: support of bazaar implemented
+#   chg: re-indentend some lines @cosmetic
+#   new: dev: updated code to be compatible with last version of killer lib.
+#   fix: pkg: updated year of licence coverage.
+#   new: test: added a bunch of test around user usability of feature X.
+#   fix: typo in spelling my name in comment. @minor
+#
+#   Please note that multi-line commit message are supported, and only the
+#   first line will be considered as the "summary" of the commit message. So
+#   tags, and other rules only applies to the summary.  The body of the commit
+#   message will be displayed in the changelog with minor reformating.
+
+#
+# ``ignore_regexps`` is a line of regexps
+#
+# Any commit having its full commit message matching any regexp listed here
+# will be ignored and won't be reported in the changelog.
+#
+ignore_regexps = [
+    r'(?i)^(Merge pull request|Merge branch|Release|Update)',
+]
+
+
+#
+# ``replace_regexps`` is a dict associating a regexp pattern and its replacement
+#
+# It will be applied to get the summary line from the full commit message.
+#
+# Note that you can provide multiple replacement patterns, they will be all
+# tried. If None matches, the summary line will be the full commit message.
+#
+replace_regexps = {
+    # current format (ie: 'chg: dev: my commit msg @tag1 @tag2')
+
+    r'^([cC]hg|[fF]ix|[nN]ew)\s*:\s*((dev|use?r|pkg|test|doc)\s*:\s*)?([^\n@]*)(@[a-z]+\s+)*$':
+    r'\4',
+}
+
+
+# ``section_regexps`` is a list of 2-tuples associating a string label and a
+# list of regexp
+#
+# Commit messages will be classified in sections thanks to this. Section
+# titles are the label, and a commit is classified under this section if any
+# of the regexps associated is matching.
+#
+section_regexps = [
+    ('New', [
+        r'^[nN]ew\s*:\s*((dev|use?r|pkg|test|doc)\s*:\s*)?([^\n]*)$',
+    ]),
+    ('Changes', [
+        r'^[cC]hg\s*:\s*((dev|use?r|pkg|test|doc)\s*:\s*)?([^\n]*)$',
+    ]),
+    ('Fix', [
+        r'^[fF]ix\s*:\s*((dev|use?r|pkg|test|doc)\s*:\s*)?([^\n]*)$',
+    ]),
+    ('Other', None  # Match all lines
+     ),
+
+]
+
+# ``body_split_regexp`` is a regexp
+#
+# Commit message body (not the summary) if existing will be split
+# (new line) on this regexp
+#
+body_split_regexp = r'[\n-]'
+
+
+# ``tag_filter_regexp`` is a regexp
+#
+# Tags that will be used for the changelog must match this regexp.
+#
+# tag_filter_regexp = r'^[0-9]+$'
+tag_filter_regexp = r'^(?:[vV])?[0-9\.]+$'
+
+
+# ``unreleased_version_label`` is a string
+#
+# This label will be used as the changelog Title of the last set of changes
+# between last valid tag and HEAD if any.
+unreleased_version_label = "%%version%% (unreleased)"

.github/workflows/automatic-release.yml

@@ -0,0 +1,40 @@
+name: automatic-release
+
+on:
+    workflow_dispatch:
+        inputs:
+            release_type:
+                description: Release type
+                required: true
+                type: choice
+                options:
+                    - patch
+                    - minor
+                    - major
+
+jobs:
+    release:
+        name: Release
+        runs-on: ubuntu-22.04
+        steps:
+            - name: Checkout repository
+              uses: actions/checkout@v4
+              with:
+                fetch-depth: 0
+            - name: Setup Git
+              run: |
+                git config --local user.email "action@github.com"
+                git config --local user.name "GitHub Action"
+            - name: Setup Python
+              uses: actions/setup-python@v4
+              with:
+                python-version: '3.8'
+            - name: Install prerequisites
+              run: pip install -r release-requirements.txt
+            - name: Execute release
+              env:
+                SEMVER_BUMP: ${{ github.event.inputs.release_type }}
+                TWINE_REPOSITORY: ${{ vars.TWINE_REPOSITORY }}
+                TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
+                TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
+              run: ./release $SEMVER_BUMP

.github/workflows/tagged-release.yml

@@ -0,0 +1,19 @@
+---
+name: "tagged-release"
+
+# yamllint disable-line rule:truthy
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  tagged-release:
+    name: tagged-release
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: "marvinpinto/action-automatic-releases@v1.2.1"
+        with:
+          repo_token: "${{ secrets.GITHUB_TOKEN }}"
+          prerelease: false

.gitignore

@@ -34,3 +34,4 @@ doc/aws_hostname.1
 .vscode
 .atom
 
+README

CHANGES.rst

@@ -1,9 +1,258 @@
 Changelog
 =========
 
-0.31.0 (2020-02-25)
+0.44.0 (2023-12-09)
 -------------------
 ------------------------
+
+Fix
+~~~
+- Do not use raw property in readme. [Jose Diaz-Gonzalez]
+
+  This is disabled on pypi.
+- Validate release before committing and uploading it. [Jose Diaz-
+  Gonzalez]
+- Correct lint issues and show errors on lint. [Jose Diaz-Gonzalez]
+- Minor cosmetic changes. [ZhymabekRoman]
+- Add forgotten variable formatting. [ZhymabekRoman]
+- Refactor logging Based on #195. [ZhymabekRoman]
+- Minor typo fix. [Zhymabek Roman]
+
+Other
+~~~~~
+- Bump certifi from 2023.5.7 to 2023.7.22. [dependabot[bot]]
+
+  Bumps [certifi](https://github.com/certifi/python-certifi) from 2023.5.7 to 2023.7.22.
+  - [Commits](https://github.com/certifi/python-certifi/compare/2023.05.07...2023.07.22)
+
+  ---
+  updated-dependencies:
+  - dependency-name: certifi
+    dependency-type: direct:production
+  ...
+- Checkout everything. [Halvor Holsten Strand]
+- Added automatic release workflow, for use with GitHub Actions. [Halvor
+  Holsten Strand]
+- Feat: create Dockerfile. [Tom Plant]
+- Fix rst html. [hozza]
+- Add contributor section. [hozza]
+- Fix readme wording and format. [hozza]
+- Fixed readme working and layout. [hozza]
+- Fix readme formatting, spelling and layout. [hozza]
+- Added details usage and examples including gotchas, errors and
+  development instructions. [hozza]
+- Added verbose install instructions. [hozza]
+- Bump urllib3 from 2.0.2 to 2.0.7. [dependabot[bot]]
+
+  Bumps [urllib3](https://github.com/urllib3/urllib3) from 2.0.2 to 2.0.7.
+  - [Release notes](https://github.com/urllib3/urllib3/releases)
+  - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst)
+  - [Commits](https://github.com/urllib3/urllib3/compare/2.0.2...2.0.7)
+
+  ---
+  updated-dependencies:
+  - dependency-name: urllib3
+    dependency-type: direct:production
+  ...
+- Suggested modification to fix win32 logging failure, due to local
+  variable scope. Logger does not appear to have any utility within
+  "logging_subprocess". [Halvor Holsten Strand]
+- Simplified one if/elif scenario. Extracted file reading of another
+  if/elif scenario. [Halvor Holsten Strand]
+- Ran black. [Halvor Holsten Strand]
+- Keep backwards compatability by going back to "--token" for classic.
+  Allow "file://" uri for "--token-fine". [Halvor Holsten Strand]
+- Add support for fine-grained tokens. [froggleston]
+- Refactor logging and add support for quiet flag. [Harrison Wright]
+
+
+0.43.1 (2023-05-29)
+-------------------
+- Chore: add release requirements. [Jose Diaz-Gonzalez]
+
+
+0.43.0 (2023-05-29)
+-------------------
+
+Fix
+~~~
+- Do not update readme. [Jose Diaz-Gonzalez]
+- Adjust for black. [Jose Diaz-Gonzalez]
+- Adjust for black. [Jose Diaz-Gonzalez]
+- Adjust for black. [Jose Diaz-Gonzalez]
+
+Other
+~~~~~
+- Feat: commit gitchangelog.rc to repo so anyone can generate a
+  changelog. [Jose Diaz-Gonzalez]
+- Feat: add release tagging. [Jose Diaz-Gonzalez]
+- Chore: formatting. [Jose Diaz-Gonzalez]
+- Chore: run black. [Jose Diaz-Gonzalez]
+- Add --log-level command line argument. [Enrico Tröger]
+
+  Support changing the log level to the desired value easily.
+  For example, this is useful to suppress progress messages but
+  keep logging warnings and errors.
+- Check both updated_at and pushed_at properties. [Ken Bailey]
+
+  Check both updated_at and pushed_at dates to get the last_update to reduce data retrieved on incremental api calls using since.
+
+
+0.42.0 (2022-11-28)
+-------------------
+- Add option to exclude repositories. [npounder]
+- Backup regular pull request comments as well. [Oneric]
+
+  Before, only review comments were backed up;
+  regular comments need to be fetched via issue API.
+- Fix bug forever retry when request url error. [kornpisey]
+- Added --no-prune option to disable prune option when doing git fetch.
+  [kornpisey]
+
+
+0.41.0 (2022-03-02)
+-------------------
+- Git lfs clone doe snot respect --mirror. [Louis Parisot]
+
+
+0.40.2 (2021-12-29)
+-------------------
+- Fix lint issues raised by Flake8. [atinary-afoulon]
+
+  According to job:
+  [ https://app.circleci.com/pipelines/github/josegonzalez/python-github-backup/30/workflows/74eb93f2-2505-435d-b728-03b3cc04c14a/jobs/23 ]
+
+  Failed on the following checks:
+  ./github_backup/github_backup.py:20:1: F811 redefinition of unused 'logging' from line 14
+  ./github_backup/github_backup.py:45:1: E302 expected 2 blank lines, found 1
+  ./github_backup/github_backup.py:136:20: E251 unexpected spaces around keyword / parameter equals
+
+
+0.40.1 (2021-09-22)
+-------------------
+- Revert to fetch. [Harrison Wright]
+
+
+0.40.0 (2021-07-12)
+-------------------
+- Add retry on certain network errors. [Jacek Nykis]
+
+  This change includes certain network level errors in the retry logic.
+  It partially address #110 but I think more comprehensive fix would be useful.
+- Pull changes from remote. [Jonas]
+
+  use `git pull` to pull actual files from the remote instead of using `fetch` for only the metadata
+
+
+0.39.0 (2021-03-19)
+-------------------
+
+Fix
+~~~
+- Fix missing INFO logs. [Gallo Feliz]
+
+Other
+~~~~~
+- Try to make compatible code with direct Python call ; reduce the hard
+  link of the code with the cli. [Gallo Feliz]
+- Fixed release_name with slash bug. [Álvaro Torres Cogollo]
+
+
+0.38.0 (2021-02-13)
+-------------------
+
+Fix
+~~~
+- Always clone with OAuth token when provided. [Samantha Baldwin]
+
+  Github Enterprise servers with 'Anonymous Git read access' disabled
+  cause `git ls-remote` to fail (128) for a repo's `clone_url`. Using the
+  OAuth token when provided allows cloning private AND public repos when
+  Anonymous Git read access is disabled.
+
+Other
+~~~~~
+- Change broken link to a fork to a working link to upstream. [Rick van
+  Schijndel]
+
+
+0.37.2 (2021-01-02)
+-------------------
+
+Fix
+~~~
+- Use distutils.core on error. [Jose Diaz-Gonzalez]
+
+
+0.37.1 (2021-01-02)
+-------------------
+
+Fix
+~~~
+- Use twine for releases. [Jose Diaz-Gonzalez]
+
+  The old method of releasing to pypi broke for whatever reason and switching to a supported toolchain is easier than debugging the old one.
+
+  Additionally:
+
+  - Update gitchangelog
+  - Fix license entry
+  - Set long description type
+  - Gitignore the temporary readme file
+
+
+0.37.0 (2021-01-02)
+-------------------
+- Add support for python 3.7 and 3.8 in package classifiers. [Albert
+  Wang]
+- Remove support for python 2.7 in package classifiers. [Albert Wang]
+- Remove python 2 specific import logic. [Albert Wang]
+- Remove python 2 specific logic. [Albert Wang]
+- Add ability to skip archived repositories. [Gary Moon]
+
+
+0.36.0 (2020-08-29)
+-------------------
+- Add flake8 instructions to readme. [Albert Wang]
+- Fix regex string. [Albert Wang]
+- Fix whitespace issues. [Albert Wang]
+- Do not use bare excepts. [Albert Wang]
+- Add .circleci/config.yml. [Albert Wang]
+- Include --private flag in example. [wouter bolsterlee]
+
+  By default, private repositories are not included. This is surprising.
+  It took me a while to figure this out, and making that clear in the
+  example can help others to be aware of that.
+
+
+0.35.0 (2020-08-05)
+-------------------
+- Make API request throttling optional. [Samantha Baldwin]
+
+
+0.34.0 (2020-07-24)
+-------------------
+- Add logic for transforming gist repository urls to ssh. [Matt Fields]
+
+
+0.33.0 (2020-04-13)
+-------------------
+- Add basic API request throttling. [Enrico Tröger]
+
+  A simple approach to throttle API requests and so keep within the rate
+  limits of the API. Can be enabled with "--throttle-limit" to specify
+  when throttling should start.
+  "--throttle-pause" defines the time to sleep between further API
+  requests.
+
+
+0.32.0 (2020-04-13)
+-------------------
+- Add timestamp to log messages. [Enrico Tröger]
+
+
+0.31.0 (2020-02-25)
+-------------------
 - #123 update: changed --as-app 'help' description. [ethan]
 - #123: Support Authenticating As Github Application. [ethan]
 

Dockerfile

@@ -0,0 +1,16 @@
+FROM python:3.9.18-slim
+
+RUN --mount=type=cache,target=/var/cache/apt \
+    apt-get update && apt-get install -y git git-lfs
+
+WORKDIR /usr/src/app
+
+COPY release-requirements.txt .
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install -r release-requirements.txt
+
+COPY . .
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install .
+
+ENTRYPOINT [ "github-backup" ]

README.rst

@@ -4,14 +4,13 @@ github-backup
 
 |PyPI| |Python Versions|
 
-    This project is considered feature complete for the primary maintainer. If you would like a bugfix or enhancement and cannot sponsor the work, pull requests are welcome. Feel free to contact the maintainer for consulting estimates if desired.
+The package can be used to backup an *entire* `Github <https://github.com/>`_ organization, repository or user account, including starred repos, issues and wikis in the most appropriate format (clones for wikis, json files for issues).
-
-backup a github user or organization
 
 Requirements
 ============
 
 - GIT 1.9+
+- Python
 
 Installation
 ============
@@ -20,28 +19,38 @@ Using PIP via PyPI::
 
     pip install github-backup
 
-Using PIP via Github::
+Using PIP via Github (more likely the latest version)::
 
     pip install git+https://github.com/josegonzalez/python-github-backup.git#egg=github-backup
     
-Usage
+*Install note for python newcomers:*
-=====
 
-CLI Usage is as follows::
+Python scripts are unlikely to be included in your ``$PATH`` by default, this means it cannot be run directly in terminal with ``$ github-backup ...``, you can either add python's install path to your environments ``$PATH`` or call the script directly e.g. using ``$ ~/.local/bin/github-backup``.*
 
-    github-backup [-h] [-u USERNAME] [-p PASSWORD] [-t TOKEN]
+Basic Help
-                  [-o OUTPUT_DIRECTORY] [-i] [--starred] [--all-starred]
+==========
-                  [--watched] [--followers] [--following] [--all]
+
-                  [--issues] [--issue-comments] [--issue-events] [--pulls]
+Show the CLI help output::
-                  [--pull-comments] [--pull-commits] [--labels] [--hooks]
+
-                  [--milestones] [--repositories] [--releases] [--assets]
+    github-backup -h
+
+CLI Help output::
+
+    github-backup [-h] [-u USERNAME] [-p PASSWORD] [-t TOKEN_CLASSIC]
+                  [-f TOKEN_FINE] [--as-app] [-o OUTPUT_DIRECTORY]
+                  [-l LOG_LEVEL] [-i] [--starred] [--all-starred]
+                  [--watched] [--followers] [--following] [--all] [--issues]
+                  [--issue-comments] [--issue-events] [--pulls]
+                  [--pull-comments] [--pull-commits] [--pull-details]
+                  [--labels] [--hooks] [--milestones] [--repositories]
                   [--bare] [--lfs] [--wikis] [--gists] [--starred-gists]
-                  [--skip-existing]
+                  [--skip-archived] [--skip-existing] [-L [LANGUAGES ...]]
-                  [-L [LANGUAGES [LANGUAGES ...]]] [-N NAME_REGEX]
+                  [-N NAME_REGEX] [-H GITHUB_HOST] [-O] [-R REPOSITORY]
-                  [-H GITHUB_HOST] [-O] [-R REPOSITORY] [-P] [-F]
+                  [-P] [-F] [--prefer-ssh] [-v]
-                  [--prefer-ssh] [-v]
                   [--keychain-name OSX_KEYCHAIN_ITEM_NAME]
                   [--keychain-account OSX_KEYCHAIN_ITEM_ACCOUNT]
+                  [--releases] [--assets] [--exclude [REPOSITORY [REPOSITORY ...]]
+                  [--throttle-limit THROTTLE_LIMIT] [--throttle-pause THROTTLE_PAUSE]
                   USER
 
     Backup a github account
@@ -56,37 +65,43 @@ CLI Usage is as follows::
       -p PASSWORD, --password PASSWORD
                             password for basic auth. If a username is given but
                             not a password, the password will be prompted for.
-      -t TOKEN, --token TOKEN
+      -f TOKEN_FINE, --token-fine TOKEN_FINE
-                            personal access or OAuth token, or path to token
+                            fine-grained personal access token or path to token
                             (file://...)
+      -t TOKEN_CLASSIC, --token TOKEN_CLASSIC
+                            personal access, OAuth, or JSON Web token, or path to
+                            token (file://...)
+      --as-app              authenticate as github app instead of as a user.
       -o OUTPUT_DIRECTORY, --output-directory OUTPUT_DIRECTORY
                             directory at which to backup the repositories
+      -l LOG_LEVEL, --log-level LOG_LEVEL
+                            log level to use (default: info, possible levels:
+                            debug, info, warning, error, critical)
       -i, --incremental     incremental backup
       --starred             include JSON output of starred repositories in backup
-      --all-starred         include starred repositories in backup
+      --all-starred         include starred repositories in backup [*]
-      --watched             include watched repositories in backup
+      --watched             include JSON output of watched repositories in backup
       --followers           include JSON output of followers in backup
       --following           include JSON output of following users in backup
-      --all                 include everything in backup
+      --all                 include everything in backup (not including [*])
       --issues              include issues in backup
       --issue-comments      include issue comments in backup
       --issue-events        include issue events in backup
       --pulls               include pull requests in backup
       --pull-comments       include pull request review comments in backup
       --pull-commits        include pull request commits in backup
+      --pull-details        include more pull request details in backup [*]
       --labels              include labels in backup
       --hooks               include hooks in backup (works only when
                             authenticated)
       --milestones          include milestones in backup
       --repositories        include repository clone in backup
-      --releases            include repository releases' information without assets or binaries
-      --assets              include assets alongside release information; only applies if including releases
       --bare                clone bare repositories
       --lfs                 clone LFS repositories (requires Git LFS to be
-                            installed, https://git-lfs.github.com)
+                            installed, https://git-lfs.github.com) [*]
       --wikis               include wiki clone in backup
-      --gists               include gists in backup
+      --gists               include gists in backup [*]
-      --starred-gists       include starred gists in backup
+      --starred-gists       include starred gists in backup [*]
       --skip-existing       skip project if a backup directory exists
       -L [LANGUAGES [LANGUAGES ...]], --languages [LANGUAGES [LANGUAGES ...]]
                             only allow these languages
@@ -97,8 +112,8 @@ CLI Usage is as follows::
       -O, --organization    whether or not this is an organization user
       -R REPOSITORY, --repository REPOSITORY
                             name of repository to limit backup to
-      -P, --private         include private repositories
+      -P, --private         include private repositories [*]
-      -F, --fork            include forked repositories
+      -F, --fork            include forked repositories [*]
       --prefer-ssh          Clone repositories using SSH instead of HTTPS
       -v, --version         show program's version number and exit
       --keychain-name OSX_KEYCHAIN_ITEM_NAME
@@ -107,17 +122,58 @@ CLI Usage is as follows::
       --keychain-account OSX_KEYCHAIN_ITEM_ACCOUNT
                             OSX ONLY: account field of password item in OSX
                             keychain that holds the personal access or OAuth token
+      --releases            include release information, not including assets or
+                            binaries
+      --assets              include assets alongside release information; only
+                            applies if including releases
+      --exclude [REPOSITORY [REPOSITORY ...]]
+                            names of repositories to exclude from backup.
+      --throttle-limit THROTTLE_LIMIT
+                            start throttling of GitHub API requests after this
+                            amount of API requests remain
+      --throttle-pause THROTTLE_PAUSE
+                            wait this amount of seconds when API request
+                            throttling is active (default: 30.0, requires
+                            --throttle-limit to be set)
 
 
-The package can be used to backup an *entire* organization or repository, including issues and wikis in the most appropriate format (clones for wikis, json files for issues).
+Usage Details
+=============
 
 Authentication
-==============
+--------------
+
+**Password-based authentication** will fail if you have two-factor authentication enabled, and will `be deprecated <https://github.blog/2023-03-09-raising-the-bar-for-software-security-github-2fa-begins-march-13/>`_ by 2023 EOY.
+
+``--username`` is used for basic password authentication and separate from the positional argument ``USER``, which specifies the user account you wish to back up.
+
+**Classic tokens** are `slightly less secure <https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#personal-access-tokens-classic>`_ as they provide very coarse-grained permissions.
+
+If you need authentication for long-running backups (e.g. for a cron job) it is recommended to use **fine-grained personal access token** ``-f TOKEN_FINE``.
+
+
+Fine Tokens
+~~~~~~~~~~~
+
+You can "generate new token", choosing the repository scope by selecting specific repos or all repos. On Github this is under *Settings -> Developer Settings -> Personal access tokens -> Fine-grained Tokens*
+
+Customise the permissions for your use case, but for a personal account full backup you'll need to enable the following permissions:
+
+**User permissions**: Read access to followers, starring, and watching.
+
+**Repository permissions**: Read access to code, commit statuses, issues, metadata, pages, pull requests, and repository hooks.
+
+
+Prefer SSH
+~~~~~~~~~~
+
+If cloning repos is enabled with ``--repositories``, ``--all-starred``, ``--wikis``, ``--gists``, ``--starred-gists`` using the ``--prefer-ssh`` argument will use ssh for cloning the git repos, but all other connections will still use their own protocol, e.g. API requests for issues uses HTTPS.
+
+To clone with SSH, you'll need SSH authentication setup `as usual with Github <https://docs.github.com/en/authentication/connecting-to-github-with-ssh>`_, e.g. via SSH public and private keys.
 
-Note: Password-based authentication will fail if you have two-factor authentication enabled.
 
 Using the Keychain on Mac OSX
-=============================
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Note: On Mac OSX the token can be stored securely in the user's keychain. To do this:
 
 1. Open Keychain from "Applications -> Utilities -> Keychain Access"
@@ -131,30 +187,143 @@ Note:  When you run github-backup, you will be asked whether you want to allow "
 1. **Allow:** In this case you will need to click "Allow" each time you run `github-backup`
 2. **Always Allow:** In this case, you will not be asked for permission when you run `github-backup` in future. This is less secure, but is required if you want to schedule `github-backup` to run automatically
 
-About Git LFS
-=============
 
-When you use the "--lfs" option, you will need to make sure you have Git LFS installed.
+Github Rate-limit and Throttling
+--------------------------------
+
+"github-backup" will automatically throttle itself based on feedback from the Github API. 
+
+Their API is usually rate-limited to 5000 calls per hour. The API will ask github-backup to pause until a specific time when the limit is reset again (at the start of the next hour). This continues until the backup is complete.
+
+During a large backup, such as ``--all-starred``, and on a fast connection this can result in (~20 min) pauses with bursts of API calls periodically maxing out the API limit. If this is not suitable `it has been observed <https://github.com/josegonzalez/python-github-backup/issues/76#issuecomment-636158717>`_ under real-world conditions that overriding the throttle with ``--throttle-limit 5000 --throttle-pause 0.6`` provides a smooth rate across the hour, although a ``--throttle-pause 0.72`` (3600 seconds [1 hour] / 5000 limit) is theoretically safer to prevent large rate-limit pauses.
+
+
+About Git LFS
+-------------
+
+When you use the ``--lfs`` option, you will need to make sure you have Git LFS installed.
 
 Instructions on how to do this can be found on https://git-lfs.github.com.
 
-Examples
-========
 
-Backup all repositories::
+Gotchas / Known-issues
+======================
+
+All is not everything
+---------------------
+
+The ``--all`` argument does not include; cloning private repos (``-P, --private``), cloning forks (``-F, --fork``) cloning starred repositories (``--all-starred``), ``--pull-details``, cloning LFS repositories (``--lfs``), cloning gists (``--starred-gists``) or cloning starred gist repos (``--starred-gists``). See examples for more.
+
+Cloning all starred size
+------------------------
+
+Using the ``--all-starred`` argument to clone all starred repositories may use a large amount of storage space, especially if ``--all`` or more arguments are used. e.g. commonly starred repos can have tens of thousands of issues, many large assets and the repo itself etc. Consider just storing links to starred repos in JSON format with ``--starred``.
+
+Incremental Backup
+------------------
+
+Using (``-i, --incremental``) will only request new data from the API **since the last run (successful or not)**. e.g. only request issues from the API since the last run. 
+
+This means any blocking errors on previous runs can cause a large amount of missing data in backups.
+
+Known blocking errors
+---------------------
+
+Some errors will block the backup run by exiting the script. e.g. receiving a 403 Forbidden error from the Github API.
+
+If the incremental argument is used, this will result in the next backup only requesting API data since the last blocked/failed run. Potentially causing unexpected large amounts of missing data.
+
+It's therefore recommended to only use the incremental argument if the output/result is being actively monitored, or complimented with periodic full non-incremental runs, to avoid unexpected missing data in a regular backup runs.
+
+1. **Starred public repo hooks blocking**
+
+   Since the ``--all`` argument includes ``--hooks``, if you use ``--all`` and ``--all-starred`` together to clone a users starred public repositories, the backup will likely error and block the backup continuing. 
+
+   This is due to needing the correct permission for ``--hooks`` on public repos.
+
+2. **Releases blocking**
+
+   A known ``--releases`` (required for ``--assets``) error will sometimes block the backup. 
+
+   If you're backing up a lot of repositories with releases e.g. an organisation or ``--all-starred``. You may need to remove ``--releases`` (and therefore ``--assets``) to complete a backup. Documented in `issue 209 <https://github.com/josegonzalez/python-github-backup/issues/209>`_.
+
+
+"bare" is actually "mirror"
+---------------------------
+
+Using the bare clone argument (``--bare``) will actually call git's ``clone --mirror`` command. There's a subtle difference between `bare <https://www.git-scm.com/docs/git-clone#Documentation/git-clone.txt---bare>`_ and `mirror <https://www.git-scm.com/docs/git-clone#Documentation/git-clone.txt---mirror>`_ clone.
+
+*From git docs "Compared to --bare, --mirror not only maps local branches of the source to local branches of the target, it maps all refs (including remote-tracking branches, notes etc.) and sets up a refspec configuration such that all these refs are overwritten by a git remote update in the target repository."*
+
+
+Starred gists vs starred repo behaviour
+---------------------------------------
+
+The starred normal repo cloning (``--all-starred``) argument stores starred repos separately to the users own repositories. However, using ``--starred-gists`` will store starred gists within the same directory as the users own gists ``--gists``. Also, all gist repo directory names are IDs not the gist's name.
+
+
+Skip existing on incomplete backups
+-----------------------------------
+
+The ``--skip-existing`` argument will skip a backup if the directory already exists, even if the backup in that directory failed (perhaps due to a blocking error). This may result in unexpected missing data in a regular backup.
+
+
+Github Backup Examples
+======================
+
+Backup all repositories, including private ones using a classic token::
 
     export ACCESS_TOKEN=SOME-GITHUB-TOKEN
-    github-backup WhiteHouse --token $ACCESS_TOKEN --organization --output-directory /tmp/white-house --repositories
+    github-backup WhiteHouse --token $ACCESS_TOKEN --organization --output-directory /tmp/white-house --repositories --private
 
-Backup a single organization repository with everything else (wiki, pull requests, comments, issues etc)::
+Use a fine-grained access token to backup a single organization repository with everything else (wiki, pull requests, comments, issues etc)::
 
-    export ACCESS_TOKEN=SOME-GITHUB-TOKEN
+    export FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
     ORGANIZATION=docker
     REPO=cli
     # e.g. git@github.com:docker/cli.git
-    github-backup $ORGANIZATION -P -t $ACCESS_TOKEN -o . --all -O -R $REPO
+    github-backup $ORGANIZATION -P -f $FINE_ACCESS_TOKEN -o . --all -O -R $REPO
+
+Quietly and incrementally backup useful Github user data (public and private repos with SSH) including; all issues, pulls, all public starred repos and gists (omitting "hooks", "releases" and therefore "assets" to prevent blocking). *Great for a cron job.* ::
+
+    export FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
+    GH_USER=YOUR-GITHUB-USER
+
+    github-backup -f $FINE_ACCESS_TOKEN --prefer-ssh -o ~/github-backup/ -l error -P -i --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-commits --labels --milestones --repositories --wikis --releases --assets --pull-details --gists --starred-gists $GH_USER
+    
+Debug an error/block or incomplete backup into a temporary directory. Omit "incremental" to fill a previous incomplete backup. ::
+
+    export FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
+    GH_USER=YOUR-GITHUB-USER
+
+    github-backup -f $FINE_ACCESS_TOKEN -o /tmp/github-backup/ -l debug -P --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-commits --labels --milestones --repositories --wikis --releases --assets --pull-details --gists --starred-gists $GH_USER
+
+
+
+Development
+===========
+
+This project is considered feature complete for the primary maintainer @josegonzalez. If you would like a bugfix or enhancement, pull requests are welcome. Feel free to contact the maintainer for consulting estimates if you'd like to sponsor the work instead.
+
+Contibuters
+-----------
+
+A huge thanks to all the contibuters!
+
+.. image:: https://contrib.rocks/image?repo=josegonzalez/python-github-backup
+   :target: https://github.com/josegonzalez/python-github-backup/graphs/contributors
+   :alt: contributors
+
+Testing
+-------
+
+This project currently contains no unit tests.  To run linting::
+
+    pip install flake8
+    flake8 --ignore=E501
+
 
 .. |PyPI| image:: https://img.shields.io/pypi/v/github-backup.svg
    :target: https://pypi.python.org/pypi/github-backup/
 .. |Python Versions| image:: https://img.shields.io/pypi/pyversions/github-backup.svg
-   :target: https://github.com/albertyw/github-backup
+   :target: https://github.com/josegonzalez/python-github-backup

bin/github-backup

@@ -1,6 +1,12 @@
 #!/usr/bin/env python
 
-import os
+import os, sys, logging
+
+logging.basicConfig(
+    format='%(asctime)s.%(msecs)03d: %(message)s',
+    datefmt='%Y-%m-%dT%H:%M:%S',
+    level=logging.INFO
+)
 
 from github_backup.github_backup import (
     backup_account,
@@ -8,7 +14,7 @@ from github_backup.github_backup import (
     check_git_lfs_install,
     filter_repositories,
     get_authenticated_user,
-    log_info,
+    logger,
     mkdir_p,
     parse_args,
     retrieve_repositories,
@@ -18,16 +24,24 @@ from github_backup.github_backup import (
 def main():
     args = parse_args()
 
+    if args.quiet:
+        logger.setLevel(logging.WARNING)
+
     output_directory = os.path.realpath(args.output_directory)
     if not os.path.isdir(output_directory):
-        log_info('Create output directory {0}'.format(output_directory))
+        logger.info('Create output directory {0}'.format(output_directory))
         mkdir_p(output_directory)
 
     if args.lfs_clone:
         check_git_lfs_install()
 
+    if args.log_level:
+        log_level = logging.getLevelName(args.log_level.upper())
+        if isinstance(log_level, int):
+            logger.root.setLevel(log_level)
+
     if not args.as_app:
-        log_info('Backing up user {0} to {1}'.format(args.user, output_directory))
+        logger.info('Backing up user {0} to {1}'.format(args.user, output_directory))
         authenticated_user = get_authenticated_user(args)
     else:
         authenticated_user = {'login': None}
@@ -39,4 +53,8 @@ def main():
 
 
 if __name__ == '__main__':
+    try:
         main()
+    except Exception as e:
+        logger.error(str(e))
+        sys.exit(1)

github_backup/__init__.py

@@ -1 +1 @@
-__version__ = '0.31.0'
+__version__ = "0.44.0"

release

@@ -1,12 +1,13 @@
 #!/usr/bin/env bash
-set -eo pipefail; [[ $RELEASE_TRACE ]] && set -x
+set -eo pipefail
+[[ $RELEASE_TRACE ]] && set -x
 
 if [[ ! -f setup.py ]]; then
     echo -e "${RED}WARNING: Missing setup.py${COLOR_OFF}\n"
     exit 1
 fi
 
-PACKAGE_NAME="$(cat setup.py | grep "name='" | head | cut -d "'" -f2)"
+PACKAGE_NAME="$(cat setup.py | grep 'name="' | head | cut -d '"' -f2)"
 INIT_PACKAGE_NAME="$(echo "${PACKAGE_NAME//-/_}")"
 PUBLIC="true"
 
@@ -22,7 +23,7 @@ CYAN="\033[0;36m"     # cyan
 pip install wheel >/dev/null
 
 command -v gitchangelog >/dev/null 2>&1 || {
-    echo -e "${RED}WARNING: Missing gitchangelog binary, please run: pip install gitchangelog==2.2.0${COLOR_OFF}\n"
+    echo -e "${RED}WARNING: Missing gitchangelog binary, please run: pip install gitchangelog==3.0.4${COLOR_OFF}\n"
     exit 1
 }
 
@@ -31,6 +32,11 @@ command -v rst-lint > /dev/null || {
     exit 1
 }
 
+command -v twine >/dev/null || {
+    echo -e "${RED}WARNING: Missing twine binary, please run: pip install twine==3.2.0${COLOR_OFF}\n"
+    exit 1
+}
+
 if [[ "$@" != "major" ]] && [[ "$@" != "minor" ]] && [[ "$@" != "patch" ]]; then
     echo -e "${RED}WARNING: Invalid release type, must specify 'major', 'minor', or 'patch'${COLOR_OFF}\n"
     exit 1
@@ -38,13 +44,13 @@ fi
 
 echo -e "\n${GREEN}STARTING RELEASE PROCESS${COLOR_OFF}\n"
 
-set +e;
+set +e
 git status | grep -Eo "working (directory|tree) clean" &>/dev/null
 if [ ! $? -eq 0 ]; then # working directory is NOT clean
     echo -e "${RED}WARNING: You have uncomitted changes, you may have forgotten something${COLOR_OFF}\n"
     exit 1
 fi
-set -e;
+set -e
 
 echo -e "${YELLOW}--->${COLOR_OFF} Updating local copy"
 git pull -q origin master
@@ -57,14 +63,14 @@ minor=$(echo $current_version | awk '{split($0,a,"."); print a[2]}')
 patch=$(echo $current_version | awk '{split($0,a,"."); print a[3]}')
 
 if [[ "$@" == "major" ]]; then
-    major=$(($major + 1));
+    major=$(($major + 1))
     minor="0"
     patch="0"
 elif [[ "$@" == "minor" ]]; then
-    minor=$(($minor + 1));
+    minor=$(($minor + 1))
     patch="0"
 elif [[ "$@" == "patch" ]]; then
-    patch=$(($patch + 1));
+    patch=$(($patch + 1))
 fi
 
 next_version="${major}.${minor}.${patch}"
@@ -72,7 +78,7 @@ next_version="${major}.${minor}.${patch}"
 echo -e "${YELLOW}   >${COLOR_OFF} ${MAGENTA}${current_version}${COLOR_OFF} -> ${MAGENTA}${next_version}${COLOR_OFF}"
 
 echo -e "${YELLOW}--->${COLOR_OFF} Ensuring readme passes lint checks (if this fails, run rst-lint)"
-rst-lint README.rst > /dev/null
+rst-lint README.rst || exit 1
 
 echo -e "${YELLOW}--->${COLOR_OFF} Creating necessary temp file"
 tempfoo=$(basename $0)
@@ -81,18 +87,12 @@ TMPFILE=$(mktemp /tmp/${tempfoo}.XXXXXX) || {
     exit 1
 }
 
-find_this="__version__ = '$current_version'"
+find_this="__version__ = \"$current_version\""
-replace_with="__version__ = '$next_version'"
+replace_with="__version__ = \"$next_version\""
 
 echo -e "${YELLOW}--->${COLOR_OFF} Updating ${INIT_PACKAGE_NAME}/__init__.py"
 sed "s/$find_this/$replace_with/" ${INIT_PACKAGE_NAME}/__init__.py >$TMPFILE && mv $TMPFILE ${INIT_PACKAGE_NAME}/__init__.py
 
-find_this="${PACKAGE_NAME}.git@$current_version"
-replace_with="${PACKAGE_NAME}.git@$next_version"
-
-echo -e "${YELLOW}--->${COLOR_OFF} Updating README.rst"
-sed "s/$find_this/$replace_with/" README.rst > $TMPFILE && mv $TMPFILE README.rst
-
 if [ -f docs/conf.py ]; then
     echo -e "${YELLOW}--->${COLOR_OFF} Updating docs"
     find_this="version = '${current_version}'"
@@ -106,7 +106,9 @@ fi
 
 echo -e "${YELLOW}--->${COLOR_OFF} Updating CHANGES.rst for new release"
 version_header="$next_version ($(date +%F))"
-set +e; dashes=$(yes '-'|head -n ${#version_header}|tr -d '\n') ; set -e
+set +e
+dashes=$(yes '-' | head -n ${#version_header} | tr -d '\n')
+set -e
 gitchangelog | sed "4s/.*/$version_header/" | sed "5s/.*/$dashes/" >$TMPFILE && mv $TMPFILE CHANGES.rst
 
 echo -e "${YELLOW}--->${COLOR_OFF} Adding changed files to git"
@@ -116,6 +118,15 @@ if [ -f docs/conf.py ]; then git add docs/conf.py; fi
 echo -e "${YELLOW}--->${COLOR_OFF} Creating release"
 git commit -q -m "Release version $next_version"
 
+if [[ "$PUBLIC" == "true" ]]; then
+    echo -e "${YELLOW}--->${COLOR_OFF} Creating python release files"
+    cp README.rst README
+    python setup.py sdist bdist_wheel >/dev/null
+
+    echo -e "${YELLOW}--->${COLOR_OFF} Validating long_description"
+    twine check dist/*
+fi
+
 echo -e "${YELLOW}--->${COLOR_OFF} Tagging release"
 git tag -a $next_version -m "Release version $next_version"
 
@@ -123,9 +134,8 @@ echo -e "${YELLOW}--->${COLOR_OFF} Pushing release and tags to github"
 git push -q origin master && git push -q --tags
 
 if [[ "$PUBLIC" == "true" ]]; then
-    echo -e "${YELLOW}--->${COLOR_OFF} Creating python release"
+    echo -e "${YELLOW}--->${COLOR_OFF} Uploading python release"
-    cp README.rst README
+    twine upload dist/*
-    python setup.py sdist bdist_wheel upload > /dev/null
     rm README
 fi
 

release-requirements.txt

@@ -0,0 +1,31 @@
+bleach==6.0.0
+certifi==2023.7.22
+charset-normalizer==3.1.0
+colorama==0.4.6
+docutils==0.20.1
+flake8==6.0.0
+gitchangelog==3.0.4
+idna==3.4
+importlib-metadata==6.6.0
+jaraco.classes==3.2.3
+keyring==23.13.1
+markdown-it-py==2.2.0
+mccabe==0.7.0
+mdurl==0.1.2
+more-itertools==9.1.0
+pkginfo==1.9.6
+pycodestyle==2.10.0
+pyflakes==3.0.1
+Pygments==2.15.1
+readme-renderer==37.3
+requests==2.31.0
+requests-toolbelt==1.0.0
+restructuredtext-lint==1.4.0
+rfc3986==2.0.0
+rich==13.3.5
+six==1.16.0
+tqdm==4.65.0
+twine==4.0.2
+urllib3==2.0.7
+webencodings==0.5.1
+zipp==3.15.0

setup.py

@@ -5,6 +5,7 @@ from github_backup import __version__
 
 try:
     from setuptools import setup
+
     setup  # workaround for pyflakes issue #13
 except ImportError:
     from distutils.core import setup
@@ -15,6 +16,7 @@ except ImportError:
 # http://www.eby-sarna.com/pipermail/peak/2010-May/003357.html)
 try:
     import multiprocessing
+
     multiprocessing
 except ImportError:
     pass
@@ -25,24 +27,26 @@ def open_file(fname):
 
 
 setup(
-    name='github-backup',
+    name="github-backup",
     version=__version__,
-    author='Jose Diaz-Gonzalez',
+    author="Jose Diaz-Gonzalez",
-    author_email='github-backup@josediazgonzalez.com',
+    author_email="github-backup@josediazgonzalez.com",
-    packages=['github_backup'],
+    packages=["github_backup"],
-    scripts=['bin/github-backup'],
+    scripts=["bin/github-backup"],
-    url='http://github.com/josegonzalez/python-github-backup',
+    url="http://github.com/josegonzalez/python-github-backup",
-    license=open('LICENSE.txt').read(),
+    license="MIT",
     classifiers=[
-        'Development Status :: 5 - Production/Stable',
+        "Development Status :: 5 - Production/Stable",
-        'Topic :: System :: Archiving :: Backup',
+        "Topic :: System :: Archiving :: Backup",
-        'License :: OSI Approved :: MIT License',
+        "License :: OSI Approved :: MIT License",
-        'Programming Language :: Python :: 2.7',
+        "Programming Language :: Python :: 3.5",
-        'Programming Language :: Python :: 3.5',
+        "Programming Language :: Python :: 3.6",
-        'Programming Language :: Python :: 3.6',
+        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.8",
     ],
-    description='backup a github user or organization',
+    description="backup a github user or organization",
-    long_description=open_file('README.rst').read(),
+    long_description=open_file("README.rst").read(),
-    install_requires=open_file('requirements.txt').readlines(),
+    long_description_content_type="text/x-rst",
+    install_requires=open_file("requirements.txt").readlines(),
     zip_safe=True,
 )