Release version 0.44.1

chore(deps): bump certifi from 2023.5.7 to 2023.7.22

.circleci/config.yml

@@ -1,23 +0,0 @@
-version: 2.1
-
-orbs:
-  python: circleci/python@0.3.2
-
-jobs:
-  build-and-test:
-    executor: python/default
-    steps:
-      - checkout
-      - python/load-cache
-      - run:
-          command: pip install flake8
-          name: Install dependencies
-      - python/save-cache
-      - run:
-          command: flake8 --ignore=E501
-          name: Lint
-
-workflows:
-  main:
-    jobs:
-      - build-and-test

.github/workflows/automatic-release.yml

@@ -0,0 +1,41 @@
+name: automatic-release
+
+on:
+    workflow_dispatch:
+        inputs:
+            release_type:
+                description: Release type
+                required: true
+                type: choice
+                options:
+                    - patch
+                    - minor
+                    - major
+
+jobs:
+    release:
+        name: Release
+        runs-on: ubuntu-22.04
+        steps:
+            - name: Checkout repository
+              uses: actions/checkout@v4
+              with:
+                fetch-depth: 0
+                ssh-key: ${{ secrets.DEPLOY_PRIVATE_KEY }}
+            - name: Setup Git
+              run: |
+                git config --local user.email "action@github.com"
+                git config --local user.name "GitHub Action"
+            - name: Setup Python
+              uses: actions/setup-python@v4
+              with:
+                python-version: '3.8'
+            - name: Install prerequisites
+              run: pip install -r release-requirements.txt
+            - name: Execute release
+              env:
+                SEMVER_BUMP: ${{ github.event.inputs.release_type }}
+                TWINE_REPOSITORY: ${{ vars.TWINE_REPOSITORY }}
+                TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
+                TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
+              run: ./release $SEMVER_BUMP

.github/workflows/lint.yml

@@ -0,0 +1,32 @@
+---
+name: "lint"
+
+# yamllint disable-line rule:truthy
+on:
+  pull_request:
+    branches:
+      - '*'
+  push:
+    branches:
+      - 'main'
+      - 'master'
+
+jobs:
+  lint:
+    name: lint
+    runs-on: ubuntu-22.04
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.8"
+          cache: "pip"
+      - run: pip install -r release-requirements.txt
+      - run: flake8 --ignore=E501,E203,W503
+      - run: black .
+      - run: rst-lint README.rst

CHANGES.rst

@@ -1,10 +1,133 @@
 Changelog
 =========
 
-0.43.0 (2023-05-29)
+0.44.1 (2023-12-09)
 -------------------
 ------------------------
 
+Fix
+~~~
+- Use a deploy key to push tags so releases get auto-created. [Jose
+  Diaz-Gonzalez]
+
+Other
+~~~~~
+- Chore(deps): bump certifi from 2023.5.7 to 2023.7.22.
+  [dependabot[bot]]
+
+  Bumps [certifi](https://github.com/certifi/python-certifi) from 2023.5.7 to 2023.7.22.
+  - [Commits](https://github.com/certifi/python-certifi/compare/2023.05.07...2023.07.22)
+
+  ---
+  updated-dependencies:
+  - dependency-name: certifi
+    dependency-type: direct:production
+  ...
+- Tests: run lint on pull requests. [Jose Diaz-Gonzalez]
+- Chore(deps): bump urllib3 from 2.0.2 to 2.0.7. [dependabot[bot]]
+
+  Bumps [urllib3](https://github.com/urllib3/urllib3) from 2.0.2 to 2.0.7.
+  - [Release notes](https://github.com/urllib3/urllib3/releases)
+  - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst)
+  - [Commits](https://github.com/urllib3/urllib3/compare/2.0.2...2.0.7)
+
+  ---
+  updated-dependencies:
+  - dependency-name: urllib3
+    dependency-type: direct:production
+  ...
+- Chore: remove circleci as tests now run in github actions. [Jose Diaz-
+  Gonzalez]
+- Tests: install correct dependencies and rename job. [Jose Diaz-
+  Gonzalez]
+- Tests: add lint github action workflow. [Jose Diaz-Gonzalez]
+- Feat: install autopep8. [Jose Diaz-Gonzalez]
+- Chore(deps): bump certifi from 2023.5.7 to 2023.7.22.
+  [dependabot[bot]]
+
+  Bumps [certifi](https://github.com/certifi/python-certifi) from 2023.5.7 to 2023.7.22.
+  - [Commits](https://github.com/certifi/python-certifi/compare/2023.05.07...2023.07.22)
+
+  ---
+  updated-dependencies:
+  - dependency-name: certifi
+    dependency-type: direct:production
+  ...
+- Chore: reformat file and update flake8. [Jose Diaz-Gonzalez]
+
+
+0.44.0 (2023-12-09)
+-------------------
+
+Fix
+~~~
+- Do not use raw property in readme. [Jose Diaz-Gonzalez]
+
+  This is disabled on pypi.
+- Validate release before committing and uploading it. [Jose Diaz-
+  Gonzalez]
+- Correct lint issues and show errors on lint. [Jose Diaz-Gonzalez]
+- Minor cosmetic changes. [ZhymabekRoman]
+- Add forgotten variable formatting. [ZhymabekRoman]
+- Refactor logging Based on #195. [ZhymabekRoman]
+- Minor typo fix. [Zhymabek Roman]
+
+Other
+~~~~~
+- Bump certifi from 2023.5.7 to 2023.7.22. [dependabot[bot]]
+
+  Bumps [certifi](https://github.com/certifi/python-certifi) from 2023.5.7 to 2023.7.22.
+  - [Commits](https://github.com/certifi/python-certifi/compare/2023.05.07...2023.07.22)
+
+  ---
+  updated-dependencies:
+  - dependency-name: certifi
+    dependency-type: direct:production
+  ...
+- Checkout everything. [Halvor Holsten Strand]
+- Added automatic release workflow, for use with GitHub Actions. [Halvor
+  Holsten Strand]
+- Feat: create Dockerfile. [Tom Plant]
+- Fix rst html. [hozza]
+- Add contributor section. [hozza]
+- Fix readme wording and format. [hozza]
+- Fixed readme working and layout. [hozza]
+- Fix readme formatting, spelling and layout. [hozza]
+- Added details usage and examples including gotchas, errors and
+  development instructions. [hozza]
+- Added verbose install instructions. [hozza]
+- Bump urllib3 from 2.0.2 to 2.0.7. [dependabot[bot]]
+
+  Bumps [urllib3](https://github.com/urllib3/urllib3) from 2.0.2 to 2.0.7.
+  - [Release notes](https://github.com/urllib3/urllib3/releases)
+  - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst)
+  - [Commits](https://github.com/urllib3/urllib3/compare/2.0.2...2.0.7)
+
+  ---
+  updated-dependencies:
+  - dependency-name: urllib3
+    dependency-type: direct:production
+  ...
+- Suggested modification to fix win32 logging failure, due to local
+  variable scope. Logger does not appear to have any utility within
+  "logging_subprocess". [Halvor Holsten Strand]
+- Simplified one if/elif scenario. Extracted file reading of another
+  if/elif scenario. [Halvor Holsten Strand]
+- Ran black. [Halvor Holsten Strand]
+- Keep backwards compatability by going back to "--token" for classic.
+  Allow "file://" uri for "--token-fine". [Halvor Holsten Strand]
+- Add support for fine-grained tokens. [froggleston]
+- Refactor logging and add support for quiet flag. [Harrison Wright]
+
+
+0.43.1 (2023-05-29)
+-------------------
+- Chore: add release requirements. [Jose Diaz-Gonzalez]
+
+
+0.43.0 (2023-05-29)
+-------------------
+
 Fix
 ~~~
 - Do not update readme. [Jose Diaz-Gonzalez]

Dockerfile

@@ -0,0 +1,16 @@
+FROM python:3.9.18-slim
+
+RUN --mount=type=cache,target=/var/cache/apt \
+    apt-get update && apt-get install -y git git-lfs
+
+WORKDIR /usr/src/app
+
+COPY release-requirements.txt .
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install -r release-requirements.txt
+
+COPY . .
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install .
+
+ENTRYPOINT [ "github-backup" ]

README.rst

@@ -4,14 +4,13 @@ github-backup
 
 |PyPI| |Python Versions|
 
-    This project is considered feature complete for the primary maintainer. If you would like a bugfix or enhancement and cannot sponsor the work, pull requests are welcome. Feel free to contact the maintainer for consulting estimates if desired.
+The package can be used to backup an *entire* `Github <https://github.com/>`_ organization, repository or user account, including starred repos, issues and wikis in the most appropriate format (clones for wikis, json files for issues).
-
-backup a github user or organization
 
 Requirements
 ============
 
 - GIT 1.9+
+- Python
 
 Installation
 ============
@@ -20,25 +19,34 @@ Using PIP via PyPI::
 
     pip install github-backup
 
-Using PIP via Github::
+Using PIP via Github (more likely the latest version)::
 
     pip install git+https://github.com/josegonzalez/python-github-backup.git#egg=github-backup
+    
+*Install note for python newcomers:*
 
-Usage
+Python scripts are unlikely to be included in your ``$PATH`` by default, this means it cannot be run directly in terminal with ``$ github-backup ...``, you can either add python's install path to your environments ``$PATH`` or call the script directly e.g. using ``$ ~/.local/bin/github-backup``.*
-=====
 
-CLI Usage is as follows::
+Basic Help
+==========
 
-    github-backup [-h] [-u USERNAME] [-p PASSWORD] [-t TOKEN] [--as-app]
+Show the CLI help output::
-                  [-o OUTPUT_DIRECTORY] [-l LOG_LEVEL] [-i] [--starred]
+
-                  [--all-starred] [--watched] [--followers] [--following]
+    github-backup -h
-                  [--all] [--issues] [--issue-comments] [--issue-events]
+
-                  [--pulls] [--pull-comments] [--pull-commits]
+CLI Help output::
-                  [--pull-details] [--labels] [--hooks] [--milestones]
+
-                  [--repositories] [--bare] [--lfs] [--wikis] [--gists]
+    github-backup [-h] [-u USERNAME] [-p PASSWORD] [-t TOKEN_CLASSIC]
-                  [--starred-gists] [--skip-archived] [--skip-existing]
+                  [-f TOKEN_FINE] [--as-app] [-o OUTPUT_DIRECTORY]
-                  [-L [LANGUAGES ...]] [-N NAME_REGEX] [-H GITHUB_HOST]
+                  [-l LOG_LEVEL] [-i] [--starred] [--all-starred]
-                  [-O] [-R REPOSITORY] [-P] [-F] [--prefer-ssh] [-v]
+                  [--watched] [--followers] [--following] [--all] [--issues]
+                  [--issue-comments] [--issue-events] [--pulls]
+                  [--pull-comments] [--pull-commits] [--pull-details]
+                  [--labels] [--hooks] [--milestones] [--repositories]
+                  [--bare] [--lfs] [--wikis] [--gists] [--starred-gists]
+                  [--skip-archived] [--skip-existing] [-L [LANGUAGES ...]]
+                  [-N NAME_REGEX] [-H GITHUB_HOST] [-O] [-R REPOSITORY]
+                  [-P] [-F] [--prefer-ssh] [-v]
                   [--keychain-name OSX_KEYCHAIN_ITEM_NAME]
                   [--keychain-account OSX_KEYCHAIN_ITEM_ACCOUNT]
                   [--releases] [--assets] [--exclude [REPOSITORY [REPOSITORY ...]]
@@ -57,7 +65,10 @@ CLI Usage is as follows::
       -p PASSWORD, --password PASSWORD
                             password for basic auth. If a username is given but
                             not a password, the password will be prompted for.
-      -t TOKEN, --token TOKEN
+      -f TOKEN_FINE, --token-fine TOKEN_FINE
+                            fine-grained personal access token or path to token
+                            (file://...)
+      -t TOKEN_CLASSIC, --token TOKEN_CLASSIC
                             personal access, OAuth, or JSON Web token, or path to
                             token (file://...)
       --as-app              authenticate as github app instead of as a user.
@@ -126,15 +137,43 @@ CLI Usage is as follows::
                             --throttle-limit to be set)
 
 
-The package can be used to backup an *entire* organization or repository, including issues and wikis in the most appropriate format (clones for wikis, json files for issues).
+Usage Details
+=============
 
 Authentication
-==============
+--------------
+
+**Password-based authentication** will fail if you have two-factor authentication enabled, and will `be deprecated <https://github.blog/2023-03-09-raising-the-bar-for-software-security-github-2fa-begins-march-13/>`_ by 2023 EOY.
+
+``--username`` is used for basic password authentication and separate from the positional argument ``USER``, which specifies the user account you wish to back up.
+
+**Classic tokens** are `slightly less secure <https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#personal-access-tokens-classic>`_ as they provide very coarse-grained permissions.
+
+If you need authentication for long-running backups (e.g. for a cron job) it is recommended to use **fine-grained personal access token** ``-f TOKEN_FINE``.
+
+
+Fine Tokens
+~~~~~~~~~~~
+
+You can "generate new token", choosing the repository scope by selecting specific repos or all repos. On Github this is under *Settings -> Developer Settings -> Personal access tokens -> Fine-grained Tokens*
+
+Customise the permissions for your use case, but for a personal account full backup you'll need to enable the following permissions:
+
+**User permissions**: Read access to followers, starring, and watching.
+
+**Repository permissions**: Read access to code, commit statuses, issues, metadata, pages, pull requests, and repository hooks.
+
+
+Prefer SSH
+~~~~~~~~~~
+
+If cloning repos is enabled with ``--repositories``, ``--all-starred``, ``--wikis``, ``--gists``, ``--starred-gists`` using the ``--prefer-ssh`` argument will use ssh for cloning the git repos, but all other connections will still use their own protocol, e.g. API requests for issues uses HTTPS.
+
+To clone with SSH, you'll need SSH authentication setup `as usual with Github <https://docs.github.com/en/authentication/connecting-to-github-with-ssh>`_, e.g. via SSH public and private keys.
 
-Note: Password-based authentication will fail if you have two-factor authentication enabled.
 
 Using the Keychain on Mac OSX
-=============================
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Note: On Mac OSX the token can be stored securely in the user's keychain. To do this:
 
 1. Open Keychain from "Applications -> Utilities -> Keychain Access"
@@ -148,31 +187,135 @@ Note:  When you run github-backup, you will be asked whether you want to allow "
 1. **Allow:** In this case you will need to click "Allow" each time you run `github-backup`
 2. **Always Allow:** In this case, you will not be asked for permission when you run `github-backup` in future. This is less secure, but is required if you want to schedule `github-backup` to run automatically
 
-About Git LFS
-=============
 
-When you use the "--lfs" option, you will need to make sure you have Git LFS installed.
+Github Rate-limit and Throttling
+--------------------------------
+
+"github-backup" will automatically throttle itself based on feedback from the Github API. 
+
+Their API is usually rate-limited to 5000 calls per hour. The API will ask github-backup to pause until a specific time when the limit is reset again (at the start of the next hour). This continues until the backup is complete.
+
+During a large backup, such as ``--all-starred``, and on a fast connection this can result in (~20 min) pauses with bursts of API calls periodically maxing out the API limit. If this is not suitable `it has been observed <https://github.com/josegonzalez/python-github-backup/issues/76#issuecomment-636158717>`_ under real-world conditions that overriding the throttle with ``--throttle-limit 5000 --throttle-pause 0.6`` provides a smooth rate across the hour, although a ``--throttle-pause 0.72`` (3600 seconds [1 hour] / 5000 limit) is theoretically safer to prevent large rate-limit pauses.
+
+
+About Git LFS
+-------------
+
+When you use the ``--lfs`` option, you will need to make sure you have Git LFS installed.
 
 Instructions on how to do this can be found on https://git-lfs.github.com.
 
-Examples
-========
 
-Backup all repositories, including private ones::
+Gotchas / Known-issues
+======================
+
+All is not everything
+---------------------
+
+The ``--all`` argument does not include; cloning private repos (``-P, --private``), cloning forks (``-F, --fork``) cloning starred repositories (``--all-starred``), ``--pull-details``, cloning LFS repositories (``--lfs``), cloning gists (``--starred-gists``) or cloning starred gist repos (``--starred-gists``). See examples for more.
+
+Cloning all starred size
+------------------------
+
+Using the ``--all-starred`` argument to clone all starred repositories may use a large amount of storage space, especially if ``--all`` or more arguments are used. e.g. commonly starred repos can have tens of thousands of issues, many large assets and the repo itself etc. Consider just storing links to starred repos in JSON format with ``--starred``.
+
+Incremental Backup
+------------------
+
+Using (``-i, --incremental``) will only request new data from the API **since the last run (successful or not)**. e.g. only request issues from the API since the last run. 
+
+This means any blocking errors on previous runs can cause a large amount of missing data in backups.
+
+Known blocking errors
+---------------------
+
+Some errors will block the backup run by exiting the script. e.g. receiving a 403 Forbidden error from the Github API.
+
+If the incremental argument is used, this will result in the next backup only requesting API data since the last blocked/failed run. Potentially causing unexpected large amounts of missing data.
+
+It's therefore recommended to only use the incremental argument if the output/result is being actively monitored, or complimented with periodic full non-incremental runs, to avoid unexpected missing data in a regular backup runs.
+
+1. **Starred public repo hooks blocking**
+
+   Since the ``--all`` argument includes ``--hooks``, if you use ``--all`` and ``--all-starred`` together to clone a users starred public repositories, the backup will likely error and block the backup continuing. 
+
+   This is due to needing the correct permission for ``--hooks`` on public repos.
+
+2. **Releases blocking**
+
+   A known ``--releases`` (required for ``--assets``) error will sometimes block the backup. 
+
+   If you're backing up a lot of repositories with releases e.g. an organisation or ``--all-starred``. You may need to remove ``--releases`` (and therefore ``--assets``) to complete a backup. Documented in `issue 209 <https://github.com/josegonzalez/python-github-backup/issues/209>`_.
+
+
+"bare" is actually "mirror"
+---------------------------
+
+Using the bare clone argument (``--bare``) will actually call git's ``clone --mirror`` command. There's a subtle difference between `bare <https://www.git-scm.com/docs/git-clone#Documentation/git-clone.txt---bare>`_ and `mirror <https://www.git-scm.com/docs/git-clone#Documentation/git-clone.txt---mirror>`_ clone.
+
+*From git docs "Compared to --bare, --mirror not only maps local branches of the source to local branches of the target, it maps all refs (including remote-tracking branches, notes etc.) and sets up a refspec configuration such that all these refs are overwritten by a git remote update in the target repository."*
+
+
+Starred gists vs starred repo behaviour
+---------------------------------------
+
+The starred normal repo cloning (``--all-starred``) argument stores starred repos separately to the users own repositories. However, using ``--starred-gists`` will store starred gists within the same directory as the users own gists ``--gists``. Also, all gist repo directory names are IDs not the gist's name.
+
+
+Skip existing on incomplete backups
+-----------------------------------
+
+The ``--skip-existing`` argument will skip a backup if the directory already exists, even if the backup in that directory failed (perhaps due to a blocking error). This may result in unexpected missing data in a regular backup.
+
+
+Github Backup Examples
+======================
+
+Backup all repositories, including private ones using a classic token::
 
     export ACCESS_TOKEN=SOME-GITHUB-TOKEN
     github-backup WhiteHouse --token $ACCESS_TOKEN --organization --output-directory /tmp/white-house --repositories --private
 
-Backup a single organization repository with everything else (wiki, pull requests, comments, issues etc)::
+Use a fine-grained access token to backup a single organization repository with everything else (wiki, pull requests, comments, issues etc)::
 
-    export ACCESS_TOKEN=SOME-GITHUB-TOKEN
+    export FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
     ORGANIZATION=docker
     REPO=cli
     # e.g. git@github.com:docker/cli.git
-    github-backup $ORGANIZATION -P -t $ACCESS_TOKEN -o . --all -O -R $REPO
+    github-backup $ORGANIZATION -P -f $FINE_ACCESS_TOKEN -o . --all -O -R $REPO
+
+Quietly and incrementally backup useful Github user data (public and private repos with SSH) including; all issues, pulls, all public starred repos and gists (omitting "hooks", "releases" and therefore "assets" to prevent blocking). *Great for a cron job.* ::
+
+    export FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
+    GH_USER=YOUR-GITHUB-USER
+
+    github-backup -f $FINE_ACCESS_TOKEN --prefer-ssh -o ~/github-backup/ -l error -P -i --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-commits --labels --milestones --repositories --wikis --releases --assets --pull-details --gists --starred-gists $GH_USER
+    
+Debug an error/block or incomplete backup into a temporary directory. Omit "incremental" to fill a previous incomplete backup. ::
+
+    export FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
+    GH_USER=YOUR-GITHUB-USER
+
+    github-backup -f $FINE_ACCESS_TOKEN -o /tmp/github-backup/ -l debug -P --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-commits --labels --milestones --repositories --wikis --releases --assets --pull-details --gists --starred-gists $GH_USER
+
+
+
+Development
+===========
+
+This project is considered feature complete for the primary maintainer @josegonzalez. If you would like a bugfix or enhancement, pull requests are welcome. Feel free to contact the maintainer for consulting estimates if you'd like to sponsor the work instead.
+
+Contibuters
+-----------
+
+A huge thanks to all the contibuters!
+
+.. image:: https://contrib.rocks/image?repo=josegonzalez/python-github-backup
+   :target: https://github.com/josegonzalez/python-github-backup/graphs/contributors
+   :alt: contributors
 
 Testing
-=======
+-------
 
 This project currently contains no unit tests.  To run linting::
 

bin/github-backup

@@ -2,31 +2,34 @@
 
 import os, sys, logging
 
-from github_backup.github_backup import (
-    backup_account,
-    backup_repositories,
-    check_git_lfs_install,
-    filter_repositories,
-    get_authenticated_user,
-    log_info,
-    log_warning,
-    mkdir_p,
-    parse_args,
-    retrieve_repositories,
-)
-
 logging.basicConfig(
     format='%(asctime)s.%(msecs)03d: %(message)s',
     datefmt='%Y-%m-%dT%H:%M:%S',
     level=logging.INFO
 )
 
+from github_backup.github_backup import (
+    backup_account,
+    backup_repositories,
+    check_git_lfs_install,
+    filter_repositories,
+    get_authenticated_user,
+    logger,
+    mkdir_p,
+    parse_args,
+    retrieve_repositories,
+)
+
+
 def main():
     args = parse_args()
 
+    if args.quiet:
+        logger.setLevel(logging.WARNING)
+
     output_directory = os.path.realpath(args.output_directory)
     if not os.path.isdir(output_directory):
-        log_info('Create output directory {0}'.format(output_directory))
+        logger.info('Create output directory {0}'.format(output_directory))
         mkdir_p(output_directory)
 
     if args.lfs_clone:
@@ -35,10 +38,10 @@ def main():
     if args.log_level:
         log_level = logging.getLevelName(args.log_level.upper())
         if isinstance(log_level, int):
-            logging.root.setLevel(log_level)
+            logger.root.setLevel(log_level)
 
     if not args.as_app:
-        log_info('Backing up user {0} to {1}'.format(args.user, output_directory))
+        logger.info('Backing up user {0} to {1}'.format(args.user, output_directory))
         authenticated_user = get_authenticated_user(args)
     else:
         authenticated_user = {'login': None}
@@ -53,5 +56,5 @@ if __name__ == '__main__':
     try:
         main()
     except Exception as e:
-        log_warning(str(e))
+        logger.error(str(e))
         sys.exit(1)

github_backup/__init__.py

@@ -1 +1 @@
-__version__ = "0.43.0"
+__version__ = "0.44.1"

github_backup/github_backup.py

@@ -1,33 +1,28 @@
 #!/usr/bin/env python
 
 from __future__ import print_function
-import socket
 
 import argparse
 import base64
 import calendar
 import codecs
-import datetime
 import errno
 import getpass
 import json
+import logging
 import os
+import platform
 import re
 import select
+import socket
 import subprocess
 import sys
-import logging
 import time
-import platform
-from urllib.parse import urlparse
-from urllib.parse import quote as urlquote
-from urllib.parse import urlencode
-from urllib.error import HTTPError, URLError
-from urllib.request import urlopen
-from urllib.request import Request
-from urllib.request import HTTPRedirectHandler
-from urllib.request import build_opener
 from http.client import IncompleteRead
+from urllib.error import HTTPError, URLError
+from urllib.parse import quote as urlquote
+from urllib.parse import urlencode, urlparse
+from urllib.request import HTTPRedirectHandler, Request, build_opener, urlopen
 
 try:
     from . import __version__
@@ -37,40 +32,12 @@ except ImportError:
     VERSION = "unknown"
 
 FNULL = open(os.devnull, "w")
-
+FILE_URI_PREFIX = "file://"
-
+logger = logging.getLogger(__name__)
-def _get_log_date():
-    return datetime.datetime.isoformat(datetime.datetime.now())
-
-
-def log_info(message):
-    """
-    Log message (str) or messages (List[str]) to stdout
-    """
-    if type(message) == str:
-        message = [message]
-
-    for msg in message:
-        logging.info(msg)
-
-
-def log_warning(message):
-    """
-    Log message (str) or messages (List[str]) to stderr
-    """
-    if type(message) == str:
-        message = [message]
-
-    for msg in message:
-        logging.warning(msg)
 
 
 def logging_subprocess(
-    popenargs,
+    popenargs, stdout_log_level=logging.DEBUG, stderr_log_level=logging.ERROR, **kwargs
-    logger,
-    stdout_log_level=logging.DEBUG,
-    stderr_log_level=logging.ERROR,
-    **kwargs
 ):
     """
     Variant of subprocess.call that accepts a logger instead of stdout/stderr,
@@ -81,7 +48,7 @@ def logging_subprocess(
         popenargs, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs
     )
     if sys.platform == "win32":
-        log_info(
+        logger.info(
             "Windows operating system detected - no subprocess logging will be returned"
         )
 
@@ -152,9 +119,22 @@ def parse_args(args=None):
     parser.add_argument(
         "-t",
         "--token",
-        dest="token",
+        dest="token_classic",
         help="personal access, OAuth, or JSON Web token, or path to token (file://...)",
     )  # noqa
+    parser.add_argument(
+        "-f",
+        "--token-fine",
+        dest="token_fine",
+        help="fine-grained personal access token (github_pat_....), or path to token (file://...)",
+    )  # noqa
+    parser.add_argument(
+        "-q",
+        "--quiet",
+        action="store_true",
+        dest="quiet",
+        help="supress log messages less severe than warning, e.g. info",
+    )
     parser.add_argument(
         "--as-app",
         action="store_true",
@@ -457,18 +437,27 @@ def get_auth(args, encode=True, for_git_cli=False):
         raise Exception(
             "You must specify both name and account fields for osx keychain password items"
         )
-    elif args.token:
+    elif args.token_fine:
-        _path_specifier = "file://"
+        if args.token_fine.startswith(FILE_URI_PREFIX):
-        if args.token.startswith(_path_specifier):
+            args.token_fine = read_file_contents(args.token_fine)
-            path_specifier_len = len(_path_specifier)
+
-            args.token = open(args.token[path_specifier_len:], "rt").readline().strip()
+        if args.token_fine.startswith("github_pat_"):
+            auth = args.token_fine
+        else:
+            raise Exception(
+                "Fine-grained token supplied does not look like a GitHub PAT"
+            )
+    elif args.token_classic:
+        if args.token_classic.startswith(FILE_URI_PREFIX):
+            args.token_classic = read_file_contents(args.token_classic)
+
         if not args.as_app:
-            auth = args.token + ":" + "x-oauth-basic"
+            auth = args.token_classic + ":" + "x-oauth-basic"
         else:
             if not for_git_cli:
-                auth = args.token
+                auth = args.token_classic
             else:
-                auth = "x-access-token:" + args.token
+                auth = "x-access-token:" + args.token_classic
     elif args.username:
         if not args.password:
             args.password = getpass.getpass()
@@ -483,7 +472,7 @@ def get_auth(args, encode=True, for_git_cli=False):
     if not auth:
         return None
 
-    if not encode:
+    if not encode or args.token_fine is not None:
         return auth
 
     return base64.b64encode(auth.encode("ascii"))
@@ -507,6 +496,10 @@ def get_github_host(args):
     return host
 
 
+def read_file_contents(file_uri):
+    return open(file_uri[len(FILE_URI_PREFIX) :], "rt").readline().strip()
+
+
 def get_github_repo_url(args, repository):
     if repository.get("is_gist"):
         if args.prefer_ssh:
@@ -529,7 +522,7 @@ def get_github_repo_url(args, repository):
     auth = get_auth(args, encode=False, for_git_cli=True)
     if auth:
         repo_url = "https://{0}@{1}/{2}/{3}.git".format(
-            auth,
+            auth if args.token_fine is None else "oauth2:" + auth,
             get_github_host(args),
             repository["owner"]["login"],
             repository["name"],
@@ -549,7 +542,13 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False):
     while True:
         page = page + 1
         request = _construct_request(
-            per_page, page, query_args, template, auth, as_app=args.as_app
+            per_page,
+            page,
+            query_args,
+            template,
+            auth,
+            as_app=args.as_app,
+            fine=True if args.token_fine is not None else False,
         )  # noqa
         r, errors = _get_response(request, auth, template)
 
@@ -558,13 +557,13 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False):
         try:
             response = json.loads(r.read().decode("utf-8"))
         except IncompleteRead:
-            log_warning("Incomplete read error detected")
+            logger.warning("Incomplete read error detected")
             read_error = True
         except json.decoder.JSONDecodeError:
-            log_warning("JSON decode error detected")
+            logger.warning("JSON decode error detected")
             read_error = True
         except TimeoutError:
-            log_warning("Tiemout error detected")
+            logger.warning("Tiemout error detected")
             read_error = True
         else:
             read_error = False
@@ -572,7 +571,7 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False):
         # be gentle with API request limit and throttle requests if remaining requests getting low
         limit_remaining = int(r.headers.get("x-ratelimit-remaining", 0))
         if args.throttle_limit and limit_remaining <= args.throttle_limit:
-            log_info(
+            logger.info(
                 "API request limit hit: {} requests left, pausing further requests for {}s".format(
                     limit_remaining, args.throttle_pause
                 )
@@ -581,11 +580,17 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False):
 
         retries = 0
         while retries < 3 and (status_code == 502 or read_error):
-            log_warning("API request failed. Retrying in 5 seconds")
+            logger.warning("API request failed. Retrying in 5 seconds")
             retries += 1
             time.sleep(5)
             request = _construct_request(
-                per_page, page, query_args, template, auth, as_app=args.as_app
+                per_page,
+                page,
+                query_args,
+                template,
+                auth,
+                as_app=args.as_app,
+                fine=True if args.token_fine is not None else False,
             )  # noqa
             r, errors = _get_response(request, auth, template)
 
@@ -594,13 +599,13 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False):
                 response = json.loads(r.read().decode("utf-8"))
                 read_error = False
             except IncompleteRead:
-                log_warning("Incomplete read error detected")
+                logger.warning("Incomplete read error detected")
                 read_error = True
             except json.decoder.JSONDecodeError:
-                log_warning("JSON decode error detected")
+                logger.warning("JSON decode error detected")
                 read_error = True
             except TimeoutError:
-                log_warning("Tiemout error detected")
+                logger.warning("Tiemout error detected")
                 read_error = True
 
         if status_code != 200:
@@ -614,12 +619,12 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False):
             raise Exception(", ".join(errors))
 
         if len(errors) == 0:
-            if type(response) == list:
+            if type(response) is list:
                 for resp in response:
                     yield resp
                 if len(response) < per_page:
                     break
-            elif type(response) == dict and single_request:
+            elif type(response) is dict and single_request:
                 yield response
 
         if len(errors) > 0:
@@ -652,12 +657,12 @@ def _get_response(request, auth, template):
             errors, should_continue = _request_http_error(exc, auth, errors)  # noqa
             r = exc
         except URLError as e:
-            log_warning(e.reason)
+            logger.warning(e.reason)
             should_continue, retry_timeout = _request_url_error(template, retry_timeout)
             if not should_continue:
                 raise
         except socket.error as e:
-            log_warning(e.strerror)
+            logger.warning(e.strerror)
             should_continue, retry_timeout = _request_url_error(template, retry_timeout)
             if not should_continue:
                 raise
@@ -669,24 +674,30 @@ def _get_response(request, auth, template):
     return r, errors
 
 
-def _construct_request(per_page, page, query_args, template, auth, as_app=None):
+def _construct_request(
+    per_page, page, query_args, template, auth, as_app=None, fine=False
+):
     querystring = urlencode(
         dict(
-            list({"per_page": per_page, "page": page}.items()) + list(query_args.items())
+            list({"per_page": per_page, "page": page}.items())
+            + list(query_args.items())
         )
     )
 
     request = Request(template + "?" + querystring)
     if auth is not None:
         if not as_app:
-            request.add_header("Authorization", "Basic ".encode("ascii") + auth)
+            if fine:
+                request.add_header("Authorization", "token " + auth)
+            else:
+                request.add_header("Authorization", "Basic ".encode("ascii") + auth)
         else:
             auth = auth.encode("ascii")
             request.add_header("Authorization", "token ".encode("ascii") + auth)
             request.add_header(
                 "Accept", "application/vnd.github.machine-man-preview+json"
             )
-    log_info("Requesting {}?{}".format(template, querystring))
+    logger.info("Requesting {}?{}".format(template, querystring))
     return request
 
 
@@ -710,14 +721,14 @@ def _request_http_error(exc, auth, errors):
         delta = max(10, reset - gm_now)
 
         limit = headers.get("x-ratelimit-limit")
-        log_warning(
+        logger.warning(
             "Exceeded rate limit of {} requests; waiting {} seconds to reset".format(
                 limit, delta
             )
         )  # noqa
 
         if auth is None:
-            log_info("Hint: Authenticate to raise your GitHub rate limit")
+            logger.info("Hint: Authenticate to raise your GitHub rate limit")
 
         time.sleep(delta)
         should_continue = True
@@ -727,13 +738,13 @@ def _request_http_error(exc, auth, errors):
 def _request_url_error(template, retry_timeout):
     # In case of a connection timing out, we can retry a few time
     # But we won't crash and not back-up the rest now
-    log_info("{} timed out".format(template))
+    logger.info("'{}' timed out".format(template))
     retry_timeout -= 1
 
     if retry_timeout >= 0:
         return True, retry_timeout
 
-    raise Exception("{} timed out to much, skipping!")
+    raise Exception("'{}' timed out to much, skipping!".format(template))
 
 
 class S3HTTPRedirectHandler(HTTPRedirectHandler):
@@ -774,20 +785,20 @@ def download_file(url, path, auth):
                 f.write(chunk)
     except HTTPError as exc:
         # Gracefully handle 404 responses (and others) when downloading from S3
-        log_warning(
+        logger.warning(
             "Skipping download of asset {0} due to HTTPError: {1}".format(
                 url, exc.reason
             )
         )
     except URLError as e:
         # Gracefully handle other URL errors
-        log_warning(
+        logger.warning(
             "Skipping download of asset {0} due to URLError: {1}".format(url, e.reason)
         )
     except socket.error as e:
         # Gracefully handle socket errors
         # TODO: Implement retry logic
-        log_warning(
+        logger.warning(
             "Skipping download of asset {0} due to socker error: {1}".format(
                 url, e.strerror
             )
@@ -809,14 +820,14 @@ def check_git_lfs_install():
 
 
 def retrieve_repositories(args, authenticated_user):
-    log_info("Retrieving repositories")
+    logger.info("Retrieving repositories")
     single_request = False
     if args.user == authenticated_user["login"]:
         # we must use the /user/repos API to be able to access private repos
         template = "https://{0}/user/repos".format(get_github_api_host(args))
     else:
         if args.private and not args.organization:
-            log_warning(
+            logger.warning(
                 "Authenticated user is different from user being backed up, thus private repositories cannot be accessed"
             )
         template = "https://{0}/users/{1}/repos".format(
@@ -872,7 +883,7 @@ def retrieve_repositories(args, authenticated_user):
 
 
 def filter_repositories(args, unfiltered_repositories):
-    log_info("Filtering repositories")
+    logger.info("Filtering repositories")
 
     repositories = []
     for r in unfiltered_repositories:
@@ -911,7 +922,7 @@ def filter_repositories(args, unfiltered_repositories):
 
 
 def backup_repositories(args, output_directory, repositories):
-    log_info("Backing up repositories")
+    logger.info("Backing up repositories")
     repos_template = "https://{0}/repos".format(get_github_api_host(args))
 
     if args.incremental:
@@ -1023,7 +1034,7 @@ def backup_issues(args, repo_cwd, repository, repos_template):
     if args.skip_existing and has_issues_dir:
         return
 
-    log_info("Retrieving {0} issues".format(repository["full_name"]))
+    logger.info("Retrieving {0} issues".format(repository["full_name"]))
     issue_cwd = os.path.join(repo_cwd, "issues")
     mkdir_p(repo_cwd, issue_cwd)
 
@@ -1052,7 +1063,7 @@ def backup_issues(args, repo_cwd, repository, repos_template):
     if issues_skipped:
         issues_skipped_message = " (skipped {0} pull requests)".format(issues_skipped)
 
-    log_info(
+    logger.info(
         "Saving {0} issues to disk{1}".format(
             len(list(issues.keys())), issues_skipped_message
         )
@@ -1077,7 +1088,7 @@ def backup_pulls(args, repo_cwd, repository, repos_template):
     if args.skip_existing and has_pulls_dir:
         return
 
-    log_info("Retrieving {0} pull requests".format(repository["full_name"]))  # noqa
+    logger.info("Retrieving {0} pull requests".format(repository["full_name"]))  # noqa
     pulls_cwd = os.path.join(repo_cwd, "pulls")
     mkdir_p(repo_cwd, pulls_cwd)
 
@@ -1113,7 +1124,7 @@ def backup_pulls(args, repo_cwd, repository, repos_template):
                     single_request=True,
                 )[0]
 
-    log_info("Saving {0} pull requests to disk".format(len(list(pulls.keys()))))
+    logger.info("Saving {0} pull requests to disk".format(len(list(pulls.keys()))))
     # Comments from pulls API are only _review_ comments
     # regular comments need to be fetched via issue API.
     # For backwards compatibility with versions <= 0.41.0
@@ -1141,7 +1152,7 @@ def backup_milestones(args, repo_cwd, repository, repos_template):
     if args.skip_existing and os.path.isdir(milestone_cwd):
         return
 
-    log_info("Retrieving {0} milestones".format(repository["full_name"]))
+    logger.info("Retrieving {0} milestones".format(repository["full_name"]))
     mkdir_p(repo_cwd, milestone_cwd)
 
     template = "{0}/{1}/milestones".format(repos_template, repository["full_name"])
@@ -1154,7 +1165,7 @@ def backup_milestones(args, repo_cwd, repository, repos_template):
     for milestone in _milestones:
         milestones[milestone["number"]] = milestone
 
-    log_info("Saving {0} milestones to disk".format(len(list(milestones.keys()))))
+    logger.info("Saving {0} milestones to disk".format(len(list(milestones.keys()))))
     for number, milestone in list(milestones.items()):
         milestone_file = "{0}/{1}.json".format(milestone_cwd, number)
         with codecs.open(milestone_file, "w", encoding="utf-8") as f:
@@ -1171,7 +1182,7 @@ def backup_labels(args, repo_cwd, repository, repos_template):
 def backup_hooks(args, repo_cwd, repository, repos_template):
     auth = get_auth(args)
     if not auth:
-        log_info("Skipping hooks since no authentication provided")
+        logger.info("Skipping hooks since no authentication provided")
         return
     hook_cwd = os.path.join(repo_cwd, "hooks")
     output_file = "{0}/hooks.json".format(hook_cwd)
@@ -1179,7 +1190,7 @@ def backup_hooks(args, repo_cwd, repository, repos_template):
     try:
         _backup_data(args, "hooks", template, output_file, hook_cwd)
     except SystemExit:
-        log_info("Unable to read hooks, skipping")
+        logger.info("Unable to read hooks, skipping")
 
 
 def backup_releases(args, repo_cwd, repository, repos_template, include_assets=False):
@@ -1187,7 +1198,7 @@ def backup_releases(args, repo_cwd, repository, repos_template, include_assets=F
 
     # give release files somewhere to live & log intent
     release_cwd = os.path.join(repo_cwd, "releases")
-    log_info("Retrieving {0} releases".format(repository_fullname))
+    logger.info("Retrieving {0} releases".format(repository_fullname))
     mkdir_p(repo_cwd, release_cwd)
 
     query_args = {}
@@ -1196,7 +1207,7 @@ def backup_releases(args, repo_cwd, repository, repos_template, include_assets=F
     releases = retrieve_data(args, release_template, query_args=query_args)
 
     # for each release, store it
-    log_info("Saving {0} releases to disk".format(len(releases)))
+    logger.info("Saving {0} releases to disk".format(len(releases)))
     for release in releases:
         release_name = release["tag_name"]
         release_name_safe = release_name.replace("/", "__")
@@ -1251,7 +1262,7 @@ def fetch_repository(
         "git ls-remote " + remote_url, stdout=FNULL, stderr=FNULL, shell=True
     )
     if initialized == 128:
-        log_info(
+        logger.info(
             "Skipping {0} ({1}) since it's not initialized".format(
                 name, masked_remote_url
             )
@@ -1259,19 +1270,19 @@ def fetch_repository(
         return
 
     if clone_exists:
-        log_info("Updating {0} in {1}".format(name, local_dir))
+        logger.info("Updating {0} in {1}".format(name, local_dir))
 
         remotes = subprocess.check_output(["git", "remote", "show"], cwd=local_dir)
         remotes = [i.strip() for i in remotes.decode("utf-8").splitlines()]
 
         if "origin" not in remotes:
             git_command = ["git", "remote", "rm", "origin"]
-            logging_subprocess(git_command, None, cwd=local_dir)
+            logging_subprocess(git_command, cwd=local_dir)
             git_command = ["git", "remote", "add", "origin", remote_url]
-            logging_subprocess(git_command, None, cwd=local_dir)
+            logging_subprocess(git_command, cwd=local_dir)
         else:
             git_command = ["git", "remote", "set-url", "origin", remote_url]
-            logging_subprocess(git_command, None, cwd=local_dir)
+            logging_subprocess(git_command, cwd=local_dir)
 
         if lfs_clone:
             git_command = ["git", "lfs", "fetch", "--all", "--prune"]
@@ -1279,27 +1290,27 @@ def fetch_repository(
             git_command = ["git", "fetch", "--all", "--force", "--tags", "--prune"]
         if no_prune:
             git_command.pop()
-        logging_subprocess(git_command, None, cwd=local_dir)
+        logging_subprocess(git_command, cwd=local_dir)
     else:
-        log_info(
+        logger.info(
             "Cloning {0} repository from {1} to {2}".format(
                 name, masked_remote_url, local_dir
             )
         )
         if bare_clone:
             git_command = ["git", "clone", "--mirror", remote_url, local_dir]
-            logging_subprocess(git_command, None)
+            logging_subprocess(git_command)
             if lfs_clone:
                 git_command = ["git", "lfs", "fetch", "--all", "--prune"]
                 if no_prune:
                     git_command.pop()
-                logging_subprocess(git_command, None, cwd=local_dir)
+                logging_subprocess(git_command, cwd=local_dir)
         else:
             if lfs_clone:
                 git_command = ["git", "lfs", "clone", remote_url, local_dir]
             else:
                 git_command = ["git", "clone", remote_url, local_dir]
-            logging_subprocess(git_command, None)
+            logging_subprocess(git_command)
 
 
 def backup_account(args, output_directory):
@@ -1337,11 +1348,11 @@ def backup_account(args, output_directory):
 def _backup_data(args, name, template, output_file, output_directory):
     skip_existing = args.skip_existing
     if not skip_existing or not os.path.exists(output_file):
-        log_info("Retrieving {0} {1}".format(args.user, name))
+        logger.info("Retrieving {0} {1}".format(args.user, name))
         mkdir_p(output_directory)
         data = retrieve_data(args, template)
 
-        log_info("Writing {0} {1} to disk".format(len(data), name))
+        logger.info("Writing {0} {1} to disk".format(len(data), name))
         with codecs.open(output_file, "w", encoding="utf-8") as f:
             json_dump(data, f)
 

release

@@ -1,9 +1,10 @@
 #!/usr/bin/env bash
-set -eo pipefail; [[ $RELEASE_TRACE ]] && set -x
+set -eo pipefail
+[[ $RELEASE_TRACE ]] && set -x
 
 if [[ ! -f setup.py ]]; then
-  echo -e "${RED}WARNING: Missing setup.py${COLOR_OFF}\n"
+    echo -e "${RED}WARNING: Missing setup.py${COLOR_OFF}\n"
-  exit 1
+    exit 1
 fi
 
 PACKAGE_NAME="$(cat setup.py | grep 'name="' | head | cut -d '"' -f2)"
@@ -11,27 +12,27 @@ INIT_PACKAGE_NAME="$(echo "${PACKAGE_NAME//-/_}")"
 PUBLIC="true"
 
 # Colors
-COLOR_OFF="\033[0m"   # unsets color to term fg color
+COLOR_OFF="\033[0m"  # unsets color to term fg color
-RED="\033[0;31m"      # red
+RED="\033[0;31m"     # red
-GREEN="\033[0;32m"    # green
+GREEN="\033[0;32m"   # green
-YELLOW="\033[0;33m"   # yellow
+YELLOW="\033[0;33m"  # yellow
-MAGENTA="\033[0;35m"  # magenta
+MAGENTA="\033[0;35m" # magenta
-CYAN="\033[0;36m"     # cyan
+CYAN="\033[0;36m"    # cyan
 
 # ensure wheel is available
-pip install wheel > /dev/null
+pip install wheel >/dev/null
 
 command -v gitchangelog >/dev/null 2>&1 || {
     echo -e "${RED}WARNING: Missing gitchangelog binary, please run: pip install gitchangelog==3.0.4${COLOR_OFF}\n"
     exit 1
 }
 
-command -v rst-lint > /dev/null || {
+command -v rst-lint >/dev/null || {
     echo -e "${RED}WARNING: Missing rst-lint binary, please run: pip install restructuredtext_lint${COLOR_OFF}\n"
     exit 1
 }
 
-command -v twine > /dev/null || {
+command -v twine >/dev/null || {
     echo -e "${RED}WARNING: Missing twine binary, please run: pip install twine==3.2.0${COLOR_OFF}\n"
     exit 1
 }
@@ -43,41 +44,41 @@ fi
 
 echo -e "\n${GREEN}STARTING RELEASE PROCESS${COLOR_OFF}\n"
 
-set +e;
+set +e
-git status | grep -Eo "working (directory|tree) clean" &> /dev/null
+git status | grep -Eo "working (directory|tree) clean" &>/dev/null
 if [ ! $? -eq 0 ]; then # working directory is NOT clean
     echo -e "${RED}WARNING: You have uncomitted changes, you may have forgotten something${COLOR_OFF}\n"
     exit 1
 fi
-set -e;
+set -e
 
 echo -e "${YELLOW}--->${COLOR_OFF} Updating local copy"
 git pull -q origin master
 
 echo -e "${YELLOW}--->${COLOR_OFF} Retrieving release versions"
 
-current_version=$(cat ${INIT_PACKAGE_NAME}/__init__.py |grep '__version__ ='|sed 's/[^0-9.]//g')
+current_version=$(cat ${INIT_PACKAGE_NAME}/__init__.py | grep '__version__ =' | sed 's/[^0-9.]//g')
 major=$(echo $current_version | awk '{split($0,a,"."); print a[1]}')
 minor=$(echo $current_version | awk '{split($0,a,"."); print a[2]}')
 patch=$(echo $current_version | awk '{split($0,a,"."); print a[3]}')
 
 if [[ "$@" == "major" ]]; then
-    major=$(($major + 1));
+    major=$(($major + 1))
     minor="0"
     patch="0"
 elif [[ "$@" == "minor" ]]; then
-    minor=$(($minor + 1));
+    minor=$(($minor + 1))
     patch="0"
 elif [[ "$@" == "patch" ]]; then
-    patch=$(($patch + 1));
+    patch=$(($patch + 1))
 fi
 
 next_version="${major}.${minor}.${patch}"
 
-echo -e  "${YELLOW}   >${COLOR_OFF} ${MAGENTA}${current_version}${COLOR_OFF} -> ${MAGENTA}${next_version}${COLOR_OFF}"
+echo -e "${YELLOW}   >${COLOR_OFF} ${MAGENTA}${current_version}${COLOR_OFF} -> ${MAGENTA}${next_version}${COLOR_OFF}"
 
 echo -e "${YELLOW}--->${COLOR_OFF} Ensuring readme passes lint checks (if this fails, run rst-lint)"
-rst-lint README.rst > /dev/null
+rst-lint README.rst || exit 1
 
 echo -e "${YELLOW}--->${COLOR_OFF} Creating necessary temp file"
 tempfoo=$(basename $0)
@@ -90,23 +91,25 @@ find_this="__version__ = \"$current_version\""
 replace_with="__version__ = \"$next_version\""
 
 echo -e "${YELLOW}--->${COLOR_OFF} Updating ${INIT_PACKAGE_NAME}/__init__.py"
-sed "s/$find_this/$replace_with/" ${INIT_PACKAGE_NAME}/__init__.py > $TMPFILE && mv $TMPFILE ${INIT_PACKAGE_NAME}/__init__.py
+sed "s/$find_this/$replace_with/" ${INIT_PACKAGE_NAME}/__init__.py >$TMPFILE && mv $TMPFILE ${INIT_PACKAGE_NAME}/__init__.py
 
 if [ -f docs/conf.py ]; then
     echo -e "${YELLOW}--->${COLOR_OFF} Updating docs"
     find_this="version = '${current_version}'"
     replace_with="version = '${next_version}'"
-    sed "s/$find_this/$replace_with/" docs/conf.py > $TMPFILE && mv $TMPFILE docs/conf.py
+    sed "s/$find_this/$replace_with/" docs/conf.py >$TMPFILE && mv $TMPFILE docs/conf.py
 
     find_this="version = '${current_version}'"
     replace_with="release = '${next_version}'"
-    sed "s/$find_this/$replace_with/" docs/conf.py > $TMPFILE && mv $TMPFILE docs/conf.py
+    sed "s/$find_this/$replace_with/" docs/conf.py >$TMPFILE && mv $TMPFILE docs/conf.py
 fi
 
 echo -e "${YELLOW}--->${COLOR_OFF} Updating CHANGES.rst for new release"
 version_header="$next_version ($(date +%F))"
-set +e; dashes=$(yes '-'|head -n ${#version_header}|tr -d '\n') ; set -e
+set +e
-gitchangelog |sed "4s/.*/$version_header/"|sed "5s/.*/$dashes/" > $TMPFILE && mv $TMPFILE CHANGES.rst
+dashes=$(yes '-' | head -n ${#version_header} | tr -d '\n')
+set -e
+gitchangelog | sed "4s/.*/$version_header/" | sed "5s/.*/$dashes/" >$TMPFILE && mv $TMPFILE CHANGES.rst
 
 echo -e "${YELLOW}--->${COLOR_OFF} Adding changed files to git"
 git add CHANGES.rst README.rst ${INIT_PACKAGE_NAME}/__init__.py
@@ -115,6 +118,15 @@ if [ -f docs/conf.py ]; then git add docs/conf.py; fi
 echo -e "${YELLOW}--->${COLOR_OFF} Creating release"
 git commit -q -m "Release version $next_version"
 
+if [[ "$PUBLIC" == "true" ]]; then
+    echo -e "${YELLOW}--->${COLOR_OFF} Creating python release files"
+    cp README.rst README
+    python setup.py sdist bdist_wheel >/dev/null
+
+    echo -e "${YELLOW}--->${COLOR_OFF} Validating long_description"
+    twine check dist/*
+fi
+
 echo -e "${YELLOW}--->${COLOR_OFF} Tagging release"
 git tag -a $next_version -m "Release version $next_version"
 
@@ -122,9 +134,7 @@ echo -e "${YELLOW}--->${COLOR_OFF} Pushing release and tags to github"
 git push -q origin master && git push -q --tags
 
 if [[ "$PUBLIC" == "true" ]]; then
-    echo -e "${YELLOW}--->${COLOR_OFF} Creating python release"
+    echo -e "${YELLOW}--->${COLOR_OFF} Uploading python release"
-    cp README.rst README
-    python setup.py sdist bdist_wheel > /dev/null
     twine upload dist/*
     rm README
 fi

release-requirements.txt

@@ -0,0 +1,38 @@
+autopep8==2.0.4
+black==23.11.0
+bleach==6.0.0
+certifi==2023.7.22
+charset-normalizer==3.1.0
+click==8.1.7
+colorama==0.4.6
+docutils==0.20.1
+flake8==6.1.0
+gitchangelog==3.0.4
+idna==3.4
+importlib-metadata==6.6.0
+jaraco.classes==3.2.3
+keyring==23.13.1
+markdown-it-py==2.2.0
+mccabe==0.7.0
+mdurl==0.1.2
+more-itertools==9.1.0
+mypy-extensions==1.0.0
+packaging==23.2
+pathspec==0.11.2
+pkginfo==1.9.6
+platformdirs==4.1.0
+pycodestyle==2.11.1
+pyflakes==3.1.0
+Pygments==2.15.1
+readme-renderer==37.3
+requests==2.31.0
+requests-toolbelt==1.0.0
+restructuredtext-lint==1.4.0
+rfc3986==2.0.0
+rich==13.3.5
+six==1.16.0
+tqdm==4.65.0
+twine==4.0.2
+urllib3==2.0.7
+webencodings==0.5.1
+zipp==3.15.0