Release version 0.50.1

chore(deps): bump setuptools from 75.8.1 to 75.8.2 in the python-packages group

.circleci/config.yml

@@ -1,23 +0,0 @@
-version: 2.1
-
-orbs:
-  python: circleci/python@0.3.2
-
-jobs:
-  build-and-test:
-    executor: python/default
-    steps:
-      - checkout
-      - python/load-cache
-      - run:
-          command: pip install flake8
-          name: Install dependencies
-      - python/save-cache
-      - run:
-          command: flake8 --ignore=E501
-          name: Lint
-
-workflows:
-  main:
-    jobs:
-      - build-and-test

.github/dependabot.yml

@@ -0,0 +1,15 @@
+version: 2
+updates:
+- package-ecosystem: pip
+  directory: "/"
+  schedule:
+    interval: daily
+    time: "13:00"
+  groups:
+    python-packages:
+      patterns:
+        - "*"
+- package-ecosystem: "github-actions"
+  directory: "/"
+  schedule:
+    interval: "weekly"

.github/workflows/automatic-release.yml

@@ -0,0 +1,41 @@
+name: automatic-release
+
+on:
+    workflow_dispatch:
+        inputs:
+            release_type:
+                description: Release type
+                required: true
+                type: choice
+                options:
+                    - patch
+                    - minor
+                    - major
+
+jobs:
+    release:
+        name: Release
+        runs-on: ubuntu-24.04
+        steps:
+            - name: Checkout repository
+              uses: actions/checkout@v4
+              with:
+                fetch-depth: 0
+                ssh-key: ${{ secrets.DEPLOY_PRIVATE_KEY }}
+            - name: Setup Git
+              run: |
+                git config --local user.email "action@github.com"
+                git config --local user.name "GitHub Action"
+            - name: Setup Python
+              uses: actions/setup-python@v5
+              with:
+                python-version: '3.12'
+            - name: Install prerequisites
+              run: pip install -r release-requirements.txt
+            - name: Execute release
+              env:
+                SEMVER_BUMP: ${{ github.event.inputs.release_type }}
+                TWINE_REPOSITORY: ${{ vars.TWINE_REPOSITORY }}
+                TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
+                TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
+              run: ./release $SEMVER_BUMP

.github/workflows/docker.yml

@@ -0,0 +1,77 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: Create and publish a Docker image
+
+on:
+  push:
+    branches:
+      - 'master'
+      - 'main'
+      - 'dev'
+
+    tags:
+      - 'v*'
+      - 'v*.*'
+      - 'v*.*.*'
+      - '*'
+      - '*.*'
+      - '*.*.*'
+  pull_request:
+    branches:
+      - 'main'
+      - 'dev'
+
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+            type=sha
+            type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }}
+
+      - name: Build and push Docker image
+        uses:  docker/build-push-action@v6
+        with:
+          context: .
+          push: true
+          platforms: linux/amd64,linux/arm64
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/lint.yml

@@ -0,0 +1,33 @@
+---
+name: "lint"
+
+# yamllint disable-line rule:truthy
+on:
+  pull_request:
+    branches:
+      - "*"
+  push:
+    branches:
+      - "main"
+      - "master"
+
+jobs:
+  lint:
+    name: lint
+    runs-on: ubuntu-24.04
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: "pip"
+      - run: pip install -r release-requirements.txt && pip install wheel
+      - run: flake8 --ignore=E501,E203,W503
+      - run: black .
+      - run: rst-lint README.rst
+      - run: python setup.py sdist bdist_wheel && twine check dist/*

.gitignore

@@ -18,13 +18,13 @@ pkg
 
 # Debian Files
 debian/files
-debian/python-aws-hostname*
+debian/python-github-backup*
 
 # Sphinx build
 doc/_build
 
 # Generated man page
-doc/aws_hostname.1
+doc/github_backup.1
 
 # Annoying macOS files
 .DS_Store
@@ -35,3 +35,10 @@ doc/aws_hostname.1
 .atom
 
 README
+
+# RSA
+id_rsa
+id_rsa.pub
+
+# Virtual env
+venv

Dockerfile

@@ -0,0 +1,16 @@
+FROM python:3.9.18-slim
+
+RUN --mount=type=cache,target=/var/cache/apt \
+    apt-get update && apt-get install -y git git-lfs
+
+WORKDIR /usr/src/app
+
+COPY release-requirements.txt .
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install -r release-requirements.txt
+
+COPY . .
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install .
+
+ENTRYPOINT [ "github-backup" ]

README.rst

@@ -4,14 +4,13 @@ github-backup
 
 |PyPI| |Python Versions|
 
-    This project is considered feature complete for the primary maintainer. If you would like a bugfix or enhancement and cannot sponsor the work, pull requests are welcome. Feel free to contact the maintainer for consulting estimates if desired.
-
-backup a github user or organization
+The package can be used to backup an *entire* `Github <https://github.com/>`_ organization, repository or user account, including starred repos, issues and wikis in the most appropriate format (clones for wikis, json files for issues).
 
 Requirements
 ============
 
 - GIT 1.9+
+- Python
 
 Installation
 ============
@@ -20,28 +19,39 @@ Using PIP via PyPI::
 
     pip install github-backup
 
-Using PIP via Github::
+Using PIP via Github (more likely the latest version)::
 
     pip install git+https://github.com/josegonzalez/python-github-backup.git#egg=github-backup
     
-Usage
-=====
+*Install note for python newcomers:*
 
-CLI Usage is as follows::
+Python scripts are unlikely to be included in your ``$PATH`` by default, this means it cannot be run directly in terminal with ``$ github-backup ...``, you can either add python's install path to your environments ``$PATH`` or call the script directly e.g. using ``$ ~/.local/bin/github-backup``.*
 
-    github-backup [-h] [-u USERNAME] [-p PASSWORD] [-t TOKEN] [--as-app]
-                  [-o OUTPUT_DIRECTORY] [-l LOG_LEVEL] [-i] [--starred]
-                  [--all-starred] [--watched] [--followers] [--following]
-                  [--all] [--issues] [--issue-comments] [--issue-events]
-                  [--pulls] [--pull-comments] [--pull-commits]
-                  [--pull-details] [--labels] [--hooks] [--milestones]
-                  [--repositories] [--bare] [--lfs] [--wikis] [--gists]
-                  [--starred-gists] [--skip-archived] [--skip-existing]
-                  [-L [LANGUAGES ...]] [-N NAME_REGEX] [-H GITHUB_HOST]
-                  [-O] [-R REPOSITORY] [-P] [-F] [--prefer-ssh] [-v]
+Basic Help
+==========
+
+Show the CLI help output::
+
+    github-backup -h
+
+CLI Help output::
+
+    github-backup [-h] [-u USERNAME] [-p PASSWORD] [-t TOKEN_CLASSIC]
+                  [-f TOKEN_FINE] [--as-app] [-o OUTPUT_DIRECTORY]
+                  [-l LOG_LEVEL] [-i] [--starred] [--all-starred]
+                  [--watched] [--followers] [--following] [--all] [--issues]
+                  [--issue-comments] [--issue-events] [--pulls]
+                  [--pull-comments] [--pull-commits] [--pull-details]
+                  [--labels] [--hooks] [--milestones] [--repositories]
+                  [--bare] [--lfs] [--wikis] [--gists] [--starred-gists]
+                  [--skip-archived] [--skip-existing] [-L [LANGUAGES ...]]
+                  [-N NAME_REGEX] [-H GITHUB_HOST] [-O] [-R REPOSITORY]
+                  [-P] [-F] [--prefer-ssh] [-v]
                   [--keychain-name OSX_KEYCHAIN_ITEM_NAME]
                   [--keychain-account OSX_KEYCHAIN_ITEM_ACCOUNT]
-                  [--releases] [--assets] [--exclude [REPOSITORY [REPOSITORY ...]]
+                  [--releases] [--latest-releases NUMBER_OF_LATEST_RELEASES]
+                  [--skip-prerelease] [--assets]
+                  [--exclude [REPOSITORY [REPOSITORY ...]]
                   [--throttle-limit THROTTLE_LIMIT] [--throttle-pause THROTTLE_PAUSE]
                   USER
 
@@ -57,7 +67,10 @@ CLI Usage is as follows::
       -p PASSWORD, --password PASSWORD
                             password for basic auth. If a username is given but
                             not a password, the password will be prompted for.
-      -t TOKEN, --token TOKEN
+      -f TOKEN_FINE, --token-fine TOKEN_FINE
+                            fine-grained personal access token or path to token
+                            (file://...)
+      -t TOKEN_CLASSIC, --token TOKEN_CLASSIC
                             personal access, OAuth, or JSON Web token, or path to
                             token (file://...)
       --as-app              authenticate as github app instead of as a user.
@@ -67,6 +80,7 @@ CLI Usage is as follows::
                             log level to use (default: info, possible levels:
                             debug, info, warning, error, critical)
       -i, --incremental     incremental backup
+      --incremental-by-files incremental backup using modified time of files
       --starred             include JSON output of starred repositories in backup
       --all-starred         include starred repositories in backup [*]
       --watched             include JSON output of watched repositories in backup
@@ -113,6 +127,10 @@ CLI Usage is as follows::
                             keychain that holds the personal access or OAuth token
       --releases            include release information, not including assets or
                             binaries
+      --latest-releases NUMBER_OF_LATEST_RELEASES
+                            include certain number of the latest releases;
+                            only applies if including releases
+      --skip-prerelease     skip prerelease and draft versions; only applies if including releases
       --assets              include assets alongside release information; only
                             applies if including releases
       --exclude [REPOSITORY [REPOSITORY ...]]
@@ -126,15 +144,43 @@ CLI Usage is as follows::
                             --throttle-limit to be set)
 
 
-The package can be used to backup an *entire* organization or repository, including issues and wikis in the most appropriate format (clones for wikis, json files for issues).
+Usage Details
+=============
 
 Authentication
-==============
+--------------
+
+**Password-based authentication** will fail if you have two-factor authentication enabled, and will `be deprecated <https://github.blog/2023-03-09-raising-the-bar-for-software-security-github-2fa-begins-march-13/>`_ by 2023 EOY.
+
+``--username`` is used for basic password authentication and separate from the positional argument ``USER``, which specifies the user account you wish to back up.
+
+**Classic tokens** are `slightly less secure <https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#personal-access-tokens-classic>`_ as they provide very coarse-grained permissions.
+
+If you need authentication for long-running backups (e.g. for a cron job) it is recommended to use **fine-grained personal access token** ``-f TOKEN_FINE``.
+
+
+Fine Tokens
+~~~~~~~~~~~
+
+You can "generate new token", choosing the repository scope by selecting specific repos or all repos. On Github this is under *Settings -> Developer Settings -> Personal access tokens -> Fine-grained Tokens*
+
+Customise the permissions for your use case, but for a personal account full backup you'll need to enable the following permissions:
+
+**User permissions**: Read access to followers, starring, and watching.
+
+**Repository permissions**: Read access to contents, issues, metadata, pull requests, and webhooks.
+
+
+Prefer SSH
+~~~~~~~~~~
+
+If cloning repos is enabled with ``--repositories``, ``--all-starred``, ``--wikis``, ``--gists``, ``--starred-gists`` using the ``--prefer-ssh`` argument will use ssh for cloning the git repos, but all other connections will still use their own protocol, e.g. API requests for issues uses HTTPS.
+
+To clone with SSH, you'll need SSH authentication setup `as usual with Github <https://docs.github.com/en/authentication/connecting-to-github-with-ssh>`_, e.g. via SSH public and private keys.
 
-Note: Password-based authentication will fail if you have two-factor authentication enabled.
 
 Using the Keychain on Mac OSX
-=============================
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Note: On Mac OSX the token can be stored securely in the user's keychain. To do this:
 
 1. Open Keychain from "Applications -> Utilities -> Keychain Access"
@@ -148,31 +194,142 @@ Note:  When you run github-backup, you will be asked whether you want to allow "
 1. **Allow:** In this case you will need to click "Allow" each time you run `github-backup`
 2. **Always Allow:** In this case, you will not be asked for permission when you run `github-backup` in future. This is less secure, but is required if you want to schedule `github-backup` to run automatically
 
-About Git LFS
-=============
 
-When you use the "--lfs" option, you will need to make sure you have Git LFS installed.
+Github Rate-limit and Throttling
+--------------------------------
+
+"github-backup" will automatically throttle itself based on feedback from the Github API. 
+
+Their API is usually rate-limited to 5000 calls per hour. The API will ask github-backup to pause until a specific time when the limit is reset again (at the start of the next hour). This continues until the backup is complete.
+
+During a large backup, such as ``--all-starred``, and on a fast connection this can result in (~20 min) pauses with bursts of API calls periodically maxing out the API limit. If this is not suitable `it has been observed <https://github.com/josegonzalez/python-github-backup/issues/76#issuecomment-636158717>`_ under real-world conditions that overriding the throttle with ``--throttle-limit 5000 --throttle-pause 0.6`` provides a smooth rate across the hour, although a ``--throttle-pause 0.72`` (3600 seconds [1 hour] / 5000 limit) is theoretically safer to prevent large rate-limit pauses.
+
+
+About Git LFS
+-------------
+
+When you use the ``--lfs`` option, you will need to make sure you have Git LFS installed.
 
 Instructions on how to do this can be found on https://git-lfs.github.com.
 
-Examples
-========
 
-Backup all repositories, including private ones::
+Run in Docker container
+-----------------------
+
+To run the tool in a Docker container use the following command:
+
+    sudo docker run --rm -v /path/to/backup:/data --name github-backup ghcr.io/josegonzalez/python-github-backup -o /data $OPTIONS $USER
+
+Gotchas / Known-issues
+======================
+
+All is not everything
+---------------------
+
+The ``--all`` argument does not include: cloning private repos (``-P, --private``), cloning forks (``-F, --fork``), cloning starred repositories (``--all-starred``), ``--pull-details``, cloning LFS repositories (``--lfs``), cloning gists (``--gists``) or cloning starred gist repos (``--starred-gists``). See examples for more.
+
+Cloning all starred size
+------------------------
+
+Using the ``--all-starred`` argument to clone all starred repositories may use a large amount of storage space, especially if ``--all`` or more arguments are used. e.g. commonly starred repos can have tens of thousands of issues, many large assets and the repo itself etc. Consider just storing links to starred repos in JSON format with ``--starred``.
+
+Incremental Backup
+------------------
+
+Using (``-i, --incremental``) will only request new data from the API **since the last run (successful or not)**. e.g. only request issues from the API since the last run. 
+
+This means any blocking errors on previous runs can cause a large amount of missing data in backups.
+
+Using (``--incremental-by-files``) will request new data from the API **based on when the file was modified on filesystem**. e.g. if you modify the file yourself you may miss something.
+
+Still saver than the previous version.
+
+Specifically, issues and pull requests are handled like this.
+
+Known blocking errors
+---------------------
+
+Some errors will block the backup run by exiting the script. e.g. receiving a 403 Forbidden error from the Github API.
+
+If the incremental argument is used, this will result in the next backup only requesting API data since the last blocked/failed run. Potentially causing unexpected large amounts of missing data.
+
+It's therefore recommended to only use the incremental argument if the output/result is being actively monitored, or complimented with periodic full non-incremental runs, to avoid unexpected missing data in a regular backup runs.
+
+1. **Starred public repo hooks blocking**
+
+   Since the ``--all`` argument includes ``--hooks``, if you use ``--all`` and ``--all-starred`` together to clone a users starred public repositories, the backup will likely error and block the backup continuing. 
+
+   This is due to needing the correct permission for ``--hooks`` on public repos.
+
+
+"bare" is actually "mirror"
+---------------------------
+
+Using the bare clone argument (``--bare``) will actually call git's ``clone --mirror`` command. There's a subtle difference between `bare <https://www.git-scm.com/docs/git-clone#Documentation/git-clone.txt---bare>`_ and `mirror <https://www.git-scm.com/docs/git-clone#Documentation/git-clone.txt---mirror>`_ clone.
+
+*From git docs "Compared to --bare, --mirror not only maps local branches of the source to local branches of the target, it maps all refs (including remote-tracking branches, notes etc.) and sets up a refspec configuration such that all these refs are overwritten by a git remote update in the target repository."*
+
+
+Starred gists vs starred repo behaviour
+---------------------------------------
+
+The starred normal repo cloning (``--all-starred``) argument stores starred repos separately to the users own repositories. However, using ``--starred-gists`` will store starred gists within the same directory as the users own gists ``--gists``. Also, all gist repo directory names are IDs not the gist's name.
+
+
+Skip existing on incomplete backups
+-----------------------------------
+
+The ``--skip-existing`` argument will skip a backup if the directory already exists, even if the backup in that directory failed (perhaps due to a blocking error). This may result in unexpected missing data in a regular backup.
+
+
+Github Backup Examples
+======================
+
+Backup all repositories, including private ones using a classic token::
 
     export ACCESS_TOKEN=SOME-GITHUB-TOKEN
     github-backup WhiteHouse --token $ACCESS_TOKEN --organization --output-directory /tmp/white-house --repositories --private
 
-Backup a single organization repository with everything else (wiki, pull requests, comments, issues etc)::
+Use a fine-grained access token to backup a single organization repository with everything else (wiki, pull requests, comments, issues etc)::
 
-    export ACCESS_TOKEN=SOME-GITHUB-TOKEN
+    export FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
     ORGANIZATION=docker
     REPO=cli
     # e.g. git@github.com:docker/cli.git
-    github-backup $ORGANIZATION -P -t $ACCESS_TOKEN -o . --all -O -R $REPO
+    github-backup $ORGANIZATION -P -f $FINE_ACCESS_TOKEN -o . --all -O -R $REPO
+
+Quietly and incrementally backup useful Github user data (public and private repos with SSH) including; all issues, pulls, all public starred repos and gists (omitting "hooks", "releases" and therefore "assets" to prevent blocking). *Great for a cron job.* ::
+
+    export FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
+    GH_USER=YOUR-GITHUB-USER
+
+    github-backup -f $FINE_ACCESS_TOKEN --prefer-ssh -o ~/github-backup/ -l error -P -i --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-commits --labels --milestones --repositories --wikis --releases --assets --pull-details --gists --starred-gists $GH_USER
+    
+Debug an error/block or incomplete backup into a temporary directory. Omit "incremental" to fill a previous incomplete backup. ::
+
+    export FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
+    GH_USER=YOUR-GITHUB-USER
+
+    github-backup -f $FINE_ACCESS_TOKEN -o /tmp/github-backup/ -l debug -P --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-commits --labels --milestones --repositories --wikis --releases --assets --pull-details --gists --starred-gists $GH_USER
+
+
+
+Development
+===========
+
+This project is considered feature complete for the primary maintainer @josegonzalez. If you would like a bugfix or enhancement, pull requests are welcome. Feel free to contact the maintainer for consulting estimates if you'd like to sponsor the work instead.
+
+Contibuters
+-----------
+
+A huge thanks to all the contibuters!
+
+.. image:: https://contrib.rocks/image?repo=josegonzalez/python-github-backup
+   :target: https://github.com/josegonzalez/python-github-backup/graphs/contributors
+   :alt: contributors
 
 Testing
-=======
+-------
 
 This project currently contains no unit tests.  To run linting::
 

bin/github-backup

@@ -1,6 +1,8 @@
 #!/usr/bin/env python
 
-import os, sys, logging
+import logging
+import os
+import sys
 
 from github_backup.github_backup import (
     backup_account,
@@ -8,25 +10,28 @@ from github_backup.github_backup import (
     check_git_lfs_install,
     filter_repositories,
     get_authenticated_user,
-    log_info,
-    log_warning,
+    logger,
     mkdir_p,
     parse_args,
     retrieve_repositories,
 )
 
 logging.basicConfig(
-    format='%(asctime)s.%(msecs)03d: %(message)s',
-    datefmt='%Y-%m-%dT%H:%M:%S',
-    level=logging.INFO
+    format="%(asctime)s.%(msecs)03d: %(message)s",
+    datefmt="%Y-%m-%dT%H:%M:%S",
+    level=logging.INFO,
 )
 
+
 def main():
     args = parse_args()
 
+    if args.quiet:
+        logger.setLevel(logging.WARNING)
+
     output_directory = os.path.realpath(args.output_directory)
     if not os.path.isdir(output_directory):
-        log_info('Create output directory {0}'.format(output_directory))
+        logger.info("Create output directory {0}".format(output_directory))
         mkdir_p(output_directory)
 
     if args.lfs_clone:
@@ -35,13 +40,13 @@ def main():
     if args.log_level:
         log_level = logging.getLevelName(args.log_level.upper())
         if isinstance(log_level, int):
-            logging.root.setLevel(log_level)
+            logger.root.setLevel(log_level)
 
     if not args.as_app:
-        log_info('Backing up user {0} to {1}'.format(args.user, output_directory))
+        logger.info("Backing up user {0} to {1}".format(args.user, output_directory))
         authenticated_user = get_authenticated_user(args)
     else:
-        authenticated_user = {'login': None}
+        authenticated_user = {"login": None}
 
     repositories = retrieve_repositories(args, authenticated_user)
     repositories = filter_repositories(args, repositories)
@@ -49,9 +54,9 @@ def main():
     backup_account(args, output_directory)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     try:
         main()
     except Exception as e:
-        log_warning(str(e))
+        logger.error(str(e))
         sys.exit(1)

github_backup/__init__.py

@@ -1 +1 @@
-__version__ = "0.43.0"
+__version__ = "0.50.1"

github_backup/github_backup.py

@@ -1,33 +1,30 @@
 #!/usr/bin/env python
 
 from __future__ import print_function
-import socket
 
 import argparse
 import base64
 import calendar
 import codecs
-import datetime
 import errno
 import getpass
 import json
+import logging
 import os
+import platform
 import re
 import select
+import socket
+import ssl
 import subprocess
 import sys
-import logging
 import time
-import platform
-from urllib.parse import urlparse
-from urllib.parse import quote as urlquote
-from urllib.parse import urlencode
-from urllib.error import HTTPError, URLError
-from urllib.request import urlopen
-from urllib.request import Request
-from urllib.request import HTTPRedirectHandler
-from urllib.request import build_opener
+from datetime import datetime
 from http.client import IncompleteRead
+from urllib.error import HTTPError, URLError
+from urllib.parse import quote as urlquote
+from urllib.parse import urlencode, urlparse
+from urllib.request import HTTPRedirectHandler, Request, build_opener, urlopen
 
 try:
     from . import __version__
@@ -37,40 +34,29 @@ except ImportError:
     VERSION = "unknown"
 
 FNULL = open(os.devnull, "w")
+FILE_URI_PREFIX = "file://"
+logger = logging.getLogger(__name__)
 
+https_ctx = ssl.create_default_context()
+if not https_ctx.get_ca_certs():
+    import warnings
 
-def _get_log_date():
-    return datetime.datetime.isoformat(datetime.datetime.now())
+    warnings.warn(
+        "\n\nYOUR DEFAULT CA CERTS ARE EMPTY.\n"
+        + "PLEASE POPULATE ANY OF:"
+        + "".join(
+            ["\n - " + x for x in ssl.get_default_verify_paths() if type(x) is str]
+        )
+        + "\n",
+        stacklevel=2,
+    )
+    import certifi
 
-
-def log_info(message):
-    """
-    Log message (str) or messages (List[str]) to stdout
-    """
-    if type(message) == str:
-        message = [message]
-
-    for msg in message:
-        logging.info(msg)
-
-
-def log_warning(message):
-    """
-    Log message (str) or messages (List[str]) to stderr
-    """
-    if type(message) == str:
-        message = [message]
-
-    for msg in message:
-        logging.warning(msg)
+    https_ctx = ssl.create_default_context(cafile=certifi.where())
 
 
 def logging_subprocess(
-    popenargs,
-    logger,
-    stdout_log_level=logging.DEBUG,
-    stderr_log_level=logging.ERROR,
-    **kwargs
+    popenargs, stdout_log_level=logging.DEBUG, stderr_log_level=logging.ERROR, **kwargs
 ):
     """
     Variant of subprocess.call that accepts a logger instead of stdout/stderr,
@@ -81,7 +67,7 @@ def logging_subprocess(
         popenargs, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs
     )
     if sys.platform == "win32":
-        log_info(
+        logger.info(
             "Windows operating system detected - no subprocess logging will be returned"
         )
 
@@ -152,9 +138,22 @@ def parse_args(args=None):
     parser.add_argument(
         "-t",
         "--token",
-        dest="token",
+        dest="token_classic",
         help="personal access, OAuth, or JSON Web token, or path to token (file://...)",
     )  # noqa
+    parser.add_argument(
+        "-f",
+        "--token-fine",
+        dest="token_fine",
+        help="fine-grained personal access token (github_pat_....), or path to token (file://...)",
+    )  # noqa
+    parser.add_argument(
+        "-q",
+        "--quiet",
+        action="store_true",
+        dest="quiet",
+        help="supress log messages less severe than warning, e.g. info",
+    )
     parser.add_argument(
         "--as-app",
         action="store_true",
@@ -182,6 +181,12 @@ def parse_args(args=None):
         dest="incremental",
         help="incremental backup",
     )
+    parser.add_argument(
+        "--incremental-by-files",
+        action="store_true",
+        dest="incremental_by_files",
+        help="incremental backup based on modification date of files",
+    )
     parser.add_argument(
         "--starred",
         action="store_true",
@@ -396,6 +401,19 @@ def parse_args(args=None):
         dest="include_releases",
         help="include release information, not including assets or binaries",
     )
+    parser.add_argument(
+        "--latest-releases",
+        type=int,
+        default=0,
+        dest="number_of_latest_releases",
+        help="include certain number of the latest releases; only applies if including releases",
+    )
+    parser.add_argument(
+        "--skip-prerelease",
+        action="store_true",
+        dest="skip_prerelease",
+        help="skip prerelease and draft versions; only applies if including releases",
+    )
     parser.add_argument(
         "--assets",
         action="store_true",
@@ -457,18 +475,27 @@ def get_auth(args, encode=True, for_git_cli=False):
         raise Exception(
             "You must specify both name and account fields for osx keychain password items"
         )
-    elif args.token:
-        _path_specifier = "file://"
-        if args.token.startswith(_path_specifier):
-            path_specifier_len = len(_path_specifier)
-            args.token = open(args.token[path_specifier_len:], "rt").readline().strip()
+    elif args.token_fine:
+        if args.token_fine.startswith(FILE_URI_PREFIX):
+            args.token_fine = read_file_contents(args.token_fine)
+
+        if args.token_fine.startswith("github_pat_"):
+            auth = args.token_fine
+        else:
+            raise Exception(
+                "Fine-grained token supplied does not look like a GitHub PAT"
+            )
+    elif args.token_classic:
+        if args.token_classic.startswith(FILE_URI_PREFIX):
+            args.token_classic = read_file_contents(args.token_classic)
+
         if not args.as_app:
-            auth = args.token + ":" + "x-oauth-basic"
+            auth = args.token_classic + ":" + "x-oauth-basic"
         else:
             if not for_git_cli:
-                auth = args.token
+                auth = args.token_classic
             else:
-                auth = "x-access-token:" + args.token
+                auth = "x-access-token:" + args.token_classic
     elif args.username:
         if not args.password:
             args.password = getpass.getpass()
@@ -483,7 +510,7 @@ def get_auth(args, encode=True, for_git_cli=False):
     if not auth:
         return None
 
-    if not encode:
+    if not encode or args.token_fine is not None:
         return auth
 
     return base64.b64encode(auth.encode("ascii"))
@@ -507,6 +534,10 @@ def get_github_host(args):
     return host
 
 
+def read_file_contents(file_uri):
+    return open(file_uri[len(FILE_URI_PREFIX) :], "rt").readline().strip()
+
+
 def get_github_repo_url(args, repository):
     if repository.get("is_gist"):
         if args.prefer_ssh:
@@ -529,7 +560,7 @@ def get_github_repo_url(args, repository):
     auth = get_auth(args, encode=False, for_git_cli=True)
     if auth:
         repo_url = "https://{0}@{1}/{2}/{3}.git".format(
-            auth,
+            auth if args.token_fine is None else "oauth2:" + auth,
             get_github_host(args),
             repository["owner"]["login"],
             repository["name"],
@@ -549,7 +580,13 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False):
     while True:
         page = page + 1
         request = _construct_request(
-            per_page, page, query_args, template, auth, as_app=args.as_app
+            per_page,
+            page,
+            query_args,
+            template,
+            auth,
+            as_app=args.as_app,
+            fine=True if args.token_fine is not None else False,
         )  # noqa
         r, errors = _get_response(request, auth, template)
 
@@ -558,13 +595,13 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False):
         try:
             response = json.loads(r.read().decode("utf-8"))
         except IncompleteRead:
-            log_warning("Incomplete read error detected")
+            logger.warning("Incomplete read error detected")
             read_error = True
         except json.decoder.JSONDecodeError:
-            log_warning("JSON decode error detected")
+            logger.warning("JSON decode error detected")
             read_error = True
         except TimeoutError:
-            log_warning("Tiemout error detected")
+            logger.warning("Tiemout error detected")
             read_error = True
         else:
             read_error = False
@@ -572,7 +609,7 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False):
         # be gentle with API request limit and throttle requests if remaining requests getting low
         limit_remaining = int(r.headers.get("x-ratelimit-remaining", 0))
         if args.throttle_limit and limit_remaining <= args.throttle_limit:
-            log_info(
+            logger.info(
                 "API request limit hit: {} requests left, pausing further requests for {}s".format(
                     limit_remaining, args.throttle_pause
                 )
@@ -581,11 +618,17 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False):
 
         retries = 0
         while retries < 3 and (status_code == 502 or read_error):
-            log_warning("API request failed. Retrying in 5 seconds")
+            logger.warning("API request failed. Retrying in 5 seconds")
             retries += 1
             time.sleep(5)
             request = _construct_request(
-                per_page, page, query_args, template, auth, as_app=args.as_app
+                per_page,
+                page,
+                query_args,
+                template,
+                auth,
+                as_app=args.as_app,
+                fine=True if args.token_fine is not None else False,
             )  # noqa
             r, errors = _get_response(request, auth, template)
 
@@ -594,13 +637,13 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False):
                 response = json.loads(r.read().decode("utf-8"))
                 read_error = False
             except IncompleteRead:
-                log_warning("Incomplete read error detected")
+                logger.warning("Incomplete read error detected")
                 read_error = True
             except json.decoder.JSONDecodeError:
-                log_warning("JSON decode error detected")
+                logger.warning("JSON decode error detected")
                 read_error = True
             except TimeoutError:
-                log_warning("Tiemout error detected")
+                logger.warning("Tiemout error detected")
                 read_error = True
 
         if status_code != 200:
@@ -614,12 +657,12 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False):
             raise Exception(", ".join(errors))
 
         if len(errors) == 0:
-            if type(response) == list:
+            if type(response) is list:
                 for resp in response:
                     yield resp
                 if len(response) < per_page:
                     break
-            elif type(response) == dict and single_request:
+            elif type(response) is dict and single_request:
                 yield response
 
         if len(errors) > 0:
@@ -647,17 +690,17 @@ def _get_response(request, auth, template):
     while True:
         should_continue = False
         try:
-            r = urlopen(request)
+            r = urlopen(request, context=https_ctx)
         except HTTPError as exc:
             errors, should_continue = _request_http_error(exc, auth, errors)  # noqa
             r = exc
         except URLError as e:
-            log_warning(e.reason)
+            logger.warning(e.reason)
             should_continue, retry_timeout = _request_url_error(template, retry_timeout)
             if not should_continue:
                 raise
         except socket.error as e:
-            log_warning(e.strerror)
+            logger.warning(e.strerror)
             should_continue, retry_timeout = _request_url_error(template, retry_timeout)
             if not should_continue:
                 raise
@@ -669,16 +712,22 @@ def _get_response(request, auth, template):
     return r, errors
 
 
-def _construct_request(per_page, page, query_args, template, auth, as_app=None):
+def _construct_request(
+    per_page, page, query_args, template, auth, as_app=None, fine=False
+):
     querystring = urlencode(
         dict(
-            list({"per_page": per_page, "page": page}.items()) + list(query_args.items())
+            list({"per_page": per_page, "page": page}.items())
+            + list(query_args.items())
         )
     )
 
     request = Request(template + "?" + querystring)
     if auth is not None:
         if not as_app:
+            if fine:
+                request.add_header("Authorization", "token " + auth)
+            else:
                 request.add_header("Authorization", "Basic ".encode("ascii") + auth)
         else:
             auth = auth.encode("ascii")
@@ -686,7 +735,7 @@ def _construct_request(per_page, page, query_args, template, auth, as_app=None):
             request.add_header(
                 "Accept", "application/vnd.github.machine-man-preview+json"
             )
-    log_info("Requesting {}?{}".format(template, querystring))
+    logger.info("Requesting {}?{}".format(template, querystring))
     return request
 
 
@@ -710,14 +759,14 @@ def _request_http_error(exc, auth, errors):
         delta = max(10, reset - gm_now)
 
         limit = headers.get("x-ratelimit-limit")
-        log_warning(
+        logger.warning(
             "Exceeded rate limit of {} requests; waiting {} seconds to reset".format(
                 limit, delta
             )
         )  # noqa
 
         if auth is None:
-            log_info("Hint: Authenticate to raise your GitHub rate limit")
+            logger.info("Hint: Authenticate to raise your GitHub rate limit")
 
         time.sleep(delta)
         should_continue = True
@@ -727,13 +776,13 @@ def _request_http_error(exc, auth, errors):
 def _request_url_error(template, retry_timeout):
     # In case of a connection timing out, we can retry a few time
     # But we won't crash and not back-up the rest now
-    log_info("{} timed out".format(template))
+    logger.info("'{}' timed out".format(template))
     retry_timeout -= 1
 
     if retry_timeout >= 0:
         return True, retry_timeout
 
-    raise Exception("{} timed out to much, skipping!")
+    raise Exception("'{}' timed out to much, skipping!".format(template))
 
 
 class S3HTTPRedirectHandler(HTTPRedirectHandler):
@@ -752,14 +801,21 @@ class S3HTTPRedirectHandler(HTTPRedirectHandler):
         return request
 
 
-def download_file(url, path, auth):
+def download_file(url, path, auth, as_app=False, fine=False):
     # Skip downloading release assets if they already exist on disk so we don't redownload on every sync
     if os.path.exists(path):
         return
 
-    request = Request(url)
+    request = _construct_request(
+        per_page=100,
+        page=1,
+        query_args={},
+        template=url,
+        auth=auth,
+        as_app=as_app,
+        fine=fine,
+    )
     request.add_header("Accept", "application/octet-stream")
-    request.add_header("Authorization", "Basic ".encode("ascii") + auth)
     opener = build_opener(S3HTTPRedirectHandler)
 
     try:
@@ -774,20 +830,20 @@ def download_file(url, path, auth):
                 f.write(chunk)
     except HTTPError as exc:
         # Gracefully handle 404 responses (and others) when downloading from S3
-        log_warning(
+        logger.warning(
             "Skipping download of asset {0} due to HTTPError: {1}".format(
                 url, exc.reason
             )
         )
     except URLError as e:
         # Gracefully handle other URL errors
-        log_warning(
+        logger.warning(
             "Skipping download of asset {0} due to URLError: {1}".format(url, e.reason)
         )
     except socket.error as e:
         # Gracefully handle socket errors
         # TODO: Implement retry logic
-        log_warning(
+        logger.warning(
             "Skipping download of asset {0} due to socker error: {1}".format(
                 url, e.strerror
             )
@@ -809,14 +865,14 @@ def check_git_lfs_install():
 
 
 def retrieve_repositories(args, authenticated_user):
-    log_info("Retrieving repositories")
+    logger.info("Retrieving repositories")
     single_request = False
     if args.user == authenticated_user["login"]:
         # we must use the /user/repos API to be able to access private repos
         template = "https://{0}/user/repos".format(get_github_api_host(args))
     else:
         if args.private and not args.organization:
-            log_warning(
+            logger.warning(
                 "Authenticated user is different from user being backed up, thus private repositories cannot be accessed"
             )
         template = "https://{0}/users/{1}/repos".format(
@@ -872,7 +928,7 @@ def retrieve_repositories(args, authenticated_user):
 
 
 def filter_repositories(args, unfiltered_repositories):
-    log_info("Filtering repositories")
+    logger.info("Filtering repositories")
 
     repositories = []
     for r in unfiltered_repositories:
@@ -901,17 +957,21 @@ def filter_repositories(args, unfiltered_repositories):
             if r.get("language") and r.get("language").lower() in languages
         ]  # noqa
     if name_regex:
-        repositories = [r for r in repositories if name_regex.match(r["name"])]
+        repositories = [
+            r for r in repositories if "name" not in r or name_regex.match(r["name"])
+        ]
     if args.skip_archived:
         repositories = [r for r in repositories if not r.get("archived")]
     if args.exclude:
-        repositories = [r for r in repositories if r["name"] not in args.exclude]
+        repositories = [
+            r for r in repositories if "name" not in r or r["name"] not in args.exclude
+        ]
 
     return repositories
 
 
 def backup_repositories(args, output_directory, repositories):
-    log_info("Backing up repositories")
+    logger.info("Backing up repositories")
     repos_template = "https://{0}/repos".format(get_github_api_host(args))
 
     if args.incremental:
@@ -1023,7 +1083,7 @@ def backup_issues(args, repo_cwd, repository, repos_template):
     if args.skip_existing and has_issues_dir:
         return
 
-    log_info("Retrieving {0} issues".format(repository["full_name"]))
+    logger.info("Retrieving {0} issues".format(repository["full_name"]))
     issue_cwd = os.path.join(repo_cwd, "issues")
     mkdir_p(repo_cwd, issue_cwd)
 
@@ -1052,7 +1112,7 @@ def backup_issues(args, repo_cwd, repository, repos_template):
     if issues_skipped:
         issues_skipped_message = " (skipped {0} pull requests)".format(issues_skipped)
 
-    log_info(
+    logger.info(
         "Saving {0} issues to disk{1}".format(
             len(list(issues.keys())), issues_skipped_message
         )
@@ -1060,6 +1120,14 @@ def backup_issues(args, repo_cwd, repository, repos_template):
     comments_template = _issue_template + "/{0}/comments"
     events_template = _issue_template + "/{0}/events"
     for number, issue in list(issues.items()):
+        issue_file = "{0}/{1}.json".format(issue_cwd, number)
+        if args.incremental_by_files and os.path.isfile(issue_file):
+            modified = os.path.getmtime(issue_file)
+            modified = datetime.fromtimestamp(modified).strftime("%Y-%m-%dT%H:%M:%SZ")
+            if modified > issue["updated_at"]:
+                logger.info("Skipping issue {0} because it wasn't modified since last backup".format(number))
+                continue
+
         if args.include_issue_comments or args.include_everything:
             template = comments_template.format(number)
             issues[number]["comment_data"] = retrieve_data(args, template)
@@ -1067,9 +1135,9 @@ def backup_issues(args, repo_cwd, repository, repos_template):
             template = events_template.format(number)
             issues[number]["event_data"] = retrieve_data(args, template)
 
-        issue_file = "{0}/{1}.json".format(issue_cwd, number)
-        with codecs.open(issue_file, "w", encoding="utf-8") as f:
+        with codecs.open(issue_file + ".temp", "w", encoding="utf-8") as f:
             json_dump(issue, f)
+            os.rename(issue_file + ".temp", issue_file)  # Unlike json_dump, this is atomic
 
 
 def backup_pulls(args, repo_cwd, repository, repos_template):
@@ -1077,7 +1145,7 @@ def backup_pulls(args, repo_cwd, repository, repos_template):
     if args.skip_existing and has_pulls_dir:
         return
 
-    log_info("Retrieving {0} pull requests".format(repository["full_name"]))  # noqa
+    logger.info("Retrieving {0} pull requests".format(repository["full_name"]))  # noqa
     pulls_cwd = os.path.join(repo_cwd, "pulls")
     mkdir_p(repo_cwd, pulls_cwd)
 
@@ -1113,7 +1181,7 @@ def backup_pulls(args, repo_cwd, repository, repos_template):
                     single_request=True,
                 )[0]
 
-    log_info("Saving {0} pull requests to disk".format(len(list(pulls.keys()))))
+    logger.info("Saving {0} pull requests to disk".format(len(list(pulls.keys()))))
     # Comments from pulls API are only _review_ comments
     # regular comments need to be fetched via issue API.
     # For backwards compatibility with versions <= 0.41.0
@@ -1122,6 +1190,13 @@ def backup_pulls(args, repo_cwd, repository, repos_template):
     comments_template = _pulls_template + "/{0}/comments"
     commits_template = _pulls_template + "/{0}/commits"
     for number, pull in list(pulls.items()):
+        pull_file = "{0}/{1}.json".format(pulls_cwd, number)
+        if args.incremental_by_files and os.path.isfile(pull_file):
+            modified = os.path.getmtime(pull_file)
+            modified = datetime.fromtimestamp(modified).strftime("%Y-%m-%dT%H:%M:%SZ")
+            if modified > pull["updated_at"]:
+                logger.info("Skipping pull request {0} because it wasn't modified since last backup".format(number))
+                continue
         if args.include_pull_comments or args.include_everything:
             template = comments_regular_template.format(number)
             pulls[number]["comment_regular_data"] = retrieve_data(args, template)
@@ -1131,9 +1206,9 @@ def backup_pulls(args, repo_cwd, repository, repos_template):
             template = commits_template.format(number)
             pulls[number]["commit_data"] = retrieve_data(args, template)
 
-        pull_file = "{0}/{1}.json".format(pulls_cwd, number)
-        with codecs.open(pull_file, "w", encoding="utf-8") as f:
+        with codecs.open(pull_file + ".temp", "w", encoding="utf-8") as f:
             json_dump(pull, f)
+            os.rename(pull_file + ".temp", pull_file)  # Unlike json_dump, this is atomic
 
 
 def backup_milestones(args, repo_cwd, repository, repos_template):
@@ -1141,7 +1216,7 @@ def backup_milestones(args, repo_cwd, repository, repos_template):
     if args.skip_existing and os.path.isdir(milestone_cwd):
         return
 
-    log_info("Retrieving {0} milestones".format(repository["full_name"]))
+    logger.info("Retrieving {0} milestones".format(repository["full_name"]))
     mkdir_p(repo_cwd, milestone_cwd)
 
     template = "{0}/{1}/milestones".format(repos_template, repository["full_name"])
@@ -1154,7 +1229,7 @@ def backup_milestones(args, repo_cwd, repository, repos_template):
     for milestone in _milestones:
         milestones[milestone["number"]] = milestone
 
-    log_info("Saving {0} milestones to disk".format(len(list(milestones.keys()))))
+    logger.info("Saving {0} milestones to disk".format(len(list(milestones.keys()))))
     for number, milestone in list(milestones.items()):
         milestone_file = "{0}/{1}.json".format(milestone_cwd, number)
         with codecs.open(milestone_file, "w", encoding="utf-8") as f:
@@ -1171,15 +1246,18 @@ def backup_labels(args, repo_cwd, repository, repos_template):
 def backup_hooks(args, repo_cwd, repository, repos_template):
     auth = get_auth(args)
     if not auth:
-        log_info("Skipping hooks since no authentication provided")
+        logger.info("Skipping hooks since no authentication provided")
         return
     hook_cwd = os.path.join(repo_cwd, "hooks")
     output_file = "{0}/hooks.json".format(hook_cwd)
     template = "{0}/{1}/hooks".format(repos_template, repository["full_name"])
     try:
         _backup_data(args, "hooks", template, output_file, hook_cwd)
-    except SystemExit:
-        log_info("Unable to read hooks, skipping")
+    except Exception as e:
+        if "404" in str(e):
+            logger.info("Unable to read hooks, skipping")
+        else:
+            raise e
 
 
 def backup_releases(args, repo_cwd, repository, repos_template, include_assets=False):
@@ -1187,7 +1265,7 @@ def backup_releases(args, repo_cwd, repository, repos_template, include_assets=F
 
     # give release files somewhere to live & log intent
     release_cwd = os.path.join(repo_cwd, "releases")
-    log_info("Retrieving {0} releases".format(repository_fullname))
+    logger.info("Retrieving {0} releases".format(repository_fullname))
     mkdir_p(repo_cwd, release_cwd)
 
     query_args = {}
@@ -1195,8 +1273,24 @@ def backup_releases(args, repo_cwd, repository, repos_template, include_assets=F
     release_template = "{0}/{1}/releases".format(repos_template, repository_fullname)
     releases = retrieve_data(args, release_template, query_args=query_args)
 
+    if args.skip_prerelease:
+        releases = [r for r in releases if not r["prerelease"] and not r["draft"]]
+
+    if args.number_of_latest_releases and args.number_of_latest_releases < len(
+        releases
+    ):
+        releases.sort(
+            key=lambda item: datetime.strptime(
+                item["created_at"], "%Y-%m-%dT%H:%M:%SZ"
+            ),
+            reverse=True,
+        )
+        releases = releases[: args.number_of_latest_releases]
+        logger.info("Saving the latest {0} releases to disk".format(len(releases)))
+    else:
+        logger.info("Saving {0} releases to disk".format(len(releases)))
+
     # for each release, store it
-    log_info("Saving {0} releases to disk".format(len(releases)))
     for release in releases:
         release_name = release["tag_name"]
         release_name_safe = release_name.replace("/", "__")
@@ -1216,7 +1310,9 @@ def backup_releases(args, repo_cwd, repository, repos_template, include_assets=F
                     download_file(
                         asset["url"],
                         os.path.join(release_assets_cwd, asset["name"]),
-                        get_auth(args),
+                        get_auth(args, encode=not args.as_app),
+                        as_app=args.as_app,
+                        fine=True if args.token_fine is not None else False,
                     )
 
 
@@ -1251,7 +1347,7 @@ def fetch_repository(
         "git ls-remote " + remote_url, stdout=FNULL, stderr=FNULL, shell=True
     )
     if initialized == 128:
-        log_info(
+        logger.info(
             "Skipping {0} ({1}) since it's not initialized".format(
                 name, masked_remote_url
             )
@@ -1259,47 +1355,49 @@ def fetch_repository(
         return
 
     if clone_exists:
-        log_info("Updating {0} in {1}".format(name, local_dir))
+        logger.info("Updating {0} in {1}".format(name, local_dir))
 
         remotes = subprocess.check_output(["git", "remote", "show"], cwd=local_dir)
         remotes = [i.strip() for i in remotes.decode("utf-8").splitlines()]
 
         if "origin" not in remotes:
             git_command = ["git", "remote", "rm", "origin"]
-            logging_subprocess(git_command, None, cwd=local_dir)
+            logging_subprocess(git_command, cwd=local_dir)
             git_command = ["git", "remote", "add", "origin", remote_url]
-            logging_subprocess(git_command, None, cwd=local_dir)
+            logging_subprocess(git_command, cwd=local_dir)
         else:
             git_command = ["git", "remote", "set-url", "origin", remote_url]
-            logging_subprocess(git_command, None, cwd=local_dir)
+            logging_subprocess(git_command, cwd=local_dir)
 
-        if lfs_clone:
-            git_command = ["git", "lfs", "fetch", "--all", "--prune"]
-        else:
         git_command = ["git", "fetch", "--all", "--force", "--tags", "--prune"]
         if no_prune:
             git_command.pop()
-        logging_subprocess(git_command, None, cwd=local_dir)
+        logging_subprocess(git_command, cwd=local_dir)
+        if lfs_clone:
+            git_command = ["git", "lfs", "fetch", "--all", "--prune"]
+            if no_prune:
+                git_command.pop()
+            logging_subprocess(git_command, cwd=local_dir)
     else:
-        log_info(
+        logger.info(
             "Cloning {0} repository from {1} to {2}".format(
                 name, masked_remote_url, local_dir
             )
         )
         if bare_clone:
             git_command = ["git", "clone", "--mirror", remote_url, local_dir]
-            logging_subprocess(git_command, None)
+            logging_subprocess(git_command)
             if lfs_clone:
                 git_command = ["git", "lfs", "fetch", "--all", "--prune"]
                 if no_prune:
                     git_command.pop()
-                logging_subprocess(git_command, None, cwd=local_dir)
+                logging_subprocess(git_command, cwd=local_dir)
         else:
             if lfs_clone:
                 git_command = ["git", "lfs", "clone", remote_url, local_dir]
             else:
                 git_command = ["git", "clone", remote_url, local_dir]
-            logging_subprocess(git_command, None)
+            logging_subprocess(git_command)
 
 
 def backup_account(args, output_directory):
@@ -1337,11 +1435,11 @@ def backup_account(args, output_directory):
 def _backup_data(args, name, template, output_file, output_directory):
     skip_existing = args.skip_existing
     if not skip_existing or not os.path.exists(output_file):
-        log_info("Retrieving {0} {1}".format(args.user, name))
+        logger.info("Retrieving {0} {1}".format(args.user, name))
         mkdir_p(output_directory)
         data = retrieve_data(args, template)
 
-        log_info("Writing {0} {1} to disk".format(len(data), name))
+        logger.info("Writing {0} {1} to disk".format(len(data), name))
         with codecs.open(output_file, "w", encoding="utf-8") as f:
             json_dump(data, f)
 

python-github-backup.code-workspace

@@ -0,0 +1,7 @@
+{
+	"folders": [
+		{
+			"path": "."
+		}
+	]
+}

release

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
-set -eo pipefail; [[ $RELEASE_TRACE ]] && set -x
+set -eo pipefail
+[[ $RELEASE_TRACE ]] && set -x
 
 if [[ ! -f setup.py ]]; then
     echo -e "${RED}WARNING: Missing setup.py${COLOR_OFF}\n"
@@ -43,13 +44,13 @@ fi
 
 echo -e "\n${GREEN}STARTING RELEASE PROCESS${COLOR_OFF}\n"
 
-set +e;
+set +e
 git status | grep -Eo "working (directory|tree) clean" &>/dev/null
 if [ ! $? -eq 0 ]; then # working directory is NOT clean
     echo -e "${RED}WARNING: You have uncomitted changes, you may have forgotten something${COLOR_OFF}\n"
     exit 1
 fi
-set -e;
+set -e
 
 echo -e "${YELLOW}--->${COLOR_OFF} Updating local copy"
 git pull -q origin master
@@ -62,14 +63,14 @@ minor=$(echo $current_version | awk '{split($0,a,"."); print a[2]}')
 patch=$(echo $current_version | awk '{split($0,a,"."); print a[3]}')
 
 if [[ "$@" == "major" ]]; then
-    major=$(($major + 1));
+    major=$(($major + 1))
     minor="0"
     patch="0"
 elif [[ "$@" == "minor" ]]; then
-    minor=$(($minor + 1));
+    minor=$(($minor + 1))
     patch="0"
 elif [[ "$@" == "patch" ]]; then
-    patch=$(($patch + 1));
+    patch=$(($patch + 1))
 fi
 
 next_version="${major}.${minor}.${patch}"
@@ -77,7 +78,7 @@ next_version="${major}.${minor}.${patch}"
 echo -e "${YELLOW}   >${COLOR_OFF} ${MAGENTA}${current_version}${COLOR_OFF} -> ${MAGENTA}${next_version}${COLOR_OFF}"
 
 echo -e "${YELLOW}--->${COLOR_OFF} Ensuring readme passes lint checks (if this fails, run rst-lint)"
-rst-lint README.rst > /dev/null
+rst-lint README.rst || exit 1
 
 echo -e "${YELLOW}--->${COLOR_OFF} Creating necessary temp file"
 tempfoo=$(basename $0)
@@ -105,7 +106,9 @@ fi
 
 echo -e "${YELLOW}--->${COLOR_OFF} Updating CHANGES.rst for new release"
 version_header="$next_version ($(date +%F))"
-set +e; dashes=$(yes '-'|head -n ${#version_header}|tr -d '\n') ; set -e
+set +e
+dashes=$(yes '-' | head -n ${#version_header} | tr -d '\n')
+set -e
 gitchangelog | sed "4s/.*/$version_header/" | sed "5s/.*/$dashes/" >$TMPFILE && mv $TMPFILE CHANGES.rst
 
 echo -e "${YELLOW}--->${COLOR_OFF} Adding changed files to git"
@@ -115,6 +118,15 @@ if [ -f docs/conf.py ]; then git add docs/conf.py; fi
 echo -e "${YELLOW}--->${COLOR_OFF} Creating release"
 git commit -q -m "Release version $next_version"
 
+if [[ "$PUBLIC" == "true" ]]; then
+    echo -e "${YELLOW}--->${COLOR_OFF} Creating python release files"
+    cp README.rst README
+    python setup.py sdist bdist_wheel >/dev/null
+
+    echo -e "${YELLOW}--->${COLOR_OFF} Validating long_description"
+    twine check dist/*
+fi
+
 echo -e "${YELLOW}--->${COLOR_OFF} Tagging release"
 git tag -a $next_version -m "Release version $next_version"
 
@@ -122,9 +134,7 @@ echo -e "${YELLOW}--->${COLOR_OFF} Pushing release and tags to github"
 git push -q origin master && git push -q --tags
 
 if [[ "$PUBLIC" == "true" ]]; then
-    echo -e "${YELLOW}--->${COLOR_OFF} Creating python release"
-    cp README.rst README
-    python setup.py sdist bdist_wheel > /dev/null
+    echo -e "${YELLOW}--->${COLOR_OFF} Uploading python release"
     twine upload dist/*
     rm README
 fi

release-requirements.txt

@@ -0,0 +1,39 @@
+autopep8==2.3.2
+black==25.1.0
+bleach==6.2.0
+certifi==2025.1.31
+charset-normalizer==3.4.1
+click==8.1.8
+colorama==0.4.6
+docutils==0.21.2
+flake8==7.1.2
+gitchangelog==3.0.4
+idna==3.10
+importlib-metadata==8.6.1
+jaraco.classes==3.4.0
+keyring==25.6.0
+markdown-it-py==3.0.0
+mccabe==0.7.0
+mdurl==0.1.2
+more-itertools==10.6.0
+mypy-extensions==1.0.0
+packaging==24.2
+pathspec==0.12.1
+pkginfo==1.12.1.2
+platformdirs==4.3.6
+pycodestyle==2.12.1
+pyflakes==3.2.0
+Pygments==2.19.1
+readme-renderer==44.0
+requests==2.32.3
+requests-toolbelt==1.0.0
+restructuredtext-lint==1.4.0
+rfc3986==2.0.0
+rich==13.9.4
+setuptools==75.8.2
+six==1.17.0
+tqdm==4.67.1
+twine==6.1.0
+urllib3==2.3.0
+webencodings==0.5.1
+zipp==3.21.0

setup.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 import os
+
 from github_backup import __version__
 
 try:
@@ -39,10 +40,11 @@ setup(
         "Development Status :: 5 - Production/Stable",
         "Topic :: System :: Archiving :: Backup",
         "License :: OSI Approved :: MIT License",
-        "Programming Language :: Python :: 3.5",
-        "Programming Language :: Python :: 3.6",
-        "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
     ],
     description="backup a github user or organization",
     long_description=open_file("README.rst").read(),