Release version 0.47.0

Detect empty HTTPS contexts.

.circleci/config.yml

@@ -1,23 +0,0 @@
-version: 2.1
-
-orbs:
-  python: circleci/python@0.3.2
-
-jobs:
-  build-and-test:
-    executor: python/default
-    steps:
-      - checkout
-      - python/load-cache
-      - run:
-          command: pip install flake8
-          name: Install dependencies
-      - python/save-cache
-      - run:
-          command: flake8 --ignore=E501
-          name: Lint
-
-workflows:
-  main:
-    jobs:
-      - build-and-test

.github/dependabot.yml

@@ -0,0 +1,15 @@
+version: 2
+updates:
+- package-ecosystem: pip
+  directory: "/"
+  schedule:
+    interval: daily
+    time: "13:00"
+  groups:
+    python-packages:
+      patterns:
+        - "*"
+- package-ecosystem: "github-actions"
+  directory: "/"
+  schedule:
+    interval: "weekly"

.github/workflows/automatic-release.yml

@@ -15,20 +15,21 @@ on:
 jobs:
     release:
         name: Release
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-24.04
         steps:
             - name: Checkout repository
               uses: actions/checkout@v4
               with:
                 fetch-depth: 0
+                ssh-key: ${{ secrets.DEPLOY_PRIVATE_KEY }}
             - name: Setup Git
               run: |
                 git config --local user.email "action@github.com"
                 git config --local user.name "GitHub Action"
             - name: Setup Python
-              uses: actions/setup-python@v4
+              uses: actions/setup-python@v5
               with:
-                python-version: '3.8'
+                python-version: '3.12'
             - name: Install prerequisites
               run: pip install -r release-requirements.txt
             - name: Execute release

.github/workflows/docker.yml

@@ -0,0 +1,77 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: Create and publish a Docker image
+
+on:
+  push:
+    branches:
+      - 'master'
+      - 'main'
+      - 'dev'
+
+    tags:
+      - 'v*'
+      - 'v*.*'
+      - 'v*.*.*'
+      - '*'
+      - '*.*'
+      - '*.*.*'
+  pull_request:
+    branches:
+      - 'main'
+      - 'dev'
+
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+            type=sha
+            type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }}
+
+      - name: Build and push Docker image
+        uses:  docker/build-push-action@v6
+        with:
+          context: .
+          push: true
+          platforms: linux/amd64,linux/arm64
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/lint.yml

@@ -0,0 +1,33 @@
+---
+name: "lint"
+
+# yamllint disable-line rule:truthy
+on:
+  pull_request:
+    branches:
+      - "*"
+  push:
+    branches:
+      - "main"
+      - "master"
+
+jobs:
+  lint:
+    name: lint
+    runs-on: ubuntu-24.04
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: "pip"
+      - run: pip install -r release-requirements.txt && pip install wheel
+      - run: flake8 --ignore=E501,E203,W503
+      - run: black .
+      - run: rst-lint README.rst
+      - run: python setup.py sdist bdist_wheel && twine check dist/*

.gitignore

@@ -18,13 +18,13 @@ pkg
 
 # Debian Files
 debian/files
-debian/python-aws-hostname*
+debian/python-github-backup*
 
 # Sphinx build
 doc/_build
 
 # Generated man page
-doc/aws_hostname.1
+doc/github_backup.1
 
 # Annoying macOS files
 .DS_Store
@@ -35,3 +35,10 @@ doc/aws_hostname.1
 .atom
 
 README
+
+# RSA
+id_rsa
+id_rsa.pub
+
+# Virtual env
+venv

README.rst

@@ -49,7 +49,9 @@ CLI Help output::
                   [-P] [-F] [--prefer-ssh] [-v]
                   [--keychain-name OSX_KEYCHAIN_ITEM_NAME]
                   [--keychain-account OSX_KEYCHAIN_ITEM_ACCOUNT]
-                  [--releases] [--assets] [--exclude [REPOSITORY [REPOSITORY ...]]
+                  [--releases] [--latest-releases NUMBER_OF_LATEST_RELEASES]
+                  [--skip-prerelease] [--assets]
+                  [--exclude [REPOSITORY [REPOSITORY ...]]
                   [--throttle-limit THROTTLE_LIMIT] [--throttle-pause THROTTLE_PAUSE]
                   USER
 
@@ -124,6 +126,10 @@ CLI Help output::
                             keychain that holds the personal access or OAuth token
       --releases            include release information, not including assets or
                             binaries
+      --latest-releases NUMBER_OF_LATEST_RELEASES
+                            include certain number of the latest releases;
+                            only applies if including releases
+      --skip-prerelease     skip prerelease and draft versions; only applies if including releases
       --assets              include assets alongside release information; only
                             applies if including releases
       --exclude [REPOSITORY [REPOSITORY ...]]
@@ -206,13 +212,20 @@ When you use the ``--lfs`` option, you will need to make sure you have Git LFS i
 Instructions on how to do this can be found on https://git-lfs.github.com.
 
 
+Run in Docker container
+-----------------------
+
+To run the tool in a Docker container use the following command:
+
+    sudo docker run --rm -v /path/to/backup:/data --name github-backup ghcr.io/josegonzalez/python-github-backup -o /data $OPTIONS $USER
+
 Gotchas / Known-issues
 ======================
 
 All is not everything
 ---------------------
 
-The ``--all`` argument does not include; cloning private repos (``-P, --private``), cloning forks (``-F, --fork``) cloning starred repositories (``--all-starred``), ``--pull-details``, cloning LFS repositories (``--lfs``), cloning gists (``--starred-gists``) or cloning starred gist repos (``--starred-gists``). See examples for more.
+The ``--all`` argument does not include: cloning private repos (``-P, --private``), cloning forks (``-F, --fork``), cloning starred repositories (``--all-starred``), ``--pull-details``, cloning LFS repositories (``--lfs``), cloning gists (``--starred-gists``) or cloning starred gist repos (``--starred-gists``). See examples for more.
 
 Cloning all starred size
 ------------------------

bin/github-backup

@@ -1,12 +1,8 @@
 #!/usr/bin/env python
 
-import os, sys, logging
-
-logging.basicConfig(
-    format='%(asctime)s.%(msecs)03d: %(message)s',
-    datefmt='%Y-%m-%dT%H:%M:%S',
-    level=logging.INFO
-)
+import logging
+import os
+import sys
 
 from github_backup.github_backup import (
     backup_account,
@@ -20,6 +16,12 @@ from github_backup.github_backup import (
     retrieve_repositories,
 )
 
+logging.basicConfig(
+    format="%(asctime)s.%(msecs)03d: %(message)s",
+    datefmt="%Y-%m-%dT%H:%M:%S",
+    level=logging.INFO,
+)
+
 
 def main():
     args = parse_args()
@@ -29,7 +31,7 @@ def main():
 
     output_directory = os.path.realpath(args.output_directory)
     if not os.path.isdir(output_directory):
-        logger.info('Create output directory {0}'.format(output_directory))
+        logger.info("Create output directory {0}".format(output_directory))
         mkdir_p(output_directory)
 
     if args.lfs_clone:
@@ -41,10 +43,10 @@ def main():
             logger.root.setLevel(log_level)
 
     if not args.as_app:
-        logger.info('Backing up user {0} to {1}'.format(args.user, output_directory))
+        logger.info("Backing up user {0} to {1}".format(args.user, output_directory))
         authenticated_user = get_authenticated_user(args)
     else:
-        authenticated_user = {'login': None}
+        authenticated_user = {"login": None}
 
     repositories = retrieve_repositories(args, authenticated_user)
     repositories = filter_repositories(args, repositories)
@@ -52,7 +54,7 @@ def main():
     backup_account(args, output_directory)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     try:
         main()
     except Exception as e:

github_backup/__init__.py

@@ -1 +1 @@
-__version__ = "0.44.0"
+__version__ = "0.47.0"

github_backup/github_backup.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python
 
 from __future__ import print_function
-import socket
 
 import argparse
 import base64
@@ -10,23 +9,22 @@ import codecs
 import errno
 import getpass
 import json
+import logging
 import os
+import platform
 import re
 import select
+import socket
+import ssl
 import subprocess
 import sys
-import logging
 import time
-import platform
-from urllib.parse import urlparse
-from urllib.parse import quote as urlquote
-from urllib.parse import urlencode
-from urllib.error import HTTPError, URLError
-from urllib.request import urlopen
-from urllib.request import Request
-from urllib.request import HTTPRedirectHandler
-from urllib.request import build_opener
 from http.client import IncompleteRead
+from urllib.error import HTTPError, URLError
+from urllib.parse import quote as urlquote
+from urllib.parse import urlencode, urlparse
+from urllib.request import HTTPRedirectHandler, Request, build_opener, urlopen
+from datetime import datetime
 
 try:
     from . import __version__
@@ -39,12 +37,21 @@ FNULL = open(os.devnull, "w")
 FILE_URI_PREFIX = "file://"
 logger = logging.getLogger(__name__)
 
+https_ctx = ssl.create_default_context()
+if not https_ctx.get_ca_certs():
+    import warnings
+    warnings.warn('\n\nYOUR DEFAULT CA CERTS ARE EMPTY.\n' +
+                  'PLEASE POPULATE ANY OF:' +
+                  ''.join([
+                    '\n - ' + x
+                    for x in ssl.get_default_verify_paths()
+                    if type(x) is str
+                  ]) + '\n', stacklevel=2)
+    import certifi
+    https_ctx = ssl.create_default_context(cafile=certifi.where())
 
 def logging_subprocess(
-    popenargs,
-    stdout_log_level=logging.DEBUG,
-    stderr_log_level=logging.ERROR,
-    **kwargs
+    popenargs, stdout_log_level=logging.DEBUG, stderr_log_level=logging.ERROR, **kwargs
 ):
     """
     Variant of subprocess.call that accepts a logger instead of stdout/stderr,
@@ -383,6 +390,19 @@ def parse_args(args=None):
         dest="include_releases",
         help="include release information, not including assets or binaries",
     )
+    parser.add_argument(
+        "--latest-releases",
+        type=int,
+        default=0,
+        dest="number_of_latest_releases",
+        help="include certain number of the latest releases; only applies if including releases",
+    )
+    parser.add_argument(
+        "--skip-prerelease",
+        action="store_true",
+        dest="skip_prerelease",
+        help="skip prerelease and draft versions; only applies if including releases",
+    )
     parser.add_argument(
         "--assets",
         action="store_true",
@@ -504,7 +524,7 @@ def get_github_host(args):
 
 
 def read_file_contents(file_uri):
-    return open(file_uri[len(FILE_URI_PREFIX) :], "rt").readline().strip()
+    return open(file_uri[len(FILE_URI_PREFIX):], "rt").readline().strip()
 
 
 def get_github_repo_url(args, repository):
@@ -626,12 +646,12 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False):
             raise Exception(", ".join(errors))
 
         if len(errors) == 0:
-            if type(response) == list:
+            if type(response) is list:
                 for resp in response:
                     yield resp
                 if len(response) < per_page:
                     break
-            elif type(response) == dict and single_request:
+            elif type(response) is dict and single_request:
                 yield response
 
         if len(errors) > 0:
@@ -659,7 +679,7 @@ def _get_response(request, auth, template):
     while True:
         should_continue = False
         try:
-            r = urlopen(request)
+            r = urlopen(request, context=https_ctx)
         except HTTPError as exc:
             errors, should_continue = _request_http_error(exc, auth, errors)  # noqa
             r = exc
@@ -770,14 +790,19 @@ class S3HTTPRedirectHandler(HTTPRedirectHandler):
         return request
 
 
-def download_file(url, path, auth):
+def download_file(url, path, auth, as_app=False, fine=False):
     # Skip downloading release assets if they already exist on disk so we don't redownload on every sync
     if os.path.exists(path):
         return
 
-    request = Request(url)
+    request = _construct_request(per_page=100,
+                                 page=1,
+                                 query_args={},
+                                 template=url,
+                                 auth=auth,
+                                 as_app=as_app,
+                                 fine=fine)
     request.add_header("Accept", "application/octet-stream")
-    request.add_header("Authorization", "Basic ".encode("ascii") + auth)
     opener = build_opener(S3HTTPRedirectHandler)
 
     try:
@@ -919,11 +944,11 @@ def filter_repositories(args, unfiltered_repositories):
             if r.get("language") and r.get("language").lower() in languages
         ]  # noqa
     if name_regex:
-        repositories = [r for r in repositories if name_regex.match(r["name"])]
+        repositories = [r for r in repositories if "name" not in r or name_regex.match(r["name"])]
     if args.skip_archived:
         repositories = [r for r in repositories if not r.get("archived")]
     if args.exclude:
-        repositories = [r for r in repositories if r["name"] not in args.exclude]
+        repositories = [r for r in repositories if "name" not in r or r["name"] not in args.exclude]
 
     return repositories
 
@@ -1196,8 +1221,11 @@ def backup_hooks(args, repo_cwd, repository, repos_template):
     template = "{0}/{1}/hooks".format(repos_template, repository["full_name"])
     try:
         _backup_data(args, "hooks", template, output_file, hook_cwd)
-    except SystemExit:
+    except Exception as e:
+        if "404" in str(e):
             logger.info("Unable to read hooks, skipping")
+        else:
+            raise e
 
 
 def backup_releases(args, repo_cwd, repository, repos_template, include_assets=False):
@@ -1213,8 +1241,18 @@ def backup_releases(args, repo_cwd, repository, repos_template, include_assets=F
     release_template = "{0}/{1}/releases".format(repos_template, repository_fullname)
     releases = retrieve_data(args, release_template, query_args=query_args)
 
-    # for each release, store it
+    if args.skip_prerelease:
+        releases = [r for r in releases if not r["prerelease"] and not r["draft"]]
+
+    if args.number_of_latest_releases and args.number_of_latest_releases < len(releases):
+        releases.sort(key=lambda item: datetime.strptime(item["created_at"], "%Y-%m-%dT%H:%M:%SZ"),
+                      reverse=True)
+        releases = releases[:args.number_of_latest_releases]
+        logger.info("Saving the latest {0} releases to disk".format(len(releases)))
+    else:
         logger.info("Saving {0} releases to disk".format(len(releases)))
+
+    # for each release, store it
     for release in releases:
         release_name = release["tag_name"]
         release_name_safe = release_name.replace("/", "__")
@@ -1234,7 +1272,9 @@ def backup_releases(args, repo_cwd, repository, repos_template, include_assets=F
                     download_file(
                         asset["url"],
                         os.path.join(release_assets_cwd, asset["name"]),
-                        get_auth(args),
+                        get_auth(args, encode=not args.as_app),
+                        as_app=args.as_app,
+                        fine=True if args.token_fine is not None else False
                     )
 
 
@@ -1291,10 +1331,12 @@ def fetch_repository(
             git_command = ["git", "remote", "set-url", "origin", remote_url]
             logging_subprocess(git_command, cwd=local_dir)
 
+        git_command = ["git", "fetch", "--all", "--force", "--tags", "--prune"]
+        if no_prune:
+            git_command.pop()
+        logging_subprocess(git_command, cwd=local_dir)
         if lfs_clone:
             git_command = ["git", "lfs", "fetch", "--all", "--prune"]
-        else:
-            git_command = ["git", "fetch", "--all", "--force", "--tags", "--prune"]
             if no_prune:
                 git_command.pop()
             logging_subprocess(git_command, cwd=local_dir)

python-github-backup.code-workspace

@@ -0,0 +1,7 @@
+{
+	"folders": [
+		{
+			"path": "."
+		}
+	]
+}

release-requirements.txt

@@ -1,31 +1,39 @@
-bleach==6.0.0
-certifi==2023.7.22
-charset-normalizer==3.1.0
+autopep8==2.3.1
+black==24.10.0
+bleach==6.2.0
+certifi==2024.8.30
+charset-normalizer==3.4.0
+click==8.1.7
 colorama==0.4.6
-docutils==0.20.1
-flake8==6.0.0
+docutils==0.21.2
+flake8==7.1.1
 gitchangelog==3.0.4
-idna==3.4
-importlib-metadata==6.6.0
-jaraco.classes==3.2.3
-keyring==23.13.1
-markdown-it-py==2.2.0
+idna==3.10
+importlib-metadata==8.5.0
+jaraco.classes==3.4.0
+keyring==25.5.0
+markdown-it-py==3.0.0
 mccabe==0.7.0
 mdurl==0.1.2
-more-itertools==9.1.0
-pkginfo==1.9.6
-pycodestyle==2.10.0
-pyflakes==3.0.1
-Pygments==2.15.1
-readme-renderer==37.3
-requests==2.31.0
+more-itertools==10.5.0
+mypy-extensions==1.0.0
+packaging==24.2
+pathspec==0.12.1
+pkginfo==1.12.0
+platformdirs==4.3.6
+pycodestyle==2.12.1
+pyflakes==3.2.0
+Pygments==2.18.0
+readme-renderer==44.0
+requests==2.32.3
 requests-toolbelt==1.0.0
 restructuredtext-lint==1.4.0
 rfc3986==2.0.0
-rich==13.3.5
-six==1.16.0
-tqdm==4.65.0
-twine==4.0.2
-urllib3==2.0.7
+rich==13.9.4
+setuptools==75.6.0
+six==1.17.0
+tqdm==4.67.1
+twine==6.0.1
+urllib3==2.2.3
 webencodings==0.5.1
-zipp==3.15.0
+zipp==3.21.0

setup.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 import os
+
 from github_backup import __version__
 
 try:
@@ -39,10 +40,11 @@ setup(
         "Development Status :: 5 - Production/Stable",
         "Topic :: System :: Archiving :: Backup",
         "License :: OSI Approved :: MIT License",
-        "Programming Language :: Python :: 3.5",
-        "Programming Language :: Python :: 3.6",
-        "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
     ],
     description="backup a github user or organization",
     long_description=open_file("README.rst").read(),