Release version 0.49.0

Implementing incremental by files, safer version of incremental backup.

.github/dependabot.yml

@@ -9,3 +9,7 @@ updates:
     python-packages:
       patterns:
         - "*"
+- package-ecosystem: "github-actions"
+  directory: "/"
+  schedule:
+    interval: "weekly"

.github/workflows/automatic-release.yml

@@ -15,7 +15,7 @@ on:
 jobs:
     release:
         name: Release
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-24.04
         steps:
             - name: Checkout repository
               uses: actions/checkout@v4
@@ -27,9 +27,9 @@ jobs:
                 git config --local user.email "action@github.com"
                 git config --local user.name "GitHub Action"
             - name: Setup Python
-              uses: actions/setup-python@v4
+              uses: actions/setup-python@v5
               with:
-                python-version: '3.8'
+                python-version: '3.12'
             - name: Install prerequisites
               run: pip install -r release-requirements.txt
             - name: Execute release

.github/workflows/docker.yml

@@ -38,16 +38,18 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Log in to the Container registry
-        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -55,7 +57,7 @@ jobs:
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: |
@@ -66,7 +68,7 @@ jobs:
             type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }}
 
       - name: Build and push Docker image
-        uses:  docker/build-push-action@v4
+        uses:  docker/build-push-action@v6
         with:
           context: .
           push: true

.github/workflows/lint.yml

@@ -14,7 +14,7 @@ on:
 jobs:
   lint:
     name: lint
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
 
     steps:
       - name: Checkout repository
@@ -22,9 +22,9 @@ jobs:
         with:
           fetch-depth: 0
       - name: Setup Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: "3.8"
+          python-version: "3.12"
           cache: "pip"
       - run: pip install -r release-requirements.txt && pip install wheel
       - run: flake8 --ignore=E501,E203,W503

README.rst

@@ -80,6 +80,7 @@ CLI Help output::
                             log level to use (default: info, possible levels:
                             debug, info, warning, error, critical)
       -i, --incremental     incremental backup
+      --incremental-by-files incremental backup using modified time of files
       --starred             include JSON output of starred repositories in backup
       --all-starred         include starred repositories in backup [*]
       --watched             include JSON output of watched repositories in backup
@@ -167,7 +168,7 @@ Customise the permissions for your use case, but for a personal account full bac
 
 **User permissions**: Read access to followers, starring, and watching.
 
-**Repository permissions**: Read access to code, commit statuses, issues, metadata, pages, pull requests, and repository hooks.
+**Repository permissions**: Read access to contents, issues, metadata, pull requests, and webhooks.
 
 
 Prefer SSH
@@ -212,13 +213,20 @@ When you use the ``--lfs`` option, you will need to make sure you have Git LFS i
 Instructions on how to do this can be found on https://git-lfs.github.com.
 
 
+Run in Docker container
+-----------------------
+
+To run the tool in a Docker container use the following command:
+
+    sudo docker run --rm -v /path/to/backup:/data --name github-backup ghcr.io/josegonzalez/python-github-backup -o /data $OPTIONS $USER
+
 Gotchas / Known-issues
 ======================
 
 All is not everything
 ---------------------
 
-The ``--all`` argument does not include; cloning private repos (``-P, --private``), cloning forks (``-F, --fork``) cloning starred repositories (``--all-starred``), ``--pull-details``, cloning LFS repositories (``--lfs``), cloning gists (``--starred-gists``) or cloning starred gist repos (``--starred-gists``). See examples for more.
+The ``--all`` argument does not include: cloning private repos (``-P, --private``), cloning forks (``-F, --fork``), cloning starred repositories (``--all-starred``), ``--pull-details``, cloning LFS repositories (``--lfs``), cloning gists (``--gists``) or cloning starred gist repos (``--starred-gists``). See examples for more.
 
 Cloning all starred size
 ------------------------
@@ -232,6 +240,12 @@ Using (``-i, --incremental``) will only request new data from the API **since th
 
 This means any blocking errors on previous runs can cause a large amount of missing data in backups.
 
+Using (``--incremental-by-files``) will request new data from the API **based on when the file was modified on filesystem**. e.g. if you modify the file yourself you may miss something.
+
+Still saver than the previous version.
+
+Specifically, issues and pull requests are handled like this.
+
 Known blocking errors
 ---------------------
 
@@ -247,12 +261,6 @@ It's therefore recommended to only use the incremental argument if the output/re
 
    This is due to needing the correct permission for ``--hooks`` on public repos.
 
-2. **Releases blocking**
-
-   A known ``--releases`` (required for ``--assets``) error will sometimes block the backup. 
-
-   If you're backing up a lot of repositories with releases e.g. an organisation or ``--all-starred``. You may need to remove ``--releases`` (and therefore ``--assets``) to complete a backup. Documented in `issue 209 <https://github.com/josegonzalez/python-github-backup/issues/209>`_.
-
 
 "bare" is actually "mirror"
 ---------------------------

github_backup/__init__.py

@@ -1 +1 @@
-__version__ = "0.45.0"
+__version__ = "0.49.0"

github_backup/github_backup.py

@@ -15,15 +15,16 @@ import platform
 import re
 import select
 import socket
+import ssl
 import subprocess
 import sys
 import time
+from datetime import datetime
 from http.client import IncompleteRead
 from urllib.error import HTTPError, URLError
 from urllib.parse import quote as urlquote
 from urllib.parse import urlencode, urlparse
 from urllib.request import HTTPRedirectHandler, Request, build_opener, urlopen
-from datetime import datetime
 
 try:
     from . import __version__
@@ -36,6 +37,23 @@ FNULL = open(os.devnull, "w")
 FILE_URI_PREFIX = "file://"
 logger = logging.getLogger(__name__)
 
+https_ctx = ssl.create_default_context()
+if not https_ctx.get_ca_certs():
+    import warnings
+
+    warnings.warn(
+        "\n\nYOUR DEFAULT CA CERTS ARE EMPTY.\n"
+        + "PLEASE POPULATE ANY OF:"
+        + "".join(
+            ["\n - " + x for x in ssl.get_default_verify_paths() if type(x) is str]
+        )
+        + "\n",
+        stacklevel=2,
+    )
+    import certifi
+
+    https_ctx = ssl.create_default_context(cafile=certifi.where())
+
 
 def logging_subprocess(
     popenargs, stdout_log_level=logging.DEBUG, stderr_log_level=logging.ERROR, **kwargs
@@ -163,6 +181,12 @@ def parse_args(args=None):
         dest="incremental",
         help="incremental backup",
     )
+    parser.add_argument(
+        "--incremental-by-files",
+        action="store_true",
+        dest="incremental_by_files",
+        help="incremental backup based on modification date of files",
+    )
     parser.add_argument(
         "--starred",
         action="store_true",
@@ -511,7 +535,7 @@ def get_github_host(args):
 
 
 def read_file_contents(file_uri):
-    return open(file_uri[len(FILE_URI_PREFIX):], "rt").readline().strip()
+    return open(file_uri[len(FILE_URI_PREFIX) :], "rt").readline().strip()
 
 
 def get_github_repo_url(args, repository):
@@ -666,7 +690,7 @@ def _get_response(request, auth, template):
     while True:
         should_continue = False
         try:
-            r = urlopen(request)
+            r = urlopen(request, context=https_ctx)
         except HTTPError as exc:
             errors, should_continue = _request_http_error(exc, auth, errors)  # noqa
             r = exc
@@ -777,14 +801,21 @@ class S3HTTPRedirectHandler(HTTPRedirectHandler):
         return request
 
 
-def download_file(url, path, auth):
+def download_file(url, path, auth, as_app=False, fine=False):
     # Skip downloading release assets if they already exist on disk so we don't redownload on every sync
     if os.path.exists(path):
         return
 
-    request = Request(url)
+    request = _construct_request(
+        per_page=100,
+        page=1,
+        query_args={},
+        template=url,
+        auth=auth,
+        as_app=as_app,
+        fine=fine,
+    )
     request.add_header("Accept", "application/octet-stream")
-    request.add_header("Authorization", "Basic ".encode("ascii") + auth)
     opener = build_opener(S3HTTPRedirectHandler)
 
     try:
@@ -926,11 +957,15 @@ def filter_repositories(args, unfiltered_repositories):
             if r.get("language") and r.get("language").lower() in languages
         ]  # noqa
     if name_regex:
-        repositories = [r for r in repositories if name_regex.match(r["name"])]
+        repositories = [
+            r for r in repositories if "name" not in r or name_regex.match(r["name"])
+        ]
     if args.skip_archived:
         repositories = [r for r in repositories if not r.get("archived")]
     if args.exclude:
-        repositories = [r for r in repositories if r["name"] not in args.exclude]
+        repositories = [
+            r for r in repositories if "name" not in r or r["name"] not in args.exclude
+        ]
 
     return repositories
 
@@ -1085,6 +1120,14 @@ def backup_issues(args, repo_cwd, repository, repos_template):
     comments_template = _issue_template + "/{0}/comments"
     events_template = _issue_template + "/{0}/events"
     for number, issue in list(issues.items()):
+        issue_file = "{0}/{1}.json".format(issue_cwd, number)
+        if args.incremental_by_files and os.path.isfile(issue_file):
+            modified = os.path.getmtime(issue_file)
+            modified = datetime.fromtimestamp(modified).strftime("%Y-%m-%dT%H:%M:%SZ")
+            if modified > issue["updated_at"]:
+                logger.info("Skipping issue {0} because it wasn't modified since last backup".format(number))
+                continue
+
         if args.include_issue_comments or args.include_everything:
             template = comments_template.format(number)
             issues[number]["comment_data"] = retrieve_data(args, template)
@@ -1092,9 +1135,9 @@ def backup_issues(args, repo_cwd, repository, repos_template):
             template = events_template.format(number)
             issues[number]["event_data"] = retrieve_data(args, template)
 
-        issue_file = "{0}/{1}.json".format(issue_cwd, number)
-        with codecs.open(issue_file, "w", encoding="utf-8") as f:
+        with codecs.open(issue_file + ".temp", "w", encoding="utf-8") as f:
             json_dump(issue, f)
+            os.rename(issue_file + ".temp", issue_file) # Unlike json_dump, this is atomic
 
 
 def backup_pulls(args, repo_cwd, repository, repos_template):
@@ -1147,6 +1190,13 @@ def backup_pulls(args, repo_cwd, repository, repos_template):
     comments_template = _pulls_template + "/{0}/comments"
     commits_template = _pulls_template + "/{0}/commits"
     for number, pull in list(pulls.items()):
+        pull_file = "{0}/{1}.json".format(pulls_cwd, number)
+        if args.incremental_by_files and os.path.isfile(pull_file):
+            modified = os.path.getmtime(pull_file)
+            modified = datetime.fromtimestamp(modified).strftime("%Y-%m-%dT%H:%M:%SZ")
+            if modified > pull["updated_at"]:
+                logger.info("Skipping pull request {0} because it wasn't modified since last backup".format(number))
+                continue
         if args.include_pull_comments or args.include_everything:
             template = comments_regular_template.format(number)
             pulls[number]["comment_regular_data"] = retrieve_data(args, template)
@@ -1156,9 +1206,9 @@ def backup_pulls(args, repo_cwd, repository, repos_template):
             template = commits_template.format(number)
             pulls[number]["commit_data"] = retrieve_data(args, template)
 
-        pull_file = "{0}/{1}.json".format(pulls_cwd, number)
-        with codecs.open(pull_file, "w", encoding="utf-8") as f:
+        with codecs.open(pull_file + ".temp", "w", encoding="utf-8") as f:
             json_dump(pull, f)
+            os.rename(pull_file + ".temp", pull_file) # Unlike json_dump, this is atomic
 
 
 def backup_milestones(args, repo_cwd, repository, repos_template):
@@ -1226,10 +1276,16 @@ def backup_releases(args, repo_cwd, repository, repos_template, include_assets=F
     if args.skip_prerelease:
         releases = [r for r in releases if not r["prerelease"] and not r["draft"]]
 
-    if args.number_of_latest_releases and args.number_of_latest_releases < len(releases):
-        releases.sort(key=lambda item: datetime.strptime(item["created_at"], "%Y-%m-%dT%H:%M:%SZ"),
-                      reverse=True)
-        releases = releases[:args.number_of_latest_releases]
+    if args.number_of_latest_releases and args.number_of_latest_releases < len(
+        releases
+    ):
+        releases.sort(
+            key=lambda item: datetime.strptime(
+                item["created_at"], "%Y-%m-%dT%H:%M:%SZ"
+            ),
+            reverse=True,
+        )
+        releases = releases[: args.number_of_latest_releases]
         logger.info("Saving the latest {0} releases to disk".format(len(releases)))
     else:
         logger.info("Saving {0} releases to disk".format(len(releases)))
@@ -1254,7 +1310,9 @@ def backup_releases(args, repo_cwd, repository, repos_template, include_assets=F
                     download_file(
                         asset["url"],
                         os.path.join(release_assets_cwd, asset["name"]),
-                        get_auth(args),
+                        get_auth(args, encode=not args.as_app),
+                        as_app=args.as_app,
+                        fine=True if args.token_fine is not None else False,
                     )
 
 
@@ -1311,13 +1369,15 @@ def fetch_repository(
             git_command = ["git", "remote", "set-url", "origin", remote_url]
             logging_subprocess(git_command, cwd=local_dir)
 
-        if lfs_clone:
-            git_command = ["git", "lfs", "fetch", "--all", "--prune"]
-        else:
-            git_command = ["git", "fetch", "--all", "--force", "--tags", "--prune"]
+        git_command = ["git", "fetch", "--all", "--force", "--tags", "--prune"]
         if no_prune:
             git_command.pop()
         logging_subprocess(git_command, cwd=local_dir)
+        if lfs_clone:
+            git_command = ["git", "lfs", "fetch", "--all", "--prune"]
+            if no_prune:
+                git_command.pop()
+            logging_subprocess(git_command, cwd=local_dir)
     else:
         logger.info(
             "Cloning {0} repository from {1} to {2}".format(

release-requirements.txt

@@ -1,38 +1,39 @@
-autopep8==2.0.4
-black==24.1.0
-bleach==6.1.0
-certifi==2023.11.17
-charset-normalizer==3.3.2
-click==8.1.7
+autopep8==2.3.2
+black==25.1.0
+bleach==6.2.0
+certifi==2025.1.31
+charset-normalizer==3.4.1
+click==8.1.8
 colorama==0.4.6
-docutils==0.20.1
-flake8==7.0.0
+docutils==0.21.2
+flake8==7.1.1
 gitchangelog==3.0.4
-idna==3.6
-importlib-metadata==7.0.1
-jaraco.classes==3.3.0
-keyring==24.3.0
+idna==3.10
+importlib-metadata==8.6.1
+jaraco.classes==3.4.0
+keyring==25.6.0
 markdown-it-py==3.0.0
 mccabe==0.7.0
 mdurl==0.1.2
-more-itertools==10.2.0
+more-itertools==10.6.0
 mypy-extensions==1.0.0
-packaging==23.2
+packaging==24.2
 pathspec==0.12.1
-pkginfo==1.9.6
-platformdirs==4.1.0
-pycodestyle==2.11.1
+pkginfo==1.12.0
+platformdirs==4.3.6
+pycodestyle==2.12.1
 pyflakes==3.2.0
-Pygments==2.17.2
-readme-renderer==42.0
-requests==2.31.0
+Pygments==2.19.1
+readme-renderer==44.0
+requests==2.32.3
 requests-toolbelt==1.0.0
 restructuredtext-lint==1.4.0
 rfc3986==2.0.0
-rich==13.7.0
-six==1.16.0
-tqdm==4.66.1
-twine==4.0.2
-urllib3==2.1.0
+rich==13.9.4
+setuptools==75.8.0
+six==1.17.0
+tqdm==4.67.1
+twine==6.1.0
+urllib3==2.3.0
 webencodings==0.5.1
-zipp==3.17.0
+zipp==3.21.0

setup.py

@@ -40,10 +40,11 @@ setup(
         "Development Status :: 5 - Production/Stable",
         "Topic :: System :: Archiving :: Backup",
         "License :: OSI Approved :: MIT License",
-        "Programming Language :: Python :: 3.5",
-        "Programming Language :: Python :: 3.6",
-        "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
     ],
     description="backup a github user or organization",
     long_description=open_file("README.rst").read(),