Release version 0.50.3

Fix -R flag to allow backups of repositories not owned by user

.github/dependabot.yml

@@ -9,3 +9,7 @@ updates:
     python-packages:
       patterns:
         - "*"
+- package-ecosystem: "github-actions"
+  directory: "/"
+  schedule:
+    interval: "weekly"

.github/workflows/automatic-release.yml

@@ -15,7 +15,7 @@ on:
 jobs:
     release:
         name: Release
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-24.04
         steps:
             - name: Checkout repository
               uses: actions/checkout@v4
@@ -27,9 +27,9 @@ jobs:
                 git config --local user.email "action@github.com"
                 git config --local user.name "GitHub Action"
             - name: Setup Python
-              uses: actions/setup-python@v4
+              uses: actions/setup-python@v5
               with:
-                python-version: '3.8'
+                python-version: '3.12'
             - name: Install prerequisites
               run: pip install -r release-requirements.txt
             - name: Execute release

.github/workflows/docker.yml

@@ -38,16 +38,18 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Log in to the Container registry
-        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -55,7 +57,7 @@ jobs:
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: |
@@ -66,7 +68,7 @@ jobs:
             type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }}
 
       - name: Build and push Docker image
-        uses:  docker/build-push-action@v4
+        uses:  docker/build-push-action@v6
         with:
           context: .
           push: true

.github/workflows/lint.yml

@@ -14,7 +14,7 @@ on:
 jobs:
   lint:
     name: lint
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
 
     steps:
       - name: Checkout repository
@@ -22,9 +22,9 @@ jobs:
         with:
           fetch-depth: 0
       - name: Setup Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: "3.8"
+          python-version: "3.12"
           cache: "pip"
       - run: pip install -r release-requirements.txt && pip install wheel
       - run: flake8 --ignore=E501,E203,W503

.github/workflows/tagged-release.yml

@@ -10,7 +10,7 @@ on:
 jobs:
   tagged-release:
     name: tagged-release
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
 
     steps:
       - uses: "marvinpinto/action-automatic-releases@v1.2.1"

README.rst

@@ -80,6 +80,7 @@ CLI Help output::
                             log level to use (default: info, possible levels:
                             debug, info, warning, error, critical)
       -i, --incremental     incremental backup
+      --incremental-by-files incremental backup using modified time of files
       --starred             include JSON output of starred repositories in backup
       --all-starred         include starred repositories in backup [*]
       --watched             include JSON output of watched repositories in backup
@@ -167,7 +168,7 @@ Customise the permissions for your use case, but for a personal account full bac
 
 **User permissions**: Read access to followers, starring, and watching.
 
-**Repository permissions**: Read access to code, commit statuses, issues, metadata, pages, pull requests, and repository hooks.
+**Repository permissions**: Read access to contents, issues, metadata, pull requests, and webhooks.
 
 
 Prefer SSH
@@ -212,13 +213,20 @@ When you use the ``--lfs`` option, you will need to make sure you have Git LFS i
 Instructions on how to do this can be found on https://git-lfs.github.com.
 
 
+Run in Docker container
+-----------------------
+
+To run the tool in a Docker container use the following command:
+
+    sudo docker run --rm -v /path/to/backup:/data --name github-backup ghcr.io/josegonzalez/python-github-backup -o /data $OPTIONS $USER
+
 Gotchas / Known-issues
 ======================
 
 All is not everything
 ---------------------
 
-The ``--all`` argument does not include; cloning private repos (``-P, --private``), cloning forks (``-F, --fork``) cloning starred repositories (``--all-starred``), ``--pull-details``, cloning LFS repositories (``--lfs``), cloning gists (``--starred-gists``) or cloning starred gist repos (``--starred-gists``). See examples for more.
+The ``--all`` argument does not include: cloning private repos (``-P, --private``), cloning forks (``-F, --fork``), cloning starred repositories (``--all-starred``), ``--pull-details``, cloning LFS repositories (``--lfs``), cloning gists (``--gists``) or cloning starred gist repos (``--starred-gists``). See examples for more.
 
 Cloning all starred size
 ------------------------
@@ -232,6 +240,12 @@ Using (``-i, --incremental``) will only request new data from the API **since th
 
 This means any blocking errors on previous runs can cause a large amount of missing data in backups.
 
+Using (``--incremental-by-files``) will request new data from the API **based on when the file was modified on filesystem**. e.g. if you modify the file yourself you may miss something.
+
+Still saver than the previous version.
+
+Specifically, issues and pull requests are handled like this.
+
 Known blocking errors
 ---------------------
 
@@ -247,12 +261,6 @@ It's therefore recommended to only use the incremental argument if the output/re
 
    This is due to needing the correct permission for ``--hooks`` on public repos.
 
-2. **Releases blocking**
-
-   A known ``--releases`` (required for ``--assets``) error will sometimes block the backup. 
-
-   If you're backing up a lot of repositories with releases e.g. an organisation or ``--all-starred``. You may need to remove ``--releases`` (and therefore ``--assets``) to complete a backup. Documented in `issue 209 <https://github.com/josegonzalez/python-github-backup/issues/209>`_.
-
 
 "bare" is actually "mirror"
 ---------------------------

github_backup/__init__.py

@@ -1 +1 @@
-__version__ = "0.45.1"
+__version__ = "0.50.3"

github_backup/github_backup.py

@@ -15,15 +15,16 @@ import platform
 import re
 import select
 import socket
+import ssl
 import subprocess
 import sys
 import time
+from datetime import datetime
 from http.client import IncompleteRead
 from urllib.error import HTTPError, URLError
 from urllib.parse import quote as urlquote
 from urllib.parse import urlencode, urlparse
 from urllib.request import HTTPRedirectHandler, Request, build_opener, urlopen
-from datetime import datetime
 
 try:
     from . import __version__
@@ -36,6 +37,23 @@ FNULL = open(os.devnull, "w")
 FILE_URI_PREFIX = "file://"
 logger = logging.getLogger(__name__)
 
+https_ctx = ssl.create_default_context()
+if not https_ctx.get_ca_certs():
+    import warnings
+
+    warnings.warn(
+        "\n\nYOUR DEFAULT CA CERTS ARE EMPTY.\n"
+        + "PLEASE POPULATE ANY OF:"
+        + "".join(
+            ["\n - " + x for x in ssl.get_default_verify_paths() if type(x) is str]
+        )
+        + "\n",
+        stacklevel=2,
+    )
+    import certifi
+
+    https_ctx = ssl.create_default_context(cafile=certifi.where())
+
 
 def logging_subprocess(
     popenargs, stdout_log_level=logging.DEBUG, stderr_log_level=logging.ERROR, **kwargs
@@ -163,6 +181,12 @@ def parse_args(args=None):
         dest="incremental",
         help="incremental backup",
     )
+    parser.add_argument(
+        "--incremental-by-files",
+        action="store_true",
+        dest="incremental_by_files",
+        help="incremental backup based on modification date of files",
+    )
     parser.add_argument(
         "--starred",
         action="store_true",
@@ -511,7 +535,7 @@ def get_github_host(args):
 
 
 def read_file_contents(file_uri):
-    return open(file_uri[len(FILE_URI_PREFIX):], "rt").readline().strip()
+    return open(file_uri[len(FILE_URI_PREFIX) :], "rt").readline().strip()
 
 
 def get_github_repo_url(args, repository):
@@ -554,10 +578,15 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False):
     page = 0
 
     while True:
+        if single_request:
+            request_page, request_per_page = None, None
+        else:
             page = page + 1
+            request_page, request_per_page = page, per_page
+
         request = _construct_request(
-            per_page,
-            page,
+            request_per_page,
+            request_page,
             query_args,
             template,
             auth,
@@ -666,7 +695,7 @@ def _get_response(request, auth, template):
     while True:
         should_continue = False
         try:
-            r = urlopen(request)
+            r = urlopen(request, context=https_ctx)
         except HTTPError as exc:
             errors, should_continue = _request_http_error(exc, auth, errors)  # noqa
             r = exc
@@ -691,14 +720,22 @@ def _get_response(request, auth, template):
 def _construct_request(
     per_page, page, query_args, template, auth, as_app=None, fine=False
 ):
-    querystring = urlencode(
-        dict(
-            list({"per_page": per_page, "page": page}.items())
-            + list(query_args.items())
-        )
-    )
+    all_query_args = {}
+    if per_page:
+        all_query_args["per_page"] = per_page
+    if page:
+        all_query_args["page"] = page
+    if query_args:
+        all_query_args.update(query_args)
 
-    request = Request(template + "?" + querystring)
+    request_url = template
+    if all_query_args:
+        querystring = urlencode(all_query_args)
+        request_url = template + "?" + querystring
+    else:
+        querystring = ""
+
+    request = Request(request_url)
     if auth is not None:
         if not as_app:
             if fine:
@@ -711,7 +748,11 @@ def _construct_request(
             request.add_header(
                 "Accept", "application/vnd.github.machine-man-preview+json"
             )
-    logger.info("Requesting {}?{}".format(template, querystring))
+
+    log_url = template
+    if querystring:
+        log_url += "?" + querystring
+    logger.info("Requesting {}".format(log_url))
     return request
 
 
@@ -782,13 +823,15 @@ def download_file(url, path, auth, as_app=False, fine=False):
     if os.path.exists(path):
         return
 
-    request = _construct_request(per_page=100,
+    request = _construct_request(
+        per_page=100,
         page=1,
         query_args={},
         template=url,
         auth=auth,
         as_app=as_app,
-                                 fine=fine)
+        fine=fine,
+    )
     request.add_header("Accept", "application/octet-stream")
     opener = build_opener(S3HTTPRedirectHandler)
 
@@ -859,9 +902,13 @@ def retrieve_repositories(args, authenticated_user):
         )
 
     if args.repository:
+        if "/" in args.repository:
+            repo_path = args.repository
+        else:
+            repo_path = "{0}/{1}".format(args.user, args.repository)
         single_request = True
-        template = "https://{0}/repos/{1}/{2}".format(
-            get_github_api_host(args), args.user, args.repository
+        template = "https://{0}/repos/{1}".format(
+            get_github_api_host(args), repo_path
         )
 
     repos = retrieve_data(args, template, single_request=single_request)
@@ -902,6 +949,8 @@ def retrieve_repositories(args, authenticated_user):
 
 
 def filter_repositories(args, unfiltered_repositories):
+    if args.repository:
+        return unfiltered_repositories
     logger.info("Filtering repositories")
 
     repositories = []
@@ -931,11 +980,15 @@ def filter_repositories(args, unfiltered_repositories):
             if r.get("language") and r.get("language").lower() in languages
         ]  # noqa
     if name_regex:
-        repositories = [r for r in repositories if name_regex.match(r["name"])]
+        repositories = [
+            r for r in repositories if "name" not in r or name_regex.match(r["name"])
+        ]
     if args.skip_archived:
         repositories = [r for r in repositories if not r.get("archived")]
     if args.exclude:
-        repositories = [r for r in repositories if r["name"] not in args.exclude]
+        repositories = [
+            r for r in repositories if "name" not in r or r["name"] not in args.exclude
+        ]
 
     return repositories
 
@@ -1090,6 +1143,14 @@ def backup_issues(args, repo_cwd, repository, repos_template):
     comments_template = _issue_template + "/{0}/comments"
     events_template = _issue_template + "/{0}/events"
     for number, issue in list(issues.items()):
+        issue_file = "{0}/{1}.json".format(issue_cwd, number)
+        if args.incremental_by_files and os.path.isfile(issue_file):
+            modified = os.path.getmtime(issue_file)
+            modified = datetime.fromtimestamp(modified).strftime("%Y-%m-%dT%H:%M:%SZ")
+            if modified > issue["updated_at"]:
+                logger.info("Skipping issue {0} because it wasn't modified since last backup".format(number))
+                continue
+
         if args.include_issue_comments or args.include_everything:
             template = comments_template.format(number)
             issues[number]["comment_data"] = retrieve_data(args, template)
@@ -1097,9 +1158,9 @@ def backup_issues(args, repo_cwd, repository, repos_template):
             template = events_template.format(number)
             issues[number]["event_data"] = retrieve_data(args, template)
 
-        issue_file = "{0}/{1}.json".format(issue_cwd, number)
-        with codecs.open(issue_file, "w", encoding="utf-8") as f:
+        with codecs.open(issue_file + ".temp", "w", encoding="utf-8") as f:
             json_dump(issue, f)
+            os.rename(issue_file + ".temp", issue_file)  # Unlike json_dump, this is atomic
 
 
 def backup_pulls(args, repo_cwd, repository, repos_template):
@@ -1152,6 +1213,13 @@ def backup_pulls(args, repo_cwd, repository, repos_template):
     comments_template = _pulls_template + "/{0}/comments"
     commits_template = _pulls_template + "/{0}/commits"
     for number, pull in list(pulls.items()):
+        pull_file = "{0}/{1}.json".format(pulls_cwd, number)
+        if args.incremental_by_files and os.path.isfile(pull_file):
+            modified = os.path.getmtime(pull_file)
+            modified = datetime.fromtimestamp(modified).strftime("%Y-%m-%dT%H:%M:%SZ")
+            if modified > pull["updated_at"]:
+                logger.info("Skipping pull request {0} because it wasn't modified since last backup".format(number))
+                continue
         if args.include_pull_comments or args.include_everything:
             template = comments_regular_template.format(number)
             pulls[number]["comment_regular_data"] = retrieve_data(args, template)
@@ -1161,9 +1229,9 @@ def backup_pulls(args, repo_cwd, repository, repos_template):
             template = commits_template.format(number)
             pulls[number]["commit_data"] = retrieve_data(args, template)
 
-        pull_file = "{0}/{1}.json".format(pulls_cwd, number)
-        with codecs.open(pull_file, "w", encoding="utf-8") as f:
+        with codecs.open(pull_file + ".temp", "w", encoding="utf-8") as f:
             json_dump(pull, f)
+            os.rename(pull_file + ".temp", pull_file)  # Unlike json_dump, this is atomic
 
 
 def backup_milestones(args, repo_cwd, repository, repos_template):
@@ -1231,10 +1299,16 @@ def backup_releases(args, repo_cwd, repository, repos_template, include_assets=F
     if args.skip_prerelease:
         releases = [r for r in releases if not r["prerelease"] and not r["draft"]]
 
-    if args.number_of_latest_releases and args.number_of_latest_releases < len(releases):
-        releases.sort(key=lambda item: datetime.strptime(item["created_at"], "%Y-%m-%dT%H:%M:%SZ"),
-                      reverse=True)
-        releases = releases[:args.number_of_latest_releases]
+    if args.number_of_latest_releases and args.number_of_latest_releases < len(
+        releases
+    ):
+        releases.sort(
+            key=lambda item: datetime.strptime(
+                item["created_at"], "%Y-%m-%dT%H:%M:%SZ"
+            ),
+            reverse=True,
+        )
+        releases = releases[: args.number_of_latest_releases]
         logger.info("Saving the latest {0} releases to disk".format(len(releases)))
     else:
         logger.info("Saving {0} releases to disk".format(len(releases)))
@@ -1259,9 +1333,9 @@ def backup_releases(args, repo_cwd, repository, repos_template, include_assets=F
                     download_file(
                         asset["url"],
                         os.path.join(release_assets_cwd, asset["name"]),
-                        get_auth(args),
+                        get_auth(args, encode=not args.as_app),
                         as_app=args.as_app,
-                        fine=True if args.token_fine is not None else False
+                        fine=True if args.token_fine is not None else False,
                     )
 
 
@@ -1318,10 +1392,12 @@ def fetch_repository(
             git_command = ["git", "remote", "set-url", "origin", remote_url]
             logging_subprocess(git_command, cwd=local_dir)
 
+        git_command = ["git", "fetch", "--all", "--force", "--tags", "--prune"]
+        if no_prune:
+            git_command.pop()
+        logging_subprocess(git_command, cwd=local_dir)
         if lfs_clone:
             git_command = ["git", "lfs", "fetch", "--all", "--prune"]
-        else:
-            git_command = ["git", "fetch", "--all", "--force", "--tags", "--prune"]
             if no_prune:
                 git_command.pop()
             logging_subprocess(git_command, cwd=local_dir)

release-requirements.txt

@@ -1,38 +1,39 @@
-autopep8==2.0.4
-black==24.2.0
-bleach==6.1.0
-certifi==2024.2.2
-charset-normalizer==3.3.2
-click==8.1.7
+autopep8==2.3.2
+black==25.1.0
+bleach==6.2.0
+certifi==2025.7.14
+charset-normalizer==3.4.2
+click==8.1.8
 colorama==0.4.6
-docutils==0.20.1
-flake8==7.0.0
+docutils==0.22
+flake8==7.3.0
 gitchangelog==3.0.4
-idna==3.6
-importlib-metadata==7.0.2
-jaraco.classes==3.3.1
-keyring==24.3.1
+idna==3.10
+importlib-metadata==8.7.0
+jaraco.classes==3.4.0
+keyring==25.6.0
 markdown-it-py==3.0.0
 mccabe==0.7.0
 mdurl==0.1.2
-more-itertools==10.2.0
-mypy-extensions==1.0.0
-packaging==24.0
+more-itertools==10.7.0
+mypy-extensions==1.1.0
+packaging==25.0
 pathspec==0.12.1
-pkginfo==1.10.0
-platformdirs==4.2.0
-pycodestyle==2.11.1
-pyflakes==3.2.0
-Pygments==2.17.2
-readme-renderer==43.0
-requests==2.31.0
+pkginfo==1.12.1.2
+platformdirs==4.3.8
+pycodestyle==2.14.0
+pyflakes==3.4.0
+Pygments==2.19.2
+readme-renderer==44.0
+requests==2.32.4
 requests-toolbelt==1.0.0
 restructuredtext-lint==1.4.0
 rfc3986==2.0.0
-rich==13.7.1
-six==1.16.0
-tqdm==4.66.2
-twine==5.0.0
-urllib3==2.2.1
+rich==14.1.0
+setuptools==80.9.0
+six==1.17.0
+tqdm==4.67.1
+twine==6.1.0
+urllib3==2.5.0
 webencodings==0.5.1
-zipp==3.18.1
+zipp==3.23.0

setup.py

@@ -40,10 +40,11 @@ setup(
         "Development Status :: 5 - Production/Stable",
         "Topic :: System :: Archiving :: Backup",
         "License :: OSI Approved :: MIT License",
-        "Programming Language :: Python :: 3.5",
-        "Programming Language :: Python :: 3.6",
-        "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
     ],
     description="backup a github user or organization",
     long_description=open_file("README.rst").read(),