Release version 0.54.0

docs: update README testing section and add fetch vs pull explanation

.dockerignore

@@ -0,0 +1,75 @@
+# Docker ignore file to reduce build context size
+
+# Temp files
+*~
+~*
+.*~
+\#*
+.#*
+*#
+dist
+
+# Build files
+build
+dist
+pkg
+*.egg
+*.egg-info
+
+# Debian Files
+debian/files
+debian/python-github-backup*
+
+# Sphinx build
+doc/_build
+
+# Generated man page
+doc/github_backup.1
+
+# Annoying macOS files
+.DS_Store
+._*
+
+# IDE configuration files
+.vscode
+.atom
+.idea
+*.code-workspace
+
+# RSA
+id_rsa
+id_rsa.pub
+
+# Virtual env
+venv
+.venv
+
+# Git
+.git
+.gitignore
+.gitchangelog.rc
+.github
+
+# Documentation
+*.md
+!README.md
+
+# Environment variables files
+.env
+.env.*
+!.env.example
+*.log
+
+# Cache files
+**/__pycache__/
+*.py[cod]
+
+# Docker files
+docker-compose.yml
+Dockerfile*
+
+# Other files
+release
+*.tar
+*.zip
+*.gzip

.gitchangelog.rc

@@ -0,0 +1,117 @@
+#
+# Format
+#
+#   ACTION: [AUDIENCE:] COMMIT_MSG [@TAG ...]
+#
+# Description
+#
+#   ACTION is one of 'chg', 'fix', 'new'
+#
+#       Is WHAT the change is about.
+#
+#       'chg' is for refactor, small improvement, cosmetic changes...
+#       'fix' is for bug fixes
+#       'new' is for new features, big improvement
+#
+#   SUBJECT is optional and one of 'dev', 'usr', 'pkg', 'test', 'doc'
+#
+#       Is WHO is concerned by the change.
+#
+#       'dev'  is for developpers (API changes, refactors...)
+#       'usr'  is for final users (UI changes)
+#       'pkg'  is for packagers   (packaging changes)
+#       'test' is for testers     (test only related changes)
+#       'doc'  is for doc guys    (doc only changes)
+#
+#   COMMIT_MSG is ... well ... the commit message itself.
+#
+#   TAGs are additionnal adjective as 'refactor' 'minor' 'cosmetic'
+#
+#       'refactor' is obviously for refactoring code only
+#       'minor' is for a very meaningless change (a typo, adding a comment)
+#       'cosmetic' is for cosmetic driven change (re-indentation, 80-col...)
+#
+# Example:
+#
+#   new: usr: support of bazaar implemented
+#   chg: re-indentend some lines @cosmetic
+#   new: dev: updated code to be compatible with last version of killer lib.
+#   fix: pkg: updated year of licence coverage.
+#   new: test: added a bunch of test around user usability of feature X.
+#   fix: typo in spelling my name in comment. @minor
+#
+#   Please note that multi-line commit message are supported, and only the
+#   first line will be considered as the "summary" of the commit message. So
+#   tags, and other rules only applies to the summary.  The body of the commit
+#   message will be displayed in the changelog with minor reformating.
+
+#
+# ``ignore_regexps`` is a line of regexps
+#
+# Any commit having its full commit message matching any regexp listed here
+# will be ignored and won't be reported in the changelog.
+#
+ignore_regexps = [
+    r'(?i)^(Merge pull request|Merge branch|Release|Update)',
+]
+
+
+#
+# ``replace_regexps`` is a dict associating a regexp pattern and its replacement
+#
+# It will be applied to get the summary line from the full commit message.
+#
+# Note that you can provide multiple replacement patterns, they will be all
+# tried. If None matches, the summary line will be the full commit message.
+#
+replace_regexps = {
+    # current format (ie: 'chg: dev: my commit msg @tag1 @tag2')
+
+    r'^([cC]hg|[fF]ix|[nN]ew)\s*:\s*((dev|use?r|pkg|test|doc)\s*:\s*)?([^\n@]*)(@[a-z]+\s+)*$':
+    r'\4',
+}
+
+
+# ``section_regexps`` is a list of 2-tuples associating a string label and a
+# list of regexp
+#
+# Commit messages will be classified in sections thanks to this. Section
+# titles are the label, and a commit is classified under this section if any
+# of the regexps associated is matching.
+#
+section_regexps = [
+    ('New', [
+        r'^[nN]ew\s*:\s*((dev|use?r|pkg|test|doc)\s*:\s*)?([^\n]*)$',
+    ]),
+    ('Changes', [
+        r'^[cC]hg\s*:\s*((dev|use?r|pkg|test|doc)\s*:\s*)?([^\n]*)$',
+    ]),
+    ('Fix', [
+        r'^[fF]ix\s*:\s*((dev|use?r|pkg|test|doc)\s*:\s*)?([^\n]*)$',
+    ]),
+    ('Other', None  # Match all lines
+     ),
+
+]
+
+# ``body_split_regexp`` is a regexp
+#
+# Commit message body (not the summary) if existing will be split
+# (new line) on this regexp
+#
+body_split_regexp = r'[\n-]'
+
+
+# ``tag_filter_regexp`` is a regexp
+#
+# Tags that will be used for the changelog must match this regexp.
+#
+# tag_filter_regexp = r'^[0-9]+$'
+tag_filter_regexp = r'^(?:[vV])?[0-9\.]+$'
+
+
+# ``unreleased_version_label`` is a string
+#
+# This label will be used as the changelog Title of the last set of changes
+# between last valid tag and HEAD if any.
+unreleased_version_label = "%%version%% (unreleased)"

.github/ISSUE_TEMPLATE/bug.yaml

@@ -0,0 +1,28 @@
+---
+name: Bug Report
+description: File a bug report.
+body:
+  - type: markdown
+    attributes:
+      value: |
+        # Important notice regarding filed issues
+
+        This project already fills my needs, and as such I have no real reason to continue it's development. This project is otherwise provided as is, and no support is given.
+
+        If pull requests implementing bug fixes or enhancements are pushed, I am happy to review and merge them (time permitting).
+
+        If you wish to have a bug fixed, you have a few options:
+
+        - Fix it yourself and file a pull request.
+        - File a bug and hope someone else fixes it for you.
+        - Pay me to fix it (my rate is $200 an hour, minimum 1 hour, contact me via my [github email address](https://github.com/josegonzalez) if you want to go this route).
+
+        In all cases, feel free to file an issue, they may be of help to others in the future.
+  - type: textarea
+    id: what-happened
+    attributes:
+      label: What happened?
+      description: Also tell us, what did you expect to happen?
+      placeholder: Tell us what you see!
+    validations:
+      required: true

.github/ISSUE_TEMPLATE/feature.yaml

@@ -0,0 +1,27 @@
+---
+name: Feature Request
+description: File a feature request.
+body:
+  - type: markdown
+    attributes:
+      value: |
+        # Important notice regarding filed issues
+
+        This project already fills my needs, and as such I have no real reason to continue it's development. This project is otherwise provided as is, and no support is given.
+
+        If pull requests implementing bug fixes or enhancements are pushed, I am happy to review and merge them (time permitting).
+
+        If you wish to have a feature implemented, you have a few options:
+
+        - Implement it yourself and file a pull request.
+        - File an issue and hope someone else implements it for you.
+        - Pay me to implement it (my rate is $200 an hour, minimum 1 hour, contact me via my [github email address](https://github.com/josegonzalez) if you want to go this route).
+
+        In all cases, feel free to file an issue, they may be of help to others in the future.
+  - type: textarea
+    id: what-would-you-like-to-happen
+    attributes:
+      label: What would you like to happen?
+      description: Please describe in detail how the new functionality should work as well as any issues with existing functionality.
+    validations:
+      required: true

.github/PULL_REQUEST.md

@@ -0,0 +1,7 @@
+# Important notice regarding filed pull requests
+
+This project already fills my needs, and as such I have no real reason to continue it's development. This project is otherwise provided as is, and no support is given.
+
+I will attempt to review pull requests at _my_ earliest convenience. If I am unable to get to your pull request in a timely fashion, it is what it is. This repository does not pay any bills, and I am not required to merge any pull request from any individual.
+
+If you wish to jump my personal priority queue, you may pay me for my time to review. My rate is $200 an hour - minimum 1 hour - feel free contact me via my github email address if you want to go this route.

.github/dependabot.yml

@@ -0,0 +1,15 @@
+version: 2
+updates:
+- package-ecosystem: pip
+  directory: "/"
+  schedule:
+    interval: daily
+    time: "13:00"
+  groups:
+    python-packages:
+      patterns:
+        - "*"
+- package-ecosystem: "github-actions"
+  directory: "/"
+  schedule:
+    interval: "weekly"

.github/workflows/automatic-release.yml

@@ -0,0 +1,41 @@
+name: automatic-release
+
+on:
+    workflow_dispatch:
+        inputs:
+            release_type:
+                description: Release type
+                required: true
+                type: choice
+                options:
+                    - patch
+                    - minor
+                    - major
+
+jobs:
+    release:
+        name: Release
+        runs-on: ubuntu-24.04
+        steps:
+            - name: Checkout repository
+              uses: actions/checkout@v6
+              with:
+                fetch-depth: 0
+                ssh-key: ${{ secrets.DEPLOY_PRIVATE_KEY }}
+            - name: Setup Git
+              run: |
+                git config --local user.email "action@github.com"
+                git config --local user.name "GitHub Action"
+            - name: Setup Python
+              uses: actions/setup-python@v6
+              with:
+                python-version: '3.12'
+            - name: Install prerequisites
+              run: pip install -r release-requirements.txt
+            - name: Execute release
+              env:
+                SEMVER_BUMP: ${{ github.event.inputs.release_type }}
+                TWINE_REPOSITORY: ${{ vars.TWINE_REPOSITORY }}
+                TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
+                TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
+              run: ./release $SEMVER_BUMP

.github/workflows/docker.yml

@@ -0,0 +1,77 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: Create and publish a Docker image
+
+on:
+  push:
+    branches:
+      - 'master'
+      - 'main'
+      - 'dev'
+
+    tags:
+      - 'v*'
+      - 'v*.*'
+      - 'v*.*.*'
+      - '*'
+      - '*.*'
+      - '*.*.*'
+  pull_request:
+    branches:
+      - 'main'
+      - 'dev'
+
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+            type=semver,pattern={{major}}
+            type=sha
+            type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }}
+
+      - name: Build and push Docker image
+        uses:  docker/build-push-action@v6
+        with:
+          context: .
+          push: true
+          platforms: linux/amd64,linux/arm64
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/lint.yml

@@ -0,0 +1,36 @@
+---
+name: "lint"
+
+# yamllint disable-line rule:truthy
+on:
+  pull_request:
+    branches:
+      - "*"
+  push:
+    branches:
+      - "main"
+      - "master"
+
+jobs:
+  lint:
+    name: lint
+    runs-on: ubuntu-24.04
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+      - name: Setup Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+      - run: pip install -r release-requirements.txt && pip install wheel
+      - run: flake8 --ignore=E501,E203,W503
+      - run: black .
+      - run: rst-lint README.rst
+      - run: python setup.py sdist bdist_wheel && twine check dist/*

.github/workflows/tagged-release.yml

@@ -0,0 +1,19 @@
+---
+name: "tagged-release"
+
+# yamllint disable-line rule:truthy
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  tagged-release:
+    name: tagged-release
+    runs-on: ubuntu-24.04
+
+    steps:
+      - uses: "marvinpinto/action-automatic-releases@v1.2.1"
+        with:
+          repo_token: "${{ secrets.GITHUB_TOKEN }}"
+          prerelease: false

.github/workflows/test.yml

@@ -0,0 +1,33 @@
+---
+name: "test"
+
+# yamllint disable-line rule:truthy
+on:
+  pull_request:
+    branches:
+      - "*"
+  push:
+    branches:
+      - "main"
+      - "master"
+
+jobs:
+  test:
+    name: test
+    runs-on: ubuntu-24.04
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+      - name: Setup Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+      - run: pip install -r release-requirements.txt
+      - run: pytest tests/ -v

.gitignore

@@ -1,4 +1,4 @@
-*.py[oc]
+*.py[cod]
 
 # Temp files
 *~
@@ -18,10 +18,29 @@ pkg
 
 # Debian Files
 debian/files
-debian/python-aws-hostname*
+debian/python-github-backup*
 
 # Sphinx build
 doc/_build
 
 # Generated man page
-doc/aws_hostname.1
+doc/github_backup.1
+
+# Annoying macOS files
+.DS_Store
+._*
+
+# IDE configuration files
+.vscode
+.atom
+.idea
+
+README
+
+# RSA
+id_rsa
+id_rsa.pub
+
+# Virtual env
+venv
+.venv

Dockerfile

@@ -0,0 +1,38 @@
+FROM python:3.12-alpine3.22 AS builder
+
+RUN pip install --no-cache-dir --upgrade pip \
+    && pip install --no-cache-dir uv
+
+WORKDIR /app
+
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=requirements.txt,target=requirements.txt \
+    --mount=type=bind,source=release-requirements.txt,target=release-requirements.txt \
+    uv venv \
+    && uv pip install -r release-requirements.txt
+
+COPY . .
+
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv pip install .
+
+
+FROM python:3.12-alpine3.22
+ENV PYTHONUNBUFFERED=1
+
+RUN apk add --no-cache \
+    ca-certificates \
+    git \
+    git-lfs \
+    && addgroup -g 1000 appuser \
+    && adduser -D -u 1000 -G appuser appuser
+
+COPY --from=builder --chown=appuser:appuser /app /app
+
+WORKDIR /app
+
+USER appuser
+
+ENV PATH="/app/.venv/bin:$PATH"
+
+ENTRYPOINT ["github-backup"]

README.rst

@@ -4,11 +4,12 @@ github-backup
 
 |PyPI| |Python Versions|
 
-backup a github user or organization
+The package can be used to backup an *entire* `Github <https://github.com/>`_ organization, repository or user account, including starred repos, issues and wikis in the most appropriate format (clones for wikis, json files for issues).
 
 Requirements
 ============
 
+- Python 3.10 or higher
 - GIT 1.9+
 
 Installation
@@ -18,27 +19,40 @@ Using PIP via PyPI::
 
     pip install github-backup
 
-Using PIP via Github::
+Using PIP via Github (more likely the latest version)::
 
     pip install git+https://github.com/josegonzalez/python-github-backup.git#egg=github-backup
     
-Usage
+*Install note for python newcomers:*
-=====
 
-CLI Usage is as follows::
+Python scripts are unlikely to be included in your ``$PATH`` by default, this means it cannot be run directly in terminal with ``$ github-backup ...``, you can either add python's install path to your environments ``$PATH`` or call the script directly e.g. using ``$ ~/.local/bin/github-backup``.*
 
-    github-backup [-h] [-u USERNAME] [-p PASSWORD] [-t TOKEN]
+Basic Help
-                  [-o OUTPUT_DIRECTORY] [-i] [--starred] [--all-starred]
+==========
-                  [--watched] [--followers] [--following] [--all]
+
-                  [--issues] [--issue-comments] [--issue-events] [--pulls]
+Show the CLI help output::
-                  [--pull-comments] [--pull-commits] [--labels] [--hooks]
+
-                  [--milestones] [--repositories] [--bare] [--lfs]
+    github-backup -h
-                  [--wikis] [--gists] [--starred-gists] [--skip-existing]
+
-                  [-L [LANGUAGES [LANGUAGES ...]]] [-N NAME_REGEX]
+CLI Help output::
-                  [-H GITHUB_HOST] [-O] [-R REPOSITORY] [-P] [-F]
+
-                  [--prefer-ssh] [-v]
+    github-backup [-h] [-u USERNAME] [-p PASSWORD] [-t TOKEN_CLASSIC]
+                  [-f TOKEN_FINE] [--as-app] [-o OUTPUT_DIRECTORY]
+                  [-l LOG_LEVEL] [-i] [--starred] [--all-starred]
+                  [--watched] [--followers] [--following] [--all] [--issues]
+                  [--issue-comments] [--issue-events] [--pulls]
+                  [--pull-comments] [--pull-commits] [--pull-details]
+                  [--labels] [--hooks] [--milestones] [--repositories]
+                  [--bare] [--lfs] [--wikis] [--gists] [--starred-gists]
+                  [--skip-archived] [--skip-existing] [-L [LANGUAGES ...]]
+                  [-N NAME_REGEX] [-H GITHUB_HOST] [-O] [-R REPOSITORY]
+                  [-P] [-F] [--prefer-ssh] [-v]
                   [--keychain-name OSX_KEYCHAIN_ITEM_NAME]
                   [--keychain-account OSX_KEYCHAIN_ITEM_ACCOUNT]
+                  [--releases] [--latest-releases NUMBER_OF_LATEST_RELEASES]
+                  [--skip-prerelease] [--assets] [--attachments]
+                  [--exclude [REPOSITORY [REPOSITORY ...]]
+                  [--throttle-limit THROTTLE_LIMIT] [--throttle-pause THROTTLE_PAUSE]
                   USER
 
     Backup a github account
@@ -53,24 +67,33 @@ CLI Usage is as follows::
       -p PASSWORD, --password PASSWORD
                             password for basic auth. If a username is given but
                             not a password, the password will be prompted for.
-      -t TOKEN, --token TOKEN
+      -f TOKEN_FINE, --token-fine TOKEN_FINE
-                            personal access or OAuth token, or path to token
+                            fine-grained personal access token or path to token
                             (file://...)
+      -t TOKEN_CLASSIC, --token TOKEN_CLASSIC
+                            personal access, OAuth, or JSON Web token, or path to
+                            token (file://...)
+      --as-app              authenticate as github app instead of as a user.
       -o OUTPUT_DIRECTORY, --output-directory OUTPUT_DIRECTORY
                             directory at which to backup the repositories
+      -l LOG_LEVEL, --log-level LOG_LEVEL
+                            log level to use (default: info, possible levels:
+                            debug, info, warning, error, critical)
       -i, --incremental     incremental backup
+      --incremental-by-files incremental backup using modified time of files
       --starred             include JSON output of starred repositories in backup
-      --all-starred         include starred repositories in backup
+      --all-starred         include starred repositories in backup [*]
-      --watched             include watched repositories in backup
+      --watched             include JSON output of watched repositories in backup
       --followers           include JSON output of followers in backup
       --following           include JSON output of following users in backup
-      --all                 include everything in backup
+      --all                 include everything in backup (not including [*])
       --issues              include issues in backup
       --issue-comments      include issue comments in backup
       --issue-events        include issue events in backup
       --pulls               include pull requests in backup
       --pull-comments       include pull request review comments in backup
       --pull-commits        include pull request commits in backup
+      --pull-details        include more pull request details in backup [*]
       --labels              include labels in backup
       --hooks               include hooks in backup (works only when
                             authenticated)
@@ -78,10 +101,10 @@ CLI Usage is as follows::
       --repositories        include repository clone in backup
       --bare                clone bare repositories
       --lfs                 clone LFS repositories (requires Git LFS to be
-                            installed, https://git-lfs.github.com)
+                            installed, https://git-lfs.github.com) [*]
       --wikis               include wiki clone in backup
-      --gists               include gists in backup
+      --gists               include gists in backup [*]
-      --starred-gists       include starred gists in backup
+      --starred-gists       include starred gists in backup [*]
       --skip-existing       skip project if a backup directory exists
       -L [LANGUAGES [LANGUAGES ...]], --languages [LANGUAGES [LANGUAGES ...]]
                             only allow these languages
@@ -92,8 +115,8 @@ CLI Usage is as follows::
       -O, --organization    whether or not this is an organization user
       -R REPOSITORY, --repository REPOSITORY
                             name of repository to limit backup to
-      -P, --private         include private repositories
+      -P, --private         include private repositories [*]
-      -F, --fork            include forked repositories
+      -F, --fork            include forked repositories [*]
       --prefer-ssh          Clone repositories using SSH instead of HTTPS
       -v, --version         show program's version number and exit
       --keychain-name OSX_KEYCHAIN_ITEM_NAME
@@ -102,17 +125,65 @@ CLI Usage is as follows::
       --keychain-account OSX_KEYCHAIN_ITEM_ACCOUNT
                             OSX ONLY: account field of password item in OSX
                             keychain that holds the personal access or OAuth token
+      --releases            include release information, not including assets or
+                            binaries
+      --latest-releases NUMBER_OF_LATEST_RELEASES
+                            include certain number of the latest releases;
+                            only applies if including releases
+      --skip-prerelease     skip prerelease and draft versions; only applies if including releases
+      --assets              include assets alongside release information; only
+                            applies if including releases
+      --attachments         download user-attachments from issues and pull requests
+                            to issues/attachments/{issue_number}/ and
+                            pulls/attachments/{pull_number}/ directories
+      --exclude [REPOSITORY [REPOSITORY ...]]
+                            names of repositories to exclude from backup.
+      --throttle-limit THROTTLE_LIMIT
+                            start throttling of GitHub API requests after this
+                            amount of API requests remain
+      --throttle-pause THROTTLE_PAUSE
+                            wait this amount of seconds when API request
+                            throttling is active (default: 30.0, requires
+                            --throttle-limit to be set)
 
 
-The package can be used to backup an *entire* organization or repository, including issues and wikis in the most appropriate format (clones for wikis, json files for issues).
+Usage Details
+=============
 
 Authentication
-==============
+--------------
+
+**Password-based authentication** will fail if you have two-factor authentication enabled, and will `be deprecated <https://github.blog/2023-03-09-raising-the-bar-for-software-security-github-2fa-begins-march-13/>`_ by 2023 EOY.
+
+``--username`` is used for basic password authentication and separate from the positional argument ``USER``, which specifies the user account you wish to back up.
+
+**Classic tokens** are `slightly less secure <https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#personal-access-tokens-classic>`_ as they provide very coarse-grained permissions.
+
+If you need authentication for long-running backups (e.g. for a cron job) it is recommended to use **fine-grained personal access token** ``-f TOKEN_FINE``.
+
+
+Fine Tokens
+~~~~~~~~~~~
+
+You can "generate new token", choosing the repository scope by selecting specific repos or all repos. On Github this is under *Settings -> Developer Settings -> Personal access tokens -> Fine-grained Tokens*
+
+Customise the permissions for your use case, but for a personal account full backup you'll need to enable the following permissions:
+
+**User permissions**: Read access to followers, starring, and watching.
+
+**Repository permissions**: Read access to contents, issues, metadata, pull requests, and webhooks.
+
+
+Prefer SSH
+~~~~~~~~~~
+
+If cloning repos is enabled with ``--repositories``, ``--all-starred``, ``--wikis``, ``--gists``, ``--starred-gists`` using the ``--prefer-ssh`` argument will use ssh for cloning the git repos, but all other connections will still use their own protocol, e.g. API requests for issues uses HTTPS.
+
+To clone with SSH, you'll need SSH authentication setup `as usual with Github <https://docs.github.com/en/authentication/connecting-to-github-with-ssh>`_, e.g. via SSH public and private keys.
 
-Note: Password-based authentication will fail if you have two-factor authentication enabled.
 
 Using the Keychain on Mac OSX
-=============================
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Note: On Mac OSX the token can be stored securely in the user's keychain. To do this:
 
 1. Open Keychain from "Applications -> Utilities -> Keychain Access"
@@ -126,30 +197,197 @@ Note:  When you run github-backup, you will be asked whether you want to allow "
 1. **Allow:** In this case you will need to click "Allow" each time you run `github-backup`
 2. **Always Allow:** In this case, you will not be asked for permission when you run `github-backup` in future. This is less secure, but is required if you want to schedule `github-backup` to run automatically
 
-About Git LFS
-=============
 
-When you use the "--lfs" option, you will need to make sure you have Git LFS installed.
+Github Rate-limit and Throttling
+--------------------------------
+
+"github-backup" will automatically throttle itself based on feedback from the Github API. 
+
+Their API is usually rate-limited to 5000 calls per hour. The API will ask github-backup to pause until a specific time when the limit is reset again (at the start of the next hour). This continues until the backup is complete.
+
+During a large backup, such as ``--all-starred``, and on a fast connection this can result in (~20 min) pauses with bursts of API calls periodically maxing out the API limit. If this is not suitable `it has been observed <https://github.com/josegonzalez/python-github-backup/issues/76#issuecomment-636158717>`_ under real-world conditions that overriding the throttle with ``--throttle-limit 5000 --throttle-pause 0.6`` provides a smooth rate across the hour, although a ``--throttle-pause 0.72`` (3600 seconds [1 hour] / 5000 limit) is theoretically safer to prevent large rate-limit pauses.
+
+
+About Git LFS
+-------------
+
+When you use the ``--lfs`` option, you will need to make sure you have Git LFS installed.
 
 Instructions on how to do this can be found on https://git-lfs.github.com.
 
-Examples
-========
 
-Backup all repositories::
+About Attachments
+-----------------
+
+When you use the ``--attachments`` option with ``--issues`` or ``--pulls``, the tool will download user-uploaded attachments (images, videos, documents, etc.) from issue and pull request descriptions and comments. In some circumstances attachments contain valuable data related to the topic, and without their backup important information or context might be lost inadvertently.
+
+Attachments are saved to ``issues/attachments/{issue_number}/`` and ``pulls/attachments/{pull_number}/`` directories, where ``{issue_number}`` is the GitHub issue number (e.g., issue #123 saves to ``issues/attachments/123/``). Each attachment directory contains:
+
+- The downloaded attachment files (named by their GitHub identifier with appropriate file extensions)
+- If multiple attachments have the same filename, conflicts are resolved with numeric suffixes (e.g., ``report.pdf``, ``report_1.pdf``, ``report_2.pdf``)
+- A ``manifest.json`` file documenting all downloads, including URLs, file metadata, and download status
+
+The tool automatically extracts file extensions from HTTP headers to ensure files can be more easily opened by your operating system.
+
+**Supported URL formats:**
+
+- Modern: ``github.com/user-attachments/{assets,files}/*``
+- Legacy: ``user-images.githubusercontent.com/*`` and ``private-user-images.githubusercontent.com/*``
+- Repo files: ``github.com/{owner}/{repo}/files/*`` (filtered to current repository)
+- Repo assets: ``github.com/{owner}/{repo}/assets/*`` (filtered to current repository)
+
+**Repository filtering** for repo files/assets handles renamed and transferred repositories gracefully. URLs are included if they either match the current repository name directly, or redirect to it (e.g., ``willmcgugan/rich`` redirects to ``Textualize/rich`` after transfer).
+
+
+Run in Docker container
+-----------------------
+
+To run the tool in a Docker container use the following command:
+
+    sudo docker run --rm -v /path/to/backup:/data --name github-backup ghcr.io/josegonzalez/python-github-backup -o /data $OPTIONS $USER
+
+Gotchas / Known-issues
+======================
+
+All is not everything
+---------------------
+
+The ``--all`` argument does not include: cloning private repos (``-P, --private``), cloning forks (``-F, --fork``), cloning starred repositories (``--all-starred``), ``--pull-details``, cloning LFS repositories (``--lfs``), cloning gists (``--gists``) or cloning starred gist repos (``--starred-gists``). See examples for more.
+
+Cloning all starred size
+------------------------
+
+Using the ``--all-starred`` argument to clone all starred repositories may use a large amount of storage space, especially if ``--all`` or more arguments are used. e.g. commonly starred repos can have tens of thousands of issues, many large assets and the repo itself etc. Consider just storing links to starred repos in JSON format with ``--starred``.
+
+Incremental Backup
+------------------
+
+Using (``-i, --incremental``) will only request new data from the API **since the last run (successful or not)**. e.g. only request issues from the API since the last run. 
+
+This means any blocking errors on previous runs can cause a large amount of missing data in backups.
+
+Using (``--incremental-by-files``) will request new data from the API **based on when the file was modified on filesystem**. e.g. if you modify the file yourself you may miss something.
+
+Still saver than the previous version.
+
+Specifically, issues and pull requests are handled like this.
+
+Known blocking errors
+---------------------
+
+Some errors will block the backup run by exiting the script. e.g. receiving a 403 Forbidden error from the Github API.
+
+If the incremental argument is used, this will result in the next backup only requesting API data since the last blocked/failed run. Potentially causing unexpected large amounts of missing data.
+
+It's therefore recommended to only use the incremental argument if the output/result is being actively monitored, or complimented with periodic full non-incremental runs, to avoid unexpected missing data in a regular backup runs.
+
+1. **Starred public repo hooks blocking**
+
+   Since the ``--all`` argument includes ``--hooks``, if you use ``--all`` and ``--all-starred`` together to clone a users starred public repositories, the backup will likely error and block the backup continuing. 
+
+   This is due to needing the correct permission for ``--hooks`` on public repos.
+
+
+"bare" is actually "mirror"
+---------------------------
+
+Using the bare clone argument (``--bare``) will actually call git's ``clone --mirror`` command. There's a subtle difference between `bare <https://www.git-scm.com/docs/git-clone#Documentation/git-clone.txt---bare>`_ and `mirror <https://www.git-scm.com/docs/git-clone#Documentation/git-clone.txt---mirror>`_ clone.
+
+*From git docs "Compared to --bare, --mirror not only maps local branches of the source to local branches of the target, it maps all refs (including remote-tracking branches, notes etc.) and sets up a refspec configuration such that all these refs are overwritten by a git remote update in the target repository."*
+
+
+Starred gists vs starred repo behaviour
+---------------------------------------
+
+The starred normal repo cloning (``--all-starred``) argument stores starred repos separately to the users own repositories. However, using ``--starred-gists`` will store starred gists within the same directory as the users own gists ``--gists``. Also, all gist repo directory names are IDs not the gist's name.
+
+
+Skip existing on incomplete backups
+-----------------------------------
+
+The ``--skip-existing`` argument will skip a backup if the directory already exists, even if the backup in that directory failed (perhaps due to a blocking error). This may result in unexpected missing data in a regular backup.
+
+
+Updates use fetch, not pull
+---------------------------
+
+When updating an existing repository backup, ``github-backup`` uses ``git fetch`` rather than ``git pull``. This is intentional - a backup tool should reliably download data without risk of failure. Using ``git pull`` would require handling merge conflicts, which adds complexity and could cause backups to fail unexpectedly.
+
+With fetch, **all branches and commits are downloaded** safely into remote-tracking branches. The working directory files won't change, but your backup is complete.
+
+If you look at files directly (e.g., ``cat README.md``), you'll see the old content. The new data is in the remote-tracking branches (confusingly named "remote" but stored locally). To view or use the latest files::
+
+    git show origin/main:README.md           # view a file
+    git merge origin/main                    # update working directory
+
+All branches are backed up as remote refs (``origin/main``, ``origin/feature-branch``, etc.).
+
+If you want to browse files directly without merging, consider using ``--bare`` which skips the working directory entirely - the backup is just the git data.
+
+See `#269 <https://github.com/josegonzalez/python-github-backup/issues/269>`_ for more discussion.
+
+
+Github Backup Examples
+======================
+
+Backup all repositories, including private ones using a classic token::
 
     export ACCESS_TOKEN=SOME-GITHUB-TOKEN
-    github-backup WhiteHouse --token $ACCESS_TOKEN --organization --output-directory /tmp/white-house --repositories
+    github-backup WhiteHouse --token $ACCESS_TOKEN --organization --output-directory /tmp/white-house --repositories --private
 
-Backup a single organization repository with everything else (wiki, pull requests, comments, issues etc)::
+Use a fine-grained access token to backup a single organization repository with everything else (wiki, pull requests, comments, issues etc)::
 
-    export ACCESS_TOKEN=SOME-GITHUB-TOKEN
+    export FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
     ORGANIZATION=docker
     REPO=cli
     # e.g. git@github.com:docker/cli.git
-    github-backup $ORGANIZATION -P -t $ACCESS_TOKEN -o . --all -O -R $REPO
+    github-backup $ORGANIZATION -P -f $FINE_ACCESS_TOKEN -o . --all -O -R $REPO
+
+Quietly and incrementally backup useful Github user data (public and private repos with SSH) including; all issues, pulls, all public starred repos and gists (omitting "hooks", "releases" and therefore "assets" to prevent blocking). *Great for a cron job.* ::
+
+    export FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
+    GH_USER=YOUR-GITHUB-USER
+
+    github-backup -f $FINE_ACCESS_TOKEN --prefer-ssh -o ~/github-backup/ -l error -P -i --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-commits --labels --milestones --repositories --wikis --releases --assets --attachments --pull-details --gists --starred-gists $GH_USER
+    
+Debug an error/block or incomplete backup into a temporary directory. Omit "incremental" to fill a previous incomplete backup. ::
+
+    export FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
+    GH_USER=YOUR-GITHUB-USER
+
+    github-backup -f $FINE_ACCESS_TOKEN -o /tmp/github-backup/ -l debug -P --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-commits --labels --milestones --repositories --wikis --releases --assets --pull-details --gists --starred-gists $GH_USER
+
+
+
+Development
+===========
+
+This project is considered feature complete for the primary maintainer @josegonzalez. If you would like a bugfix or enhancement, pull requests are welcome. Feel free to contact the maintainer for consulting estimates if you'd like to sponsor the work instead.
+
+Contibuters
+-----------
+
+A huge thanks to all the contibuters!
+
+.. image:: https://contrib.rocks/image?repo=josegonzalez/python-github-backup
+   :target: https://github.com/josegonzalez/python-github-backup/graphs/contributors
+   :alt: contributors
+
+Testing
+-------
+
+To run the test suite::
+
+    pip install pytest
+    pytest
+
+To run linting::
+
+    pip install flake8
+    flake8 --ignore=E501
+
 
 .. |PyPI| image:: https://img.shields.io/pypi/v/github-backup.svg
    :target: https://pypi.python.org/pypi/github-backup/
 .. |Python Versions| image:: https://img.shields.io/pypi/pyversions/github-backup.svg
-   :target: https://github.com/albertyw/github-backup
+   :target: https://github.com/josegonzalez/python-github-backup

github_backup/__init__.py

@@ -1 +1 @@
-__version__ = '0.19.1'
+__version__ = "0.54.0"

pytest.ini

@@ -0,0 +1,6 @@
+[pytest]
+testpaths = tests
+python_files = test_*.py
+python_classes = Test*
+python_functions = test_*
+addopts = -v

python-github-backup.code-workspace

@@ -0,0 +1,7 @@
+{
+	"folders": [
+		{
+			"path": "."
+		}
+	]
+}

release

@@ -1,8 +1,14 @@
 #!/usr/bin/env bash
-set -eo pipefail; [[ $RELEASE_TRACE ]] && set -x
+set -eo pipefail
+[[ $RELEASE_TRACE ]] && set -x
 
-PACKAGE_NAME='github-backup'
+if [[ ! -f setup.py ]]; then
-INIT_PACKAGE_NAME='github_backup'
+    echo -e "${RED}WARNING: Missing setup.py${COLOR_OFF}\n"
+    exit 1
+fi
+
+PACKAGE_NAME="$(cat setup.py | grep 'name="' | head | cut -d '"' -f2)"
+INIT_PACKAGE_NAME="$(echo "${PACKAGE_NAME//-/_}")"
 PUBLIC="true"
 
 # Colors
@@ -14,18 +20,23 @@ MAGENTA="\033[0;35m"  # magenta
 CYAN="\033[0;36m"    # cyan
 
 # ensure wheel is available
-pip install wheel > /dev/null
+pip install wheel >/dev/null
 
 command -v gitchangelog >/dev/null 2>&1 || {
-    echo -e "${RED}WARNING: Missing gitchangelog binary, please run: pip install gitchangelog==2.2.0${COLOR_OFF}\n"
+    echo -e "${RED}WARNING: Missing gitchangelog binary, please run: pip install gitchangelog==3.0.4${COLOR_OFF}\n"
     exit 1
 }
 
-command -v rst-lint > /dev/null || {
+command -v rst-lint >/dev/null || {
     echo -e "${RED}WARNING: Missing rst-lint binary, please run: pip install restructuredtext_lint${COLOR_OFF}\n"
     exit 1
 }
 
+command -v twine >/dev/null || {
+    echo -e "${RED}WARNING: Missing twine binary, please run: pip install twine==3.2.0${COLOR_OFF}\n"
+    exit 1
+}
+
 if [[ "$@" != "major" ]] && [[ "$@" != "minor" ]] && [[ "$@" != "patch" ]]; then
     echo -e "${RED}WARNING: Invalid release type, must specify 'major', 'minor', or 'patch'${COLOR_OFF}\n"
     exit 1
@@ -33,33 +44,33 @@ fi
 
 echo -e "\n${GREEN}STARTING RELEASE PROCESS${COLOR_OFF}\n"
 
-set +e;
+set +e
-git status | grep -Eo "working (directory|tree) clean" &> /dev/null
+git status | grep -Eo "working (directory|tree) clean" &>/dev/null
 if [ ! $? -eq 0 ]; then # working directory is NOT clean
     echo -e "${RED}WARNING: You have uncomitted changes, you may have forgotten something${COLOR_OFF}\n"
     exit 1
 fi
-set -e;
+set -e
 
 echo -e "${YELLOW}--->${COLOR_OFF} Updating local copy"
 git pull -q origin master
 
 echo -e "${YELLOW}--->${COLOR_OFF} Retrieving release versions"
 
-current_version=$(cat ${INIT_PACKAGE_NAME}/__init__.py |grep '__version__ ='|sed 's/[^0-9.]//g')
+current_version=$(cat ${INIT_PACKAGE_NAME}/__init__.py | grep '__version__ =' | sed 's/[^0-9.]//g')
 major=$(echo $current_version | awk '{split($0,a,"."); print a[1]}')
 minor=$(echo $current_version | awk '{split($0,a,"."); print a[2]}')
 patch=$(echo $current_version | awk '{split($0,a,"."); print a[3]}')
 
 if [[ "$@" == "major" ]]; then
-    major=$(($major + 1));
+    major=$(($major + 1))
     minor="0"
     patch="0"
 elif [[ "$@" == "minor" ]]; then
-    minor=$(($minor + 1));
+    minor=$(($minor + 1))
     patch="0"
 elif [[ "$@" == "patch" ]]; then
-    patch=$(($patch + 1));
+    patch=$(($patch + 1))
 fi
 
 next_version="${major}.${minor}.${patch}"
@@ -67,7 +78,7 @@ next_version="${major}.${minor}.${patch}"
 echo -e "${YELLOW}   >${COLOR_OFF} ${MAGENTA}${current_version}${COLOR_OFF} -> ${MAGENTA}${next_version}${COLOR_OFF}"
 
 echo -e "${YELLOW}--->${COLOR_OFF} Ensuring readme passes lint checks (if this fails, run rst-lint)"
-rst-lint README.rst > /dev/null
+rst-lint README.rst || exit 1
 
 echo -e "${YELLOW}--->${COLOR_OFF} Creating necessary temp file"
 tempfoo=$(basename $0)
@@ -76,33 +87,29 @@ TMPFILE=$(mktemp /tmp/${tempfoo}.XXXXXX) || {
     exit 1
 }
 
-find_this="__version__ = '$current_version'"
+find_this="__version__ = \"$current_version\""
-replace_with="__version__ = '$next_version'"
+replace_with="__version__ = \"$next_version\""
 
 echo -e "${YELLOW}--->${COLOR_OFF} Updating ${INIT_PACKAGE_NAME}/__init__.py"
-sed "s/$find_this/$replace_with/" ${INIT_PACKAGE_NAME}/__init__.py > $TMPFILE && mv $TMPFILE ${INIT_PACKAGE_NAME}/__init__.py
+sed "s/$find_this/$replace_with/" ${INIT_PACKAGE_NAME}/__init__.py >$TMPFILE && mv $TMPFILE ${INIT_PACKAGE_NAME}/__init__.py
-
-find_this="${PACKAGE_NAME}.git@$current_version"
-replace_with="${PACKAGE_NAME}.git@$next_version"
-
-echo -e "${YELLOW}--->${COLOR_OFF} Updating README.rst"
-sed "s/$find_this/$replace_with/" README.rst > $TMPFILE && mv $TMPFILE README.rst
 
 if [ -f docs/conf.py ]; then
     echo -e "${YELLOW}--->${COLOR_OFF} Updating docs"
     find_this="version = '${current_version}'"
     replace_with="version = '${next_version}'"
-    sed "s/$find_this/$replace_with/" docs/conf.py > $TMPFILE && mv $TMPFILE docs/conf.py
+    sed "s/$find_this/$replace_with/" docs/conf.py >$TMPFILE && mv $TMPFILE docs/conf.py
 
     find_this="version = '${current_version}'"
     replace_with="release = '${next_version}'"
-    sed "s/$find_this/$replace_with/" docs/conf.py > $TMPFILE && mv $TMPFILE docs/conf.py
+    sed "s/$find_this/$replace_with/" docs/conf.py >$TMPFILE && mv $TMPFILE docs/conf.py
 fi
 
 echo -e "${YELLOW}--->${COLOR_OFF} Updating CHANGES.rst for new release"
 version_header="$next_version ($(date +%F))"
-set +e; dashes=$(yes '-'|head -n ${#version_header}|tr -d '\n') ; set -e
+set +e
-gitchangelog |sed "4s/.*/$version_header/"|sed "5s/.*/$dashes/" > $TMPFILE && mv $TMPFILE CHANGES.rst
+dashes=$(yes '-' | head -n ${#version_header} | tr -d '\n')
+set -e
+gitchangelog | sed "4s/.*/$version_header/" | sed "5s/.*/$dashes/" >$TMPFILE && mv $TMPFILE CHANGES.rst
 
 echo -e "${YELLOW}--->${COLOR_OFF} Adding changed files to git"
 git add CHANGES.rst README.rst ${INIT_PACKAGE_NAME}/__init__.py
@@ -111,6 +118,15 @@ if [ -f docs/conf.py ]; then git add docs/conf.py; fi
 echo -e "${YELLOW}--->${COLOR_OFF} Creating release"
 git commit -q -m "Release version $next_version"
 
+if [[ "$PUBLIC" == "true" ]]; then
+    echo -e "${YELLOW}--->${COLOR_OFF} Creating python release files"
+    cp README.rst README
+    python setup.py sdist bdist_wheel >/dev/null
+
+    echo -e "${YELLOW}--->${COLOR_OFF} Validating long_description"
+    twine check dist/*
+fi
+
 echo -e "${YELLOW}--->${COLOR_OFF} Tagging release"
 git tag -a $next_version -m "Release version $next_version"
 
@@ -118,9 +134,8 @@ echo -e "${YELLOW}--->${COLOR_OFF} Pushing release and tags to github"
 git push -q origin master && git push -q --tags
 
 if [[ "$PUBLIC" == "true" ]]; then
-    echo -e "${YELLOW}--->${COLOR_OFF} Creating python release"
+    echo -e "${YELLOW}--->${COLOR_OFF} Uploading python release"
-    cp README.rst README
+    twine upload dist/*
-    python setup.py sdist bdist_wheel upload > /dev/null
     rm README
 fi
 

release-requirements.txt

@@ -0,0 +1,40 @@
+autopep8==2.3.2
+black==25.11.0
+bleach==6.3.0
+certifi==2025.11.12
+charset-normalizer==3.4.4
+click==8.3.1
+colorama==0.4.6
+docutils==0.22.3
+flake8==7.3.0
+gitchangelog==3.0.4
+pytest==9.0.1
+idna==3.11
+importlib-metadata==8.7.0
+jaraco.classes==3.4.0
+keyring==25.7.0
+markdown-it-py==4.0.0
+mccabe==0.7.0
+mdurl==0.1.2
+more-itertools==10.8.0
+mypy-extensions==1.1.0
+packaging==25.0
+pathspec==0.12.1
+pkginfo==1.12.1.2
+platformdirs==4.5.0
+pycodestyle==2.14.0
+pyflakes==3.4.0
+Pygments==2.19.2
+readme-renderer==44.0
+requests==2.32.5
+requests-toolbelt==1.0.0
+restructuredtext-lint==2.0.2
+rfc3986==2.0.0
+rich==14.2.0
+setuptools==80.9.0
+six==1.17.0
+tqdm==4.67.1
+twine==6.2.0
+urllib3==2.5.0
+webencodings==0.5.1
+zipp==3.23.0

requirements.txt

@@ -1 +0,0 @@
-

setup.py

@@ -1,10 +1,12 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 import os
+
 from github_backup import __version__
 
 try:
     from setuptools import setup
+
     setup  # workaround for pyflakes issue #13
 except ImportError:
     from distutils.core import setup
@@ -15,6 +17,7 @@ except ImportError:
 # http://www.eby-sarna.com/pipermail/peak/2010-May/003357.html)
 try:
     import multiprocessing
+
     multiprocessing
 except ImportError:
     pass
@@ -25,25 +28,28 @@ def open_file(fname):
 
 
 setup(
-    name='github-backup',
+    name="github-backup",
     version=__version__,
-    author='Jose Diaz-Gonzalez',
+    author="Jose Diaz-Gonzalez",
-    author_email='github-backup@josediazgonzalez.com',
+    author_email="github-backup@josediazgonzalez.com",
-    packages=['github_backup'],
+    packages=["github_backup"],
-    scripts=['bin/github-backup'],
+    scripts=["bin/github-backup"],
-    url='http://github.com/josegonzalez/python-github-backup',
+    url="http://github.com/josegonzalez/python-github-backup",
-    license=open('LICENSE.txt').read(),
+    license="MIT",
     classifiers=[
-        'Development Status :: 5 - Production/Stable',
+        "Development Status :: 5 - Production/Stable",
-        'Topic :: System :: Archiving :: Backup',
+        "Topic :: System :: Archiving :: Backup",
-        'License :: OSI Approved :: MIT License',
+        "License :: OSI Approved :: MIT License",
-        'Programming Language :: Python :: 2.6',
+        "Programming Language :: Python :: 3.10",
-        'Programming Language :: Python :: 2.7',
+        "Programming Language :: Python :: 3.11",
-        'Programming Language :: Python :: 3.5',
+        "Programming Language :: Python :: 3.12",
-        'Programming Language :: Python :: 3.6',
+        "Programming Language :: Python :: 3.13",
+        "Programming Language :: Python :: 3.14",
     ],
-    description='backup a github user or organization',
+    description="backup a github user or organization",
-    long_description=open_file('README.rst').read(),
+    long_description=open_file("README.rst").read(),
-    install_requires=open_file('requirements.txt').readlines(),
+    long_description_content_type="text/x-rst",
+    install_requires=open_file("requirements.txt").readlines(),
+    python_requires=">=3.10",
     zip_safe=True,
 )

tests/__init__.py

@@ -0,0 +1 @@
+"""Tests for python-github-backup."""

tests/test_attachments.py

@@ -0,0 +1,353 @@
+"""Behavioral tests for attachment functionality."""
+
+import json
+import os
+import tempfile
+from pathlib import Path
+from unittest.mock import Mock
+
+import pytest
+
+from github_backup import github_backup
+
+
+@pytest.fixture
+def attachment_test_setup(tmp_path):
+    """Fixture providing setup and helper for attachment download tests."""
+    from unittest.mock import patch
+
+    issue_cwd = tmp_path / "issues"
+    issue_cwd.mkdir()
+
+    # Mock args
+    args = Mock()
+    args.as_app = False
+    args.token_fine = None
+    args.token_classic = None
+    args.username = None
+    args.password = None
+    args.osx_keychain_item_name = None
+    args.osx_keychain_item_account = None
+    args.user = "testuser"
+    args.repository = "testrepo"
+
+    repository = {"full_name": "testuser/testrepo"}
+
+    def call_download(issue_data, issue_number=123):
+        """Call download_attachments with mocked HTTP downloads.
+
+        Returns list of URLs that were actually downloaded.
+        """
+        downloaded_urls = []
+
+        def mock_download(url, path, auth, as_app, fine):
+            downloaded_urls.append(url)
+            return {
+                "success": True,
+                "saved_as": os.path.basename(path),
+                "url": url,
+            }
+
+        with patch(
+            "github_backup.github_backup.download_attachment_file",
+            side_effect=mock_download,
+        ):
+            github_backup.download_attachments(
+                args, str(issue_cwd), issue_data, issue_number, repository
+            )
+
+        return downloaded_urls
+
+    return {
+        "issue_cwd": str(issue_cwd),
+        "args": args,
+        "repository": repository,
+        "call_download": call_download,
+    }
+
+
+class TestURLExtraction:
+    """Test URL extraction with realistic issue content."""
+
+    def test_mixed_urls(self):
+        issue_data = {
+            "body": """
+            ## Bug Report
+
+            When uploading files, I see this error. Here's a screenshot:
+            https://github.com/user-attachments/assets/abc123def456
+
+            The logs show: https://github.com/user-attachments/files/789/error-log.txt
+
+            This is similar to https://github.com/someorg/somerepo/issues/42 but different.
+
+            You can also see the video at https://user-images.githubusercontent.com/12345/video-demo.mov
+
+            Here's how to reproduce:
+            ```bash
+            # Don't extract this example URL:
+            curl https://github.com/user-attachments/assets/example999
+            ```
+
+            More info at https://docs.example.com/guide
+
+            Also see this inline code `https://github.com/user-attachments/files/111/inline.pdf` should not extract.
+
+            Final attachment: https://github.com/user-attachments/files/222/report.pdf.
+            """,
+            "comment_data": [
+                {
+                    "body": "Here's another attachment: https://private-user-images.githubusercontent.com/98765/secret.png?jwt=token123"
+                },
+                {
+                    "body": """
+                    Example code:
+                    ```python
+                    url = "https://github.com/user-attachments/assets/code-example"
+                    ```
+                    But this is real: https://github.com/user-attachments/files/333/actual.zip
+                    """
+                },
+            ],
+        }
+
+        # Extract URLs
+        urls = github_backup.extract_attachment_urls(issue_data)
+
+        expected_urls = [
+            "https://github.com/user-attachments/assets/abc123def456",
+            "https://github.com/user-attachments/files/789/error-log.txt",
+            "https://user-images.githubusercontent.com/12345/video-demo.mov",
+            "https://github.com/user-attachments/files/222/report.pdf",
+            "https://private-user-images.githubusercontent.com/98765/secret.png?jwt=token123",
+            "https://github.com/user-attachments/files/333/actual.zip",
+        ]
+
+        assert set(urls) == set(expected_urls)
+
+    def test_trailing_punctuation_stripped(self):
+        """URLs with trailing punctuation should have punctuation stripped."""
+        issue_data = {
+            "body": """
+            See this file: https://github.com/user-attachments/files/1/doc.pdf.
+            And this one (https://github.com/user-attachments/files/2/image.png).
+            Check it out! https://github.com/user-attachments/files/3/data.csv!
+            """
+        }
+
+        urls = github_backup.extract_attachment_urls(issue_data)
+
+        expected = [
+            "https://github.com/user-attachments/files/1/doc.pdf",
+            "https://github.com/user-attachments/files/2/image.png",
+            "https://github.com/user-attachments/files/3/data.csv",
+        ]
+        assert set(urls) == set(expected)
+
+    def test_deduplication_across_body_and_comments(self):
+        """Same URL in body and comments should only appear once."""
+        duplicate_url = "https://github.com/user-attachments/assets/abc123"
+
+        issue_data = {
+            "body": f"First mention: {duplicate_url}",
+            "comment_data": [
+                {"body": f"Second mention: {duplicate_url}"},
+                {"body": f"Third mention: {duplicate_url}"},
+            ],
+        }
+
+        urls = github_backup.extract_attachment_urls(issue_data)
+
+        assert set(urls) == {duplicate_url}
+
+
+class TestFilenameExtraction:
+    """Test filename extraction from different URL types."""
+
+    def test_modern_assets_url(self):
+        """Modern assets URL returns UUID."""
+        url = "https://github.com/user-attachments/assets/abc123def456"
+        filename = github_backup.get_attachment_filename(url)
+        assert filename == "abc123def456"
+
+    def test_modern_files_url(self):
+        """Modern files URL returns filename."""
+        url = "https://github.com/user-attachments/files/12345/report.pdf"
+        filename = github_backup.get_attachment_filename(url)
+        assert filename == "report.pdf"
+
+    def test_legacy_cdn_url(self):
+        """Legacy CDN URL returns filename with extension."""
+        url = "https://user-images.githubusercontent.com/123456/abc-def.png"
+        filename = github_backup.get_attachment_filename(url)
+        assert filename == "abc-def.png"
+
+    def test_private_cdn_url(self):
+        """Private CDN URL returns filename."""
+        url = "https://private-user-images.githubusercontent.com/98765/secret.png?jwt=token123"
+        filename = github_backup.get_attachment_filename(url)
+        assert filename == "secret.png"
+
+    def test_repo_files_url(self):
+        """Repo-scoped files URL returns filename."""
+        url = "https://github.com/owner/repo/files/789/document.txt"
+        filename = github_backup.get_attachment_filename(url)
+        assert filename == "document.txt"
+
+
+class TestFilenameCollision:
+    """Test filename collision resolution."""
+
+    def test_collision_behavior(self):
+        """Test filename collision resolution with real files."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            # No collision - file doesn't exist
+            result = github_backup.resolve_filename_collision(
+                os.path.join(tmpdir, "report.pdf")
+            )
+            assert result == os.path.join(tmpdir, "report.pdf")
+
+            # Create the file, now collision exists
+            Path(os.path.join(tmpdir, "report.pdf")).touch()
+            result = github_backup.resolve_filename_collision(
+                os.path.join(tmpdir, "report.pdf")
+            )
+            assert result == os.path.join(tmpdir, "report_1.pdf")
+
+            # Create report_1.pdf too
+            Path(os.path.join(tmpdir, "report_1.pdf")).touch()
+            result = github_backup.resolve_filename_collision(
+                os.path.join(tmpdir, "report.pdf")
+            )
+            assert result == os.path.join(tmpdir, "report_2.pdf")
+
+    def test_manifest_reserved(self):
+        """manifest.json is always treated as reserved."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            # Even if manifest.json doesn't exist, should get manifest_1.json
+            result = github_backup.resolve_filename_collision(
+                os.path.join(tmpdir, "manifest.json")
+            )
+            assert result == os.path.join(tmpdir, "manifest_1.json")
+
+
+class TestManifestDuplicatePrevention:
+    """Test that manifest prevents duplicate downloads (the bug fix)."""
+
+    def test_manifest_filters_existing_urls(self, attachment_test_setup):
+        """URLs in manifest are not re-downloaded."""
+        setup = attachment_test_setup
+
+        # Create manifest with existing URLs
+        attachments_dir = os.path.join(setup["issue_cwd"], "attachments", "123")
+        os.makedirs(attachments_dir)
+        manifest_path = os.path.join(attachments_dir, "manifest.json")
+
+        manifest = {
+            "attachments": [
+                {
+                    "url": "https://github.com/user-attachments/assets/old1",
+                    "success": True,
+                    "saved_as": "old1.pdf",
+                },
+                {
+                    "url": "https://github.com/user-attachments/assets/old2",
+                    "success": True,
+                    "saved_as": "old2.pdf",
+                },
+            ]
+        }
+        with open(manifest_path, "w") as f:
+            json.dump(manifest, f)
+
+        # Issue data with 2 old URLs and 1 new URL
+        issue_data = {
+            "body": """
+            Old: https://github.com/user-attachments/assets/old1
+            Old: https://github.com/user-attachments/assets/old2
+            New: https://github.com/user-attachments/assets/new1
+            """
+        }
+
+        downloaded_urls = setup["call_download"](issue_data)
+
+        # Should only download the NEW URL (old ones filtered by manifest)
+        assert len(downloaded_urls) == 1
+        assert downloaded_urls[0] == "https://github.com/user-attachments/assets/new1"
+
+    def test_no_manifest_downloads_all(self, attachment_test_setup):
+        """Without manifest, all URLs should be downloaded."""
+        setup = attachment_test_setup
+
+        # Issue data with 2 URLs
+        issue_data = {
+            "body": """
+            https://github.com/user-attachments/assets/url1
+            https://github.com/user-attachments/assets/url2
+            """
+        }
+
+        downloaded_urls = setup["call_download"](issue_data)
+
+        # Should download ALL URLs (no manifest to filter)
+        assert len(downloaded_urls) == 2
+        assert set(downloaded_urls) == {
+            "https://github.com/user-attachments/assets/url1",
+            "https://github.com/user-attachments/assets/url2",
+        }
+
+    def test_manifest_skips_permanent_failures(self, attachment_test_setup):
+        """Manifest skips permanent failures (404, 410) but retries transient (503)."""
+        setup = attachment_test_setup
+
+        # Create manifest with different failure types
+        attachments_dir = os.path.join(setup["issue_cwd"], "attachments", "123")
+        os.makedirs(attachments_dir)
+        manifest_path = os.path.join(attachments_dir, "manifest.json")
+
+        manifest = {
+            "attachments": [
+                {
+                    "url": "https://github.com/user-attachments/assets/success",
+                    "success": True,
+                    "saved_as": "success.pdf",
+                },
+                {
+                    "url": "https://github.com/user-attachments/assets/notfound",
+                    "success": False,
+                    "http_status": 404,
+                },
+                {
+                    "url": "https://github.com/user-attachments/assets/gone",
+                    "success": False,
+                    "http_status": 410,
+                },
+                {
+                    "url": "https://github.com/user-attachments/assets/unavailable",
+                    "success": False,
+                    "http_status": 503,
+                },
+            ]
+        }
+        with open(manifest_path, "w") as f:
+            json.dump(manifest, f)
+
+        # Issue data has all 4 URLs
+        issue_data = {
+            "body": """
+            https://github.com/user-attachments/assets/success
+            https://github.com/user-attachments/assets/notfound
+            https://github.com/user-attachments/assets/gone
+            https://github.com/user-attachments/assets/unavailable
+            """
+        }
+
+        downloaded_urls = setup["call_download"](issue_data)
+
+        # Should only retry 503 (transient failure)
+        # Success, 404, and 410 should be skipped
+        assert len(downloaded_urls) == 1
+        assert (
+            downloaded_urls[0]
+            == "https://github.com/user-attachments/assets/unavailable"
+        )

tests/test_http_451.py

@@ -0,0 +1,143 @@
+"""Tests for HTTP 451 (DMCA takedown) handling."""
+
+import json
+from unittest.mock import Mock, patch
+
+import pytest
+
+from github_backup import github_backup
+
+
+class TestHTTP451Exception:
+    """Test suite for HTTP 451 DMCA takedown exception handling."""
+
+    def test_repository_unavailable_error_raised(self):
+        """HTTP 451 should raise RepositoryUnavailableError with DMCA URL."""
+        # Create mock args
+        args = Mock()
+        args.as_app = False
+        args.token_fine = None
+        args.token_classic = None
+        args.username = None
+        args.password = None
+        args.osx_keychain_item_name = None
+        args.osx_keychain_item_account = None
+        args.throttle_limit = None
+        args.throttle_pause = 0
+
+        # Mock HTTPError 451 response
+        mock_response = Mock()
+        mock_response.getcode.return_value = 451
+
+        dmca_data = {
+            "message": "Repository access blocked",
+            "block": {
+                "reason": "dmca",
+                "created_at": "2024-11-12T14:38:04Z",
+                "html_url": "https://github.com/github/dmca/blob/master/2024/11/2024-11-04-source-code.md"
+            }
+        }
+        mock_response.read.return_value = json.dumps(dmca_data).encode("utf-8")
+        mock_response.headers = {"x-ratelimit-remaining": "5000"}
+        mock_response.reason = "Unavailable For Legal Reasons"
+
+        def mock_get_response(request, auth, template):
+            return mock_response, []
+
+        with patch("github_backup.github_backup._get_response", side_effect=mock_get_response):
+            with pytest.raises(github_backup.RepositoryUnavailableError) as exc_info:
+                list(github_backup.retrieve_data_gen(args, "https://api.github.com/repos/test/dmca/issues"))
+
+            # Check exception has DMCA URL
+            assert exc_info.value.dmca_url == "https://github.com/github/dmca/blob/master/2024/11/2024-11-04-source-code.md"
+            assert "451" in str(exc_info.value)
+
+    def test_repository_unavailable_error_without_dmca_url(self):
+        """HTTP 451 without DMCA details should still raise exception."""
+        args = Mock()
+        args.as_app = False
+        args.token_fine = None
+        args.token_classic = None
+        args.username = None
+        args.password = None
+        args.osx_keychain_item_name = None
+        args.osx_keychain_item_account = None
+        args.throttle_limit = None
+        args.throttle_pause = 0
+
+        mock_response = Mock()
+        mock_response.getcode.return_value = 451
+        mock_response.read.return_value = b'{"message": "Blocked"}'
+        mock_response.headers = {"x-ratelimit-remaining": "5000"}
+        mock_response.reason = "Unavailable For Legal Reasons"
+
+        def mock_get_response(request, auth, template):
+            return mock_response, []
+
+        with patch("github_backup.github_backup._get_response", side_effect=mock_get_response):
+            with pytest.raises(github_backup.RepositoryUnavailableError) as exc_info:
+                list(github_backup.retrieve_data_gen(args, "https://api.github.com/repos/test/dmca/issues"))
+
+            # Exception raised even without DMCA URL
+            assert exc_info.value.dmca_url is None
+            assert "451" in str(exc_info.value)
+
+    def test_repository_unavailable_error_with_malformed_json(self):
+        """HTTP 451 with malformed JSON should still raise exception."""
+        args = Mock()
+        args.as_app = False
+        args.token_fine = None
+        args.token_classic = None
+        args.username = None
+        args.password = None
+        args.osx_keychain_item_name = None
+        args.osx_keychain_item_account = None
+        args.throttle_limit = None
+        args.throttle_pause = 0
+
+        mock_response = Mock()
+        mock_response.getcode.return_value = 451
+        mock_response.read.return_value = b"invalid json {"
+        mock_response.headers = {"x-ratelimit-remaining": "5000"}
+        mock_response.reason = "Unavailable For Legal Reasons"
+
+        def mock_get_response(request, auth, template):
+            return mock_response, []
+
+        with patch("github_backup.github_backup._get_response", side_effect=mock_get_response):
+            with pytest.raises(github_backup.RepositoryUnavailableError):
+                list(github_backup.retrieve_data_gen(args, "https://api.github.com/repos/test/dmca/issues"))
+
+    def test_other_http_errors_unchanged(self):
+        """Other HTTP errors should still raise generic Exception."""
+        args = Mock()
+        args.as_app = False
+        args.token_fine = None
+        args.token_classic = None
+        args.username = None
+        args.password = None
+        args.osx_keychain_item_name = None
+        args.osx_keychain_item_account = None
+        args.throttle_limit = None
+        args.throttle_pause = 0
+
+        mock_response = Mock()
+        mock_response.getcode.return_value = 404
+        mock_response.read.return_value = b'{"message": "Not Found"}'
+        mock_response.headers = {"x-ratelimit-remaining": "5000"}
+        mock_response.reason = "Not Found"
+
+        def mock_get_response(request, auth, template):
+            return mock_response, []
+
+        with patch("github_backup.github_backup._get_response", side_effect=mock_get_response):
+            # Should raise generic Exception, not RepositoryUnavailableError
+            with pytest.raises(Exception) as exc_info:
+                list(github_backup.retrieve_data_gen(args, "https://api.github.com/repos/test/notfound/issues"))
+
+            assert not isinstance(exc_info.value, github_backup.RepositoryUnavailableError)
+            assert "404" in str(exc_info.value)
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])

tests/test_json_dump_if_changed.py

@@ -0,0 +1,198 @@
+"""Tests for json_dump_if_changed functionality."""
+
+import codecs
+import json
+import os
+import tempfile
+
+import pytest
+
+from github_backup import github_backup
+
+
+class TestJsonDumpIfChanged:
+    """Test suite for json_dump_if_changed function."""
+
+    def test_writes_new_file(self):
+        """Should write file when it doesn't exist."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            output_file = os.path.join(tmpdir, "test.json")
+            test_data = {"key": "value", "number": 42}
+
+            result = github_backup.json_dump_if_changed(test_data, output_file)
+
+            assert result is True
+            assert os.path.exists(output_file)
+
+            # Verify content matches expected format
+            with codecs.open(output_file, "r", encoding="utf-8") as f:
+                content = f.read()
+                loaded = json.loads(content)
+                assert loaded == test_data
+
+    def test_skips_unchanged_file(self):
+        """Should skip write when content is identical."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            output_file = os.path.join(tmpdir, "test.json")
+            test_data = {"key": "value", "number": 42}
+
+            # First write
+            result1 = github_backup.json_dump_if_changed(test_data, output_file)
+            assert result1 is True
+
+            # Get the initial mtime
+            mtime1 = os.path.getmtime(output_file)
+
+            # Second write with same data
+            result2 = github_backup.json_dump_if_changed(test_data, output_file)
+            assert result2 is False
+
+            # File should not have been modified
+            mtime2 = os.path.getmtime(output_file)
+            assert mtime1 == mtime2
+
+    def test_writes_when_content_changed(self):
+        """Should write file when content has changed."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            output_file = os.path.join(tmpdir, "test.json")
+            test_data1 = {"key": "value1"}
+            test_data2 = {"key": "value2"}
+
+            # First write
+            result1 = github_backup.json_dump_if_changed(test_data1, output_file)
+            assert result1 is True
+
+            # Second write with different data
+            result2 = github_backup.json_dump_if_changed(test_data2, output_file)
+            assert result2 is True
+
+            # Verify new content
+            with codecs.open(output_file, "r", encoding="utf-8") as f:
+                loaded = json.load(f)
+                assert loaded == test_data2
+
+    def test_uses_consistent_formatting(self):
+        """Should use same JSON formatting as json_dump."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            output_file = os.path.join(tmpdir, "test.json")
+            test_data = {"z": "last", "a": "first", "m": "middle"}
+
+            github_backup.json_dump_if_changed(test_data, output_file)
+
+            with codecs.open(output_file, "r", encoding="utf-8") as f:
+                content = f.read()
+
+            # Check for consistent formatting:
+            # - sorted keys
+            # - 4-space indent
+            # - comma-colon-space separator
+            expected = json.dumps(
+                test_data,
+                ensure_ascii=False,
+                sort_keys=True,
+                indent=4,
+                separators=(",", ": "),
+            )
+            assert content == expected
+
+    def test_atomic_write_always_used(self):
+        """Should always use temp file and rename for atomic writes."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            output_file = os.path.join(tmpdir, "test.json")
+            test_data = {"key": "value"}
+
+            result = github_backup.json_dump_if_changed(test_data, output_file)
+
+            assert result is True
+            assert os.path.exists(output_file)
+
+            # Temp file should not exist after atomic write
+            temp_file = output_file + ".temp"
+            assert not os.path.exists(temp_file)
+
+            # Verify content
+            with codecs.open(output_file, "r", encoding="utf-8") as f:
+                loaded = json.load(f)
+                assert loaded == test_data
+
+    def test_handles_unicode_content(self):
+        """Should correctly handle Unicode content."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            output_file = os.path.join(tmpdir, "test.json")
+            test_data = {
+                "emoji": "🚀",
+                "chinese": "你好",
+                "arabic": "مرحبا",
+                "cyrillic": "Привет",
+            }
+
+            result = github_backup.json_dump_if_changed(test_data, output_file)
+            assert result is True
+
+            # Verify Unicode is preserved
+            with codecs.open(output_file, "r", encoding="utf-8") as f:
+                loaded = json.load(f)
+                assert loaded == test_data
+
+            # Second write should skip
+            result2 = github_backup.json_dump_if_changed(test_data, output_file)
+            assert result2 is False
+
+    def test_handles_complex_nested_data(self):
+        """Should handle complex nested data structures."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            output_file = os.path.join(tmpdir, "test.json")
+            test_data = {
+                "users": [
+                    {"id": 1, "name": "Alice", "tags": ["admin", "user"]},
+                    {"id": 2, "name": "Bob", "tags": ["user"]},
+                ],
+                "metadata": {"version": "1.0", "nested": {"deep": {"value": 42}}},
+            }
+
+            result = github_backup.json_dump_if_changed(test_data, output_file)
+            assert result is True
+
+            # Verify structure is preserved
+            with codecs.open(output_file, "r", encoding="utf-8") as f:
+                loaded = json.load(f)
+                assert loaded == test_data
+
+    def test_overwrites_on_unicode_decode_error(self):
+        """Should overwrite if existing file has invalid UTF-8."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            output_file = os.path.join(tmpdir, "test.json")
+            test_data = {"key": "value"}
+
+            # Write invalid UTF-8 bytes
+            with open(output_file, "wb") as f:
+                f.write(b"\xff\xfe invalid utf-8")
+
+            # Should catch UnicodeDecodeError and overwrite
+            result = github_backup.json_dump_if_changed(test_data, output_file)
+            assert result is True
+
+            # Verify new content was written
+            with codecs.open(output_file, "r", encoding="utf-8") as f:
+                loaded = json.load(f)
+                assert loaded == test_data
+
+    def test_key_order_independence(self):
+        """Should treat differently-ordered dicts as same if keys/values match."""
+        with tempfile.TemporaryDirectory() as tmpdir:
+            output_file = os.path.join(tmpdir, "test.json")
+
+            # Write first dict
+            data1 = {"z": 1, "a": 2, "m": 3}
+            github_backup.json_dump_if_changed(data1, output_file)
+
+            # Try to write same data but different order
+            data2 = {"a": 2, "m": 3, "z": 1}
+            result = github_backup.json_dump_if_changed(data2, output_file)
+
+            # Should skip because content is the same (keys are sorted)
+            assert result is False
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])

tests/test_pagination.py

@@ -0,0 +1,153 @@
+"""Tests for Link header pagination handling."""
+
+import json
+from unittest.mock import Mock, patch
+
+import pytest
+
+from github_backup import github_backup
+
+
+class MockHTTPResponse:
+    """Mock HTTP response for paginated API calls."""
+
+    def __init__(self, data, link_header=None):
+        self._content = json.dumps(data).encode("utf-8")
+        self._link_header = link_header
+        self._read = False
+        self.reason = "OK"
+
+    def getcode(self):
+        return 200
+
+    def read(self):
+        if self._read:
+            return b""
+        self._read = True
+        return self._content
+
+    def get_header(self, name, default=None):
+        """Mock method for headers.get()."""
+        return self.headers.get(name, default)
+
+    @property
+    def headers(self):
+        headers = {"x-ratelimit-remaining": "5000"}
+        if self._link_header:
+            headers["Link"] = self._link_header
+        return headers
+
+
+@pytest.fixture
+def mock_args():
+    """Mock args for retrieve_data_gen."""
+    args = Mock()
+    args.as_app = False
+    args.token_fine = None
+    args.token_classic = "fake_token"
+    args.username = None
+    args.password = None
+    args.osx_keychain_item_name = None
+    args.osx_keychain_item_account = None
+    args.throttle_limit = None
+    args.throttle_pause = 0
+    return args
+
+
+def test_cursor_based_pagination(mock_args):
+    """Link header with 'after' cursor parameter works correctly."""
+
+    # Simulate issues endpoint behavior: returns cursor in Link header
+    responses = [
+        # Issues endpoint returns 'after' cursor parameter (not 'page')
+        MockHTTPResponse(
+            data=[{"issue": i} for i in range(1, 101)],  # Page 1 contents
+            link_header='<https://api.github.com/repos/owner/repo/issues?per_page=100&after=ABC123&page=2>; rel="next"',
+        ),
+        MockHTTPResponse(
+            data=[{"issue": i} for i in range(101, 151)],  # Page 2 contents
+            link_header=None,  # No Link header - signals end of pagination
+        ),
+    ]
+    requests_made = []
+
+    def mock_urlopen(request, *args, **kwargs):
+        url = request.get_full_url()
+        requests_made.append(url)
+        return responses[len(requests_made) - 1]
+
+    with patch("github_backup.github_backup.urlopen", side_effect=mock_urlopen):
+        results = list(
+            github_backup.retrieve_data_gen(
+                mock_args, "https://api.github.com/repos/owner/repo/issues"
+            )
+        )
+
+    # Verify all items retrieved and cursor was used in second request
+    assert len(results) == 150
+    assert len(requests_made) == 2
+    assert "after=ABC123" in requests_made[1]
+
+
+def test_page_based_pagination(mock_args):
+    """Link header with 'page' parameter works correctly."""
+
+    # Simulate pulls/repos endpoint behavior: returns page numbers in Link header
+    responses = [
+        # Pulls endpoint uses traditional 'page' parameter (not cursor)
+        MockHTTPResponse(
+            data=[{"pull": i} for i in range(1, 101)],  # Page 1 contents
+            link_header='<https://api.github.com/repos/owner/repo/pulls?per_page=100&page=2>; rel="next"',
+        ),
+        MockHTTPResponse(
+            data=[{"pull": i} for i in range(101, 181)],  # Page 2 contents
+            link_header=None,  # No Link header - signals end of pagination
+        ),
+    ]
+    requests_made = []
+
+    def mock_urlopen(request, *args, **kwargs):
+        url = request.get_full_url()
+        requests_made.append(url)
+        return responses[len(requests_made) - 1]
+
+    with patch("github_backup.github_backup.urlopen", side_effect=mock_urlopen):
+        results = list(
+            github_backup.retrieve_data_gen(
+                mock_args, "https://api.github.com/repos/owner/repo/pulls"
+            )
+        )
+
+    # Verify all items retrieved and page parameter was used (not cursor)
+    assert len(results) == 180
+    assert len(requests_made) == 2
+    assert "page=2" in requests_made[1]
+    assert "after" not in requests_made[1]
+
+
+def test_no_link_header_stops_pagination(mock_args):
+    """Pagination stops when Link header is absent."""
+
+    # Simulate endpoint with results that fit in a single page
+    responses = [
+        MockHTTPResponse(
+            data=[{"label": i} for i in range(1, 51)],  # Page contents
+            link_header=None,  # No Link header - signals end of pagination
+        )
+    ]
+    requests_made = []
+
+    def mock_urlopen(request, *args, **kwargs):
+        requests_made.append(request.get_full_url())
+        return responses[len(requests_made) - 1]
+
+    with patch("github_backup.github_backup.urlopen", side_effect=mock_urlopen):
+        results = list(
+            github_backup.retrieve_data_gen(
+                mock_args, "https://api.github.com/repos/owner/repo/labels"
+            )
+        )
+
+    # Verify pagination stopped after first request
+    assert len(results) == 50
+    assert len(requests_made) == 1