mirror of
https://github.com/josegonzalez/python-github-backup.git
synced 2026-01-14 18:12:40 +01:00
Compare commits
19 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6780d3ad6c | ||
|
|
65bacc27f0 | ||
|
|
ab0eebb175 | ||
|
|
fce4abb74a | ||
|
|
c63fb37d30 | ||
|
|
94b08d06c9 | ||
|
|
54a9872e47 | ||
|
|
b3d35f9d9f | ||
|
|
a175ac3ed9 | ||
|
|
9a6f0b4c21 | ||
|
|
858731ebbd | ||
|
|
2e999d0d3c | ||
|
|
44b0003ec9 | ||
|
|
5ab3852476 | ||
|
|
8b21e2501c | ||
|
|
f9827da342 | ||
|
|
1f2ec016d5 | ||
|
|
8b1b632d89 | ||
|
|
89502c326d |
53
CHANGES.rst
53
CHANGES.rst
@@ -1,9 +1,60 @@
|
|||||||
Changelog
|
Changelog
|
||||||
=========
|
=========
|
||||||
|
|
||||||
0.59.0 (2025-12-21)
|
0.61.1 (2026-01-13)
|
||||||
-------------------
|
-------------------
|
||||||
------------------------
|
------------------------
|
||||||
|
- Refactor test fixtures to use shared create_args helper. [Rodos]
|
||||||
|
|
||||||
|
Uses the real parse_args() function to get CLI defaults, so when
|
||||||
|
new arguments are added they're automatically available to all tests.
|
||||||
|
|
||||||
|
Changes:
|
||||||
|
- Add tests/conftest.py with create_args fixture
|
||||||
|
- Update 8 test files to use shared fixture
|
||||||
|
- Remove duplicate _create_mock_args methods
|
||||||
|
- Remove redundant @pytest.fixture mock_args definitions
|
||||||
|
|
||||||
|
This eliminates the need to update multiple test files when
|
||||||
|
adding new CLI arguments.
|
||||||
|
- Fix fine-grained PAT attachment downloads for private repos (#477)
|
||||||
|
[Rodos]
|
||||||
|
|
||||||
|
Fine-grained personal access tokens cannot download attachments from
|
||||||
|
private repositories directly due to a GitHub platform limitation.
|
||||||
|
|
||||||
|
This adds a workaround for image attachments (/assets/ URLs) using
|
||||||
|
GitHub's Markdown API to convert URLs to JWT-signed URLs that can be
|
||||||
|
downloaded without authentication.
|
||||||
|
|
||||||
|
Changes:
|
||||||
|
- Add get_jwt_signed_url_via_markdown_api() function
|
||||||
|
- Detect fine-grained token + private repo + /assets/ URL upfront
|
||||||
|
- Use JWT workaround for those cases, mark success with jwt_workaround flag
|
||||||
|
- Skip download with skipped_at when workaround fails
|
||||||
|
- Add startup warning when using --attachments with fine-grained tokens
|
||||||
|
- Document limitation in README (file attachments still fail)
|
||||||
|
- Add 6 unit tests for JWT workaround logic
|
||||||
|
|
||||||
|
|
||||||
|
0.61.0 (2026-01-12)
|
||||||
|
-------------------
|
||||||
|
- Docs: Add missing `--retries` argument to README. [Lukas Bestle]
|
||||||
|
- Test: Adapt tests to new argument. [Lukas Bestle]
|
||||||
|
- Feat: Backup of repository security advisories. [Lukas Bestle]
|
||||||
|
|
||||||
|
|
||||||
|
0.60.0 (2025-12-24)
|
||||||
|
-------------------
|
||||||
|
- Rm max_retries.py. [michaelmartinez]
|
||||||
|
- Readme. [michaelmartinez]
|
||||||
|
- Don't use a global variable, pass the args instead. [michaelmartinez]
|
||||||
|
- Readme, simplify the logic a bit. [michaelmartinez]
|
||||||
|
- Max_retries 5. [michaelmartinez]
|
||||||
|
|
||||||
|
|
||||||
|
0.59.0 (2025-12-21)
|
||||||
|
-------------------
|
||||||
- Add --starred-skip-size-over flag to limit starred repo size (#108)
|
- Add --starred-skip-size-over flag to limit starred repo size (#108)
|
||||||
[Rodos]
|
[Rodos]
|
||||||
|
|
||||||
|
|||||||
17
README.rst
17
README.rst
@@ -43,9 +43,9 @@ CLI Help output::
|
|||||||
[--watched] [--followers] [--following] [--all]
|
[--watched] [--followers] [--following] [--all]
|
||||||
[--issues] [--issue-comments] [--issue-events] [--pulls]
|
[--issues] [--issue-comments] [--issue-events] [--pulls]
|
||||||
[--pull-comments] [--pull-commits] [--pull-details]
|
[--pull-comments] [--pull-commits] [--pull-details]
|
||||||
[--labels] [--hooks] [--milestones] [--repositories]
|
[--labels] [--hooks] [--milestones] [--security-advisories]
|
||||||
[--bare] [--no-prune] [--lfs] [--wikis] [--gists]
|
[--repositories] [--bare] [--no-prune] [--lfs] [--wikis]
|
||||||
[--starred-gists] [--skip-archived] [--skip-existing]
|
[--gists] [--starred-gists] [--skip-archived] [--skip-existing]
|
||||||
[-L [LANGUAGES ...]] [-N NAME_REGEX] [-H GITHUB_HOST]
|
[-L [LANGUAGES ...]] [-N NAME_REGEX] [-H GITHUB_HOST]
|
||||||
[-O] [-R REPOSITORY] [-P] [-F] [--prefer-ssh] [-v]
|
[-O] [-R REPOSITORY] [-P] [-F] [--prefer-ssh] [-v]
|
||||||
[--keychain-name OSX_KEYCHAIN_ITEM_NAME]
|
[--keychain-name OSX_KEYCHAIN_ITEM_NAME]
|
||||||
@@ -55,7 +55,7 @@ CLI Help output::
|
|||||||
[--skip-assets-on [SKIP_ASSETS_ON ...]] [--attachments]
|
[--skip-assets-on [SKIP_ASSETS_ON ...]] [--attachments]
|
||||||
[--throttle-limit THROTTLE_LIMIT]
|
[--throttle-limit THROTTLE_LIMIT]
|
||||||
[--throttle-pause THROTTLE_PAUSE]
|
[--throttle-pause THROTTLE_PAUSE]
|
||||||
[--exclude [EXCLUDE ...]]
|
[--exclude [EXCLUDE ...]] [--retries MAX_RETRIES]
|
||||||
USER
|
USER
|
||||||
|
|
||||||
Backup a github account
|
Backup a github account
|
||||||
@@ -101,6 +101,8 @@ CLI Help output::
|
|||||||
--hooks include hooks in backup (works only when
|
--hooks include hooks in backup (works only when
|
||||||
authenticated)
|
authenticated)
|
||||||
--milestones include milestones in backup
|
--milestones include milestones in backup
|
||||||
|
--security-advisories
|
||||||
|
include security advisories in backup
|
||||||
--repositories include repository clone in backup
|
--repositories include repository clone in backup
|
||||||
--bare clone bare repositories
|
--bare clone bare repositories
|
||||||
--no-prune disable prune option for git fetch
|
--no-prune disable prune option for git fetch
|
||||||
@@ -152,7 +154,8 @@ CLI Help output::
|
|||||||
--throttle-limit to be set)
|
--throttle-limit to be set)
|
||||||
--exclude [EXCLUDE ...]
|
--exclude [EXCLUDE ...]
|
||||||
names of repositories to exclude
|
names of repositories to exclude
|
||||||
|
--retries MAX_RETRIES
|
||||||
|
maximum number of retries for API calls (default: 5)
|
||||||
|
|
||||||
Usage Details
|
Usage Details
|
||||||
=============
|
=============
|
||||||
@@ -278,6 +281,8 @@ The tool automatically extracts file extensions from HTTP headers to ensure file
|
|||||||
|
|
||||||
**Repository filtering** for repo files/assets handles renamed and transferred repositories gracefully. URLs are included if they either match the current repository name directly, or redirect to it (e.g., ``willmcgugan/rich`` redirects to ``Textualize/rich`` after transfer).
|
**Repository filtering** for repo files/assets handles renamed and transferred repositories gracefully. URLs are included if they either match the current repository name directly, or redirect to it (e.g., ``willmcgugan/rich`` redirects to ``Textualize/rich`` after transfer).
|
||||||
|
|
||||||
|
**Fine-grained token limitation:** Due to a GitHub platform limitation, fine-grained personal access tokens (``github_pat_...``) cannot download attachments from private repositories directly. This affects both ``/assets/`` (images) and ``/files/`` (documents) URLs. The tool implements a workaround for image attachments using GitHub's Markdown API, which converts URLs to temporary JWT-signed URLs that can be downloaded. However, this workaround only works for images - document attachments (PDFs, text files, etc.) will fail with 404 errors when using fine-grained tokens on private repos. For full attachment support on private repositories, use a classic token (``-t``) instead of a fine-grained token (``-f``). See `#477 <https://github.com/josegonzalez/python-github-backup/issues/477>`_ for details.
|
||||||
|
|
||||||
|
|
||||||
Run in Docker container
|
Run in Docker container
|
||||||
-----------------------
|
-----------------------
|
||||||
@@ -400,7 +405,7 @@ Quietly and incrementally backup useful Github user data (public and private rep
|
|||||||
export FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
|
export FINE_ACCESS_TOKEN=SOME-GITHUB-TOKEN
|
||||||
GH_USER=YOUR-GITHUB-USER
|
GH_USER=YOUR-GITHUB-USER
|
||||||
|
|
||||||
github-backup -f $FINE_ACCESS_TOKEN --prefer-ssh -o ~/github-backup/ -l error -P -i --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-commits --labels --milestones --repositories --wikis --releases --assets --attachments --pull-details --gists --starred-gists $GH_USER
|
github-backup -f $FINE_ACCESS_TOKEN --prefer-ssh -o ~/github-backup/ -l error -P -i --all-starred --starred --watched --followers --following --issues --issue-comments --issue-events --pulls --pull-comments --pull-commits --labels --milestones --security-advisories --repositories --wikis --releases --assets --attachments --pull-details --gists --starred-gists $GH_USER
|
||||||
|
|
||||||
Debug an error/block or incomplete backup into a temporary directory. Omit "incremental" to fill a previous incomplete backup. ::
|
Debug an error/block or incomplete backup into a temporary directory. Omit "incremental" to fill a previous incomplete backup. ::
|
||||||
|
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
__version__ = "0.59.0"
|
__version__ = "0.61.1"
|
||||||
|
|||||||
@@ -46,6 +46,16 @@ def main():
|
|||||||
"Use -t/--token or -f/--token-fine to authenticate."
|
"Use -t/--token or -f/--token-fine to authenticate."
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Issue #477: Fine-grained PATs cannot download all attachment types from
|
||||||
|
# private repos. Image attachments will be retried via Markdown API workaround.
|
||||||
|
if args.include_attachments and args.token_fine:
|
||||||
|
logger.warning(
|
||||||
|
"Using --attachments with fine-grained token. Due to GitHub platform "
|
||||||
|
"limitations, file attachments (PDFs, etc.) from private repos may fail. "
|
||||||
|
"Image attachments will be retried via workaround. For full attachment "
|
||||||
|
"support, use --token-classic instead."
|
||||||
|
)
|
||||||
|
|
||||||
if args.quiet:
|
if args.quiet:
|
||||||
logger.setLevel(logging.WARNING)
|
logger.setLevel(logging.WARNING)
|
||||||
|
|
||||||
|
|||||||
@@ -74,9 +74,6 @@ else:
|
|||||||
" 3. Debian/Ubuntu: apt-get install ca-certificates\n\n"
|
" 3. Debian/Ubuntu: apt-get install ca-certificates\n\n"
|
||||||
)
|
)
|
||||||
|
|
||||||
# Retry configuration
|
|
||||||
MAX_RETRIES = 5
|
|
||||||
|
|
||||||
|
|
||||||
def logging_subprocess(
|
def logging_subprocess(
|
||||||
popenargs, stdout_log_level=logging.DEBUG, stderr_log_level=logging.ERROR, **kwargs
|
popenargs, stdout_log_level=logging.DEBUG, stderr_log_level=logging.ERROR, **kwargs
|
||||||
@@ -144,6 +141,17 @@ def mask_password(url, secret="*****"):
|
|||||||
return url.replace(parsed.password, secret)
|
return url.replace(parsed.password, secret)
|
||||||
|
|
||||||
|
|
||||||
|
def non_negative_int(value):
|
||||||
|
"""Argparse type validator for non-negative integers."""
|
||||||
|
try:
|
||||||
|
ivalue = int(value)
|
||||||
|
except ValueError:
|
||||||
|
raise argparse.ArgumentTypeError(f"'{value}' is not a valid integer")
|
||||||
|
if ivalue < 0:
|
||||||
|
raise argparse.ArgumentTypeError(f"{value} must be 0 or greater")
|
||||||
|
return ivalue
|
||||||
|
|
||||||
|
|
||||||
def parse_args(args=None):
|
def parse_args(args=None):
|
||||||
parser = argparse.ArgumentParser(description="Backup a github account")
|
parser = argparse.ArgumentParser(description="Backup a github account")
|
||||||
parser.add_argument("user", metavar="USER", type=str, help="github username")
|
parser.add_argument("user", metavar="USER", type=str, help="github username")
|
||||||
@@ -302,6 +310,12 @@ def parse_args(args=None):
|
|||||||
dest="include_milestones",
|
dest="include_milestones",
|
||||||
help="include milestones in backup",
|
help="include milestones in backup",
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--security-advisories",
|
||||||
|
action="store_true",
|
||||||
|
dest="include_security_advisories",
|
||||||
|
help="include security advisories in backup",
|
||||||
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--repositories",
|
"--repositories",
|
||||||
action="store_true",
|
action="store_true",
|
||||||
@@ -468,6 +482,13 @@ def parse_args(args=None):
|
|||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--exclude", dest="exclude", help="names of repositories to exclude", nargs="*"
|
"--exclude", dest="exclude", help="names of repositories to exclude", nargs="*"
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--retries",
|
||||||
|
dest="max_retries",
|
||||||
|
type=non_negative_int,
|
||||||
|
default=5,
|
||||||
|
help="maximum number of retries for API calls (default: 5)",
|
||||||
|
)
|
||||||
return parser.parse_args(args)
|
return parser.parse_args(args)
|
||||||
|
|
||||||
|
|
||||||
@@ -622,7 +643,7 @@ def retrieve_data(args, template, query_args=None, paginated=True):
|
|||||||
def _extract_next_page_url(link_header):
|
def _extract_next_page_url(link_header):
|
||||||
for link in link_header.split(","):
|
for link in link_header.split(","):
|
||||||
if 'rel="next"' in link:
|
if 'rel="next"' in link:
|
||||||
return link[link.find("<") + 1:link.find(">")]
|
return link[link.find("<") + 1 : link.find(">")]
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def fetch_all() -> Generator[dict, None, None]:
|
def fetch_all() -> Generator[dict, None, None]:
|
||||||
@@ -631,7 +652,7 @@ def retrieve_data(args, template, query_args=None, paginated=True):
|
|||||||
while True:
|
while True:
|
||||||
# FIRST: Fetch response
|
# FIRST: Fetch response
|
||||||
|
|
||||||
for attempt in range(MAX_RETRIES):
|
for attempt in range(args.max_retries + 1):
|
||||||
request = _construct_request(
|
request = _construct_request(
|
||||||
per_page=per_page if paginated else None,
|
per_page=per_page if paginated else None,
|
||||||
query_args=query_args,
|
query_args=query_args,
|
||||||
@@ -640,7 +661,7 @@ def retrieve_data(args, template, query_args=None, paginated=True):
|
|||||||
as_app=args.as_app,
|
as_app=args.as_app,
|
||||||
fine=args.token_fine is not None,
|
fine=args.token_fine is not None,
|
||||||
)
|
)
|
||||||
http_response = make_request_with_retry(request, auth)
|
http_response = make_request_with_retry(request, auth, args.max_retries)
|
||||||
|
|
||||||
match http_response.getcode():
|
match http_response.getcode():
|
||||||
case 200:
|
case 200:
|
||||||
@@ -654,10 +675,10 @@ def retrieve_data(args, template, query_args=None, paginated=True):
|
|||||||
TimeoutError,
|
TimeoutError,
|
||||||
) as e:
|
) as e:
|
||||||
logger.warning(f"{type(e).__name__} reading response")
|
logger.warning(f"{type(e).__name__} reading response")
|
||||||
if attempt < MAX_RETRIES - 1:
|
if attempt < args.max_retries:
|
||||||
delay = calculate_retry_delay(attempt, {})
|
delay = calculate_retry_delay(attempt, {})
|
||||||
logger.warning(
|
logger.warning(
|
||||||
f"Retrying in {delay:.1f}s (attempt {attempt + 1}/{MAX_RETRIES})"
|
f"Retrying read in {delay:.1f}s (attempt {attempt + 1}/{args.max_retries + 1})"
|
||||||
)
|
)
|
||||||
time.sleep(delay)
|
time.sleep(delay)
|
||||||
continue # Next retry attempt
|
continue # Next retry attempt
|
||||||
@@ -683,10 +704,10 @@ def retrieve_data(args, template, query_args=None, paginated=True):
|
|||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
logger.error(
|
logger.error(
|
||||||
f"Failed to read response after {MAX_RETRIES} attempts for {next_url or template}"
|
f"Failed to read response after {args.max_retries + 1} attempts for {next_url or template}"
|
||||||
)
|
)
|
||||||
raise Exception(
|
raise Exception(
|
||||||
f"Failed to read response after {MAX_RETRIES} attempts for {next_url or template}"
|
f"Failed to read response after {args.max_retries + 1} attempts for {next_url or template}"
|
||||||
)
|
)
|
||||||
|
|
||||||
# SECOND: Process and paginate
|
# SECOND: Process and paginate
|
||||||
@@ -718,7 +739,7 @@ def retrieve_data(args, template, query_args=None, paginated=True):
|
|||||||
return list(fetch_all())
|
return list(fetch_all())
|
||||||
|
|
||||||
|
|
||||||
def make_request_with_retry(request, auth):
|
def make_request_with_retry(request, auth, max_retries=5):
|
||||||
"""Make HTTP request with automatic retry for transient errors."""
|
"""Make HTTP request with automatic retry for transient errors."""
|
||||||
|
|
||||||
def is_retryable_status(status_code, headers):
|
def is_retryable_status(status_code, headers):
|
||||||
@@ -730,40 +751,49 @@ def make_request_with_retry(request, auth):
|
|||||||
return int(headers.get("x-ratelimit-remaining", 1)) < 1
|
return int(headers.get("x-ratelimit-remaining", 1)) < 1
|
||||||
return False
|
return False
|
||||||
|
|
||||||
for attempt in range(MAX_RETRIES):
|
for attempt in range(max_retries + 1):
|
||||||
try:
|
try:
|
||||||
return urlopen(request, context=https_ctx)
|
return urlopen(request, context=https_ctx)
|
||||||
|
|
||||||
except HTTPError as exc:
|
except HTTPError as exc:
|
||||||
# HTTPError can be used as a response-like object
|
# HTTPError can be used as a response-like object
|
||||||
if not is_retryable_status(exc.code, exc.headers):
|
if not is_retryable_status(exc.code, exc.headers):
|
||||||
|
logger.error(
|
||||||
|
f"API Error: {exc.code} {exc.reason} for {request.full_url}"
|
||||||
|
)
|
||||||
raise # Non-retryable error
|
raise # Non-retryable error
|
||||||
|
|
||||||
if attempt >= MAX_RETRIES - 1:
|
if attempt >= max_retries:
|
||||||
logger.error(f"HTTP {exc.code} failed after {MAX_RETRIES} attempts")
|
logger.error(
|
||||||
|
f"HTTP {exc.code} failed after {max_retries + 1} attempts for {request.full_url}"
|
||||||
|
)
|
||||||
raise
|
raise
|
||||||
|
|
||||||
delay = calculate_retry_delay(attempt, exc.headers)
|
delay = calculate_retry_delay(attempt, exc.headers)
|
||||||
logger.warning(
|
logger.warning(
|
||||||
f"HTTP {exc.code}, retrying in {delay:.1f}s "
|
f"HTTP {exc.code} ({exc.reason}), retrying in {delay:.1f}s "
|
||||||
f"(attempt {attempt + 1}/{MAX_RETRIES})"
|
f"(attempt {attempt + 1}/{max_retries + 1}) for {request.full_url}"
|
||||||
)
|
)
|
||||||
if auth is None and exc.code in (403, 429):
|
if auth is None and exc.code in (403, 429):
|
||||||
logger.info("Hint: Authenticate to raise your GitHub rate limit")
|
logger.info("Hint: Authenticate to raise your GitHub rate limit")
|
||||||
time.sleep(delay)
|
time.sleep(delay)
|
||||||
|
|
||||||
except (URLError, socket.error) as e:
|
except (URLError, socket.error) as e:
|
||||||
if attempt >= MAX_RETRIES - 1:
|
if attempt >= max_retries:
|
||||||
logger.error(f"Connection error failed after {MAX_RETRIES} attempts: {e}")
|
logger.error(
|
||||||
|
f"Connection error failed after {max_retries + 1} attempts: {e} for {request.full_url}"
|
||||||
|
)
|
||||||
raise
|
raise
|
||||||
delay = calculate_retry_delay(attempt, {})
|
delay = calculate_retry_delay(attempt, {})
|
||||||
logger.warning(
|
logger.warning(
|
||||||
f"Connection error: {e}, retrying in {delay:.1f}s "
|
f"Connection error: {e}, retrying in {delay:.1f}s "
|
||||||
f"(attempt {attempt + 1}/{MAX_RETRIES})"
|
f"(attempt {attempt + 1}/{max_retries + 1}) for {request.full_url}"
|
||||||
)
|
)
|
||||||
time.sleep(delay)
|
time.sleep(delay)
|
||||||
|
|
||||||
raise Exception(f"Request failed after {MAX_RETRIES} attempts") # pragma: no cover
|
raise Exception(
|
||||||
|
f"Request failed after {max_retries + 1} attempts"
|
||||||
|
) # pragma: no cover
|
||||||
|
|
||||||
|
|
||||||
def _construct_request(per_page, query_args, template, auth, as_app=None, fine=False):
|
def _construct_request(per_page, query_args, template, auth, as_app=None, fine=False):
|
||||||
@@ -1032,6 +1062,65 @@ def download_attachment_file(url, path, auth, as_app=False, fine=False):
|
|||||||
return metadata
|
return metadata
|
||||||
|
|
||||||
|
|
||||||
|
def get_jwt_signed_url_via_markdown_api(url, token, repo_context):
|
||||||
|
"""Convert a user-attachments/assets URL to a JWT-signed URL via Markdown API.
|
||||||
|
|
||||||
|
GitHub's Markdown API renders image URLs and returns HTML containing
|
||||||
|
JWT-signed private-user-images.githubusercontent.com URLs that work
|
||||||
|
without token authentication.
|
||||||
|
|
||||||
|
This is a workaround for issue #477 where fine-grained PATs cannot
|
||||||
|
download user-attachments URLs from private repos directly.
|
||||||
|
|
||||||
|
Limitations:
|
||||||
|
- Only works for /assets/ URLs (images)
|
||||||
|
- Does NOT work for /files/ URLs (PDFs, text files, etc.)
|
||||||
|
- JWT URLs expire after ~5 minutes
|
||||||
|
|
||||||
|
Args:
|
||||||
|
url: The github.com/user-attachments/assets/UUID URL
|
||||||
|
token: Raw fine-grained PAT (github_pat_...)
|
||||||
|
repo_context: Repository context as "owner/repo"
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
str: JWT-signed URL from private-user-images.githubusercontent.com
|
||||||
|
None: If conversion fails
|
||||||
|
"""
|
||||||
|
|
||||||
|
try:
|
||||||
|
payload = json.dumps(
|
||||||
|
{"text": f"", "mode": "gfm", "context": repo_context}
|
||||||
|
).encode("utf-8")
|
||||||
|
|
||||||
|
request = Request("https://api.github.com/markdown", data=payload, method="POST")
|
||||||
|
request.add_header("Authorization", f"token {token}")
|
||||||
|
request.add_header("Content-Type", "application/json")
|
||||||
|
request.add_header("Accept", "application/vnd.github+json")
|
||||||
|
|
||||||
|
html = urlopen(request, timeout=30).read().decode("utf-8")
|
||||||
|
|
||||||
|
# Parse JWT-signed URL from HTML response
|
||||||
|
# Format: <img src="https://private-user-images.githubusercontent.com/...?jwt=..." ...>
|
||||||
|
if match := re.search(
|
||||||
|
r'src="(https://private-user-images\.githubusercontent\.com/[^"]+)"', html
|
||||||
|
):
|
||||||
|
jwt_url = match.group(1)
|
||||||
|
logger.debug("Converted attachment URL to JWT-signed URL via Markdown API")
|
||||||
|
return jwt_url
|
||||||
|
|
||||||
|
logger.debug("Markdown API response did not contain JWT-signed URL")
|
||||||
|
return None
|
||||||
|
|
||||||
|
except HTTPError as e:
|
||||||
|
logger.debug(
|
||||||
|
"Markdown API request failed with HTTP {0}: {1}".format(e.code, e.reason)
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
except Exception as e:
|
||||||
|
logger.debug("Markdown API request failed: {0}".format(str(e)))
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def extract_attachment_urls(item_data, issue_number=None, repository_full_name=None):
|
def extract_attachment_urls(item_data, issue_number=None, repository_full_name=None):
|
||||||
"""Extract GitHub-hosted attachment URLs from issue/PR body and comments.
|
"""Extract GitHub-hosted attachment URLs from issue/PR body and comments.
|
||||||
|
|
||||||
@@ -1385,15 +1474,46 @@ def download_attachments(
|
|||||||
filename = get_attachment_filename(url)
|
filename = get_attachment_filename(url)
|
||||||
filepath = os.path.join(attachments_dir, filename)
|
filepath = os.path.join(attachments_dir, filename)
|
||||||
|
|
||||||
# Download and get metadata
|
# Issue #477: Fine-grained PATs cannot download user-attachments/assets
|
||||||
metadata = download_attachment_file(
|
# from private repos directly (404). Use Markdown API workaround to get
|
||||||
url,
|
# a JWT-signed URL. Only works for /assets/ (images), not /files/.
|
||||||
filepath,
|
needs_jwt = (
|
||||||
get_auth(args, encode=not args.as_app),
|
args.token_fine is not None
|
||||||
as_app=args.as_app,
|
and repository.get("private", False)
|
||||||
fine=args.token_fine is not None,
|
and "github.com/user-attachments/assets/" in url
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if not needs_jwt:
|
||||||
|
# NORMAL download path
|
||||||
|
metadata = download_attachment_file(
|
||||||
|
url,
|
||||||
|
filepath,
|
||||||
|
get_auth(args, encode=not args.as_app),
|
||||||
|
as_app=args.as_app,
|
||||||
|
fine=args.token_fine is not None,
|
||||||
|
)
|
||||||
|
elif jwt_url := get_jwt_signed_url_via_markdown_api(
|
||||||
|
url, args.token_fine, repository["full_name"]
|
||||||
|
):
|
||||||
|
# JWT needed and extracted, download via JWT
|
||||||
|
metadata = download_attachment_file(
|
||||||
|
jwt_url, filepath, auth=None, as_app=False, fine=False
|
||||||
|
)
|
||||||
|
metadata["url"] = url # Apply back the original URL
|
||||||
|
metadata["jwt_workaround"] = True
|
||||||
|
else:
|
||||||
|
# Markdown API workaround failed - skip download we know will fail
|
||||||
|
metadata = {
|
||||||
|
"url": url,
|
||||||
|
"success": False,
|
||||||
|
"skipped_at": datetime.now(timezone.utc).isoformat(),
|
||||||
|
"error": "Fine-grained token cannot download private repo attachments. "
|
||||||
|
"Markdown API workaround failed. Use --token-classic instead.",
|
||||||
|
}
|
||||||
|
logger.warning(
|
||||||
|
"Skipping attachment {0}: {1}".format(url, metadata["error"])
|
||||||
|
)
|
||||||
|
|
||||||
# If download succeeded but we got an extension from Content-Disposition,
|
# If download succeeded but we got an extension from Content-Disposition,
|
||||||
# we may need to rename the file to add the extension
|
# we may need to rename the file to add the extension
|
||||||
if metadata["success"] and metadata.get("original_filename"):
|
if metadata["success"] and metadata.get("original_filename"):
|
||||||
@@ -1579,9 +1699,7 @@ def filter_repositories(args, unfiltered_repositories):
|
|||||||
repositories = [r for r in repositories if not r.get("archived")]
|
repositories = [r for r in repositories if not r.get("archived")]
|
||||||
if args.starred_skip_size_over is not None:
|
if args.starred_skip_size_over is not None:
|
||||||
if args.starred_skip_size_over <= 0:
|
if args.starred_skip_size_over <= 0:
|
||||||
logger.warning(
|
logger.warning("--starred-skip-size-over must be greater than 0, ignoring")
|
||||||
"--starred-skip-size-over must be greater than 0, ignoring"
|
|
||||||
)
|
|
||||||
else:
|
else:
|
||||||
size_limit_kb = args.starred_skip_size_over * 1024
|
size_limit_kb = args.starred_skip_size_over * 1024
|
||||||
filtered = []
|
filtered = []
|
||||||
@@ -1590,7 +1708,9 @@ def filter_repositories(args, unfiltered_repositories):
|
|||||||
size_mb = r.get("size", 0) / 1024
|
size_mb = r.get("size", 0) / 1024
|
||||||
logger.info(
|
logger.info(
|
||||||
"Skipping starred repo {0} ({1:.0f} MB) due to --starred-skip-size-over {2}".format(
|
"Skipping starred repo {0} ({1:.0f} MB) due to --starred-skip-size-over {2}".format(
|
||||||
r.get("full_name", r.get("name")), size_mb, args.starred_skip_size_over
|
r.get("full_name", r.get("name")),
|
||||||
|
size_mb,
|
||||||
|
args.starred_skip_size_over,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
@@ -1694,6 +1814,9 @@ def backup_repositories(args, output_directory, repositories):
|
|||||||
if args.include_milestones or args.include_everything:
|
if args.include_milestones or args.include_everything:
|
||||||
backup_milestones(args, repo_cwd, repository, repos_template)
|
backup_milestones(args, repo_cwd, repository, repos_template)
|
||||||
|
|
||||||
|
if args.include_security_advisories or args.include_everything:
|
||||||
|
backup_security_advisories(args, repo_cwd, repository, repos_template)
|
||||||
|
|
||||||
if args.include_labels or args.include_everything:
|
if args.include_labels or args.include_everything:
|
||||||
backup_labels(args, repo_cwd, repository, repos_template)
|
backup_labels(args, repo_cwd, repository, repos_template)
|
||||||
|
|
||||||
@@ -1910,6 +2033,43 @@ def backup_milestones(args, repo_cwd, repository, repos_template):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def backup_security_advisories(args, repo_cwd, repository, repos_template):
|
||||||
|
advisory_cwd = os.path.join(repo_cwd, "security-advisories")
|
||||||
|
if args.skip_existing and os.path.isdir(advisory_cwd):
|
||||||
|
return
|
||||||
|
|
||||||
|
logger.info("Retrieving {0} security advisories".format(repository["full_name"]))
|
||||||
|
mkdir_p(repo_cwd, advisory_cwd)
|
||||||
|
|
||||||
|
template = "{0}/{1}/security-advisories".format(
|
||||||
|
repos_template, repository["full_name"]
|
||||||
|
)
|
||||||
|
|
||||||
|
_advisories = retrieve_data(args, template)
|
||||||
|
|
||||||
|
advisories = {}
|
||||||
|
for advisory in _advisories:
|
||||||
|
advisories[advisory["ghsa_id"]] = advisory
|
||||||
|
|
||||||
|
written_count = 0
|
||||||
|
for ghsa_id, advisory in list(advisories.items()):
|
||||||
|
advisory_file = "{0}/{1}.json".format(advisory_cwd, ghsa_id)
|
||||||
|
if json_dump_if_changed(advisory, advisory_file):
|
||||||
|
written_count += 1
|
||||||
|
|
||||||
|
total = len(advisories)
|
||||||
|
if written_count == total:
|
||||||
|
logger.info("Saved {0} security advisories to disk".format(total))
|
||||||
|
elif written_count == 0:
|
||||||
|
logger.info("{0} security advisories unchanged, skipped write".format(total))
|
||||||
|
else:
|
||||||
|
logger.info(
|
||||||
|
"Saved {0} of {1} security advisories to disk ({2} unchanged)".format(
|
||||||
|
written_count, total, total - written_count
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def backup_labels(args, repo_cwd, repository, repos_template):
|
def backup_labels(args, repo_cwd, repository, repos_template):
|
||||||
label_cwd = os.path.join(repo_cwd, "labels")
|
label_cwd = os.path.join(repo_cwd, "labels")
|
||||||
output_file = "{0}/labels.json".format(label_cwd)
|
output_file = "{0}/labels.json".format(label_cwd)
|
||||||
|
|||||||
25
tests/conftest.py
Normal file
25
tests/conftest.py
Normal file
@@ -0,0 +1,25 @@
|
|||||||
|
"""Shared pytest fixtures for github-backup tests."""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from github_backup.github_backup import parse_args
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def create_args():
|
||||||
|
"""Factory fixture that creates args with real CLI defaults.
|
||||||
|
|
||||||
|
Uses the actual argument parser so new CLI args are automatically
|
||||||
|
available with their defaults - no test updates needed.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
def test_something(self, create_args):
|
||||||
|
args = create_args(include_releases=True, user="myuser")
|
||||||
|
"""
|
||||||
|
def _create(**overrides):
|
||||||
|
# Use real parser to get actual defaults
|
||||||
|
args = parse_args(["testuser"])
|
||||||
|
for key, value in overrides.items():
|
||||||
|
setattr(args, key, value)
|
||||||
|
return args
|
||||||
|
return _create
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
"""Tests for --all-starred flag behavior (issue #225)."""
|
"""Tests for --all-starred flag behavior (issue #225)."""
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from unittest.mock import Mock, patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
from github_backup import github_backup
|
from github_backup import github_backup
|
||||||
|
|
||||||
@@ -12,57 +12,14 @@ class TestAllStarredCloning:
|
|||||||
Issue #225: --all-starred should clone starred repos without requiring --repositories.
|
Issue #225: --all-starred should clone starred repos without requiring --repositories.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def _create_mock_args(self, **overrides):
|
|
||||||
"""Create a mock args object with sensible defaults."""
|
|
||||||
args = Mock()
|
|
||||||
args.user = "testuser"
|
|
||||||
args.output_directory = "/tmp/backup"
|
|
||||||
args.include_repository = False
|
|
||||||
args.include_everything = False
|
|
||||||
args.include_gists = False
|
|
||||||
args.include_starred_gists = False
|
|
||||||
args.all_starred = False
|
|
||||||
args.skip_existing = False
|
|
||||||
args.bare_clone = False
|
|
||||||
args.lfs_clone = False
|
|
||||||
args.no_prune = False
|
|
||||||
args.include_wiki = False
|
|
||||||
args.include_issues = False
|
|
||||||
args.include_issue_comments = False
|
|
||||||
args.include_issue_events = False
|
|
||||||
args.include_pulls = False
|
|
||||||
args.include_pull_comments = False
|
|
||||||
args.include_pull_commits = False
|
|
||||||
args.include_pull_details = False
|
|
||||||
args.include_labels = False
|
|
||||||
args.include_hooks = False
|
|
||||||
args.include_milestones = False
|
|
||||||
args.include_releases = False
|
|
||||||
args.include_assets = False
|
|
||||||
args.include_attachments = False
|
|
||||||
args.incremental = False
|
|
||||||
args.incremental_by_files = False
|
|
||||||
args.github_host = None
|
|
||||||
args.prefer_ssh = False
|
|
||||||
args.token_classic = None
|
|
||||||
args.token_fine = None
|
|
||||||
args.as_app = False
|
|
||||||
args.osx_keychain_item_name = None
|
|
||||||
args.osx_keychain_item_account = None
|
|
||||||
|
|
||||||
for key, value in overrides.items():
|
|
||||||
setattr(args, key, value)
|
|
||||||
|
|
||||||
return args
|
|
||||||
|
|
||||||
@patch('github_backup.github_backup.fetch_repository')
|
@patch('github_backup.github_backup.fetch_repository')
|
||||||
@patch('github_backup.github_backup.get_github_repo_url')
|
@patch('github_backup.github_backup.get_github_repo_url')
|
||||||
def test_all_starred_clones_without_repositories_flag(self, mock_get_url, mock_fetch):
|
def test_all_starred_clones_without_repositories_flag(self, mock_get_url, mock_fetch, create_args):
|
||||||
"""--all-starred should clone starred repos without --repositories flag.
|
"""--all-starred should clone starred repos without --repositories flag.
|
||||||
|
|
||||||
This is the core fix for issue #225.
|
This is the core fix for issue #225.
|
||||||
"""
|
"""
|
||||||
args = self._create_mock_args(all_starred=True)
|
args = create_args(all_starred=True)
|
||||||
mock_get_url.return_value = "https://github.com/otheruser/awesome-project.git"
|
mock_get_url.return_value = "https://github.com/otheruser/awesome-project.git"
|
||||||
|
|
||||||
# A starred repository (is_starred flag set by retrieve_repositories)
|
# A starred repository (is_starred flag set by retrieve_repositories)
|
||||||
@@ -87,9 +44,9 @@ class TestAllStarredCloning:
|
|||||||
|
|
||||||
@patch('github_backup.github_backup.fetch_repository')
|
@patch('github_backup.github_backup.fetch_repository')
|
||||||
@patch('github_backup.github_backup.get_github_repo_url')
|
@patch('github_backup.github_backup.get_github_repo_url')
|
||||||
def test_starred_repo_not_cloned_without_all_starred_flag(self, mock_get_url, mock_fetch):
|
def test_starred_repo_not_cloned_without_all_starred_flag(self, mock_get_url, mock_fetch, create_args):
|
||||||
"""Starred repos should NOT be cloned if --all-starred is not set."""
|
"""Starred repos should NOT be cloned if --all-starred is not set."""
|
||||||
args = self._create_mock_args(all_starred=False)
|
args = create_args(all_starred=False)
|
||||||
mock_get_url.return_value = "https://github.com/otheruser/awesome-project.git"
|
mock_get_url.return_value = "https://github.com/otheruser/awesome-project.git"
|
||||||
|
|
||||||
starred_repo = {
|
starred_repo = {
|
||||||
@@ -110,9 +67,9 @@ class TestAllStarredCloning:
|
|||||||
|
|
||||||
@patch('github_backup.github_backup.fetch_repository')
|
@patch('github_backup.github_backup.fetch_repository')
|
||||||
@patch('github_backup.github_backup.get_github_repo_url')
|
@patch('github_backup.github_backup.get_github_repo_url')
|
||||||
def test_non_starred_repo_not_cloned_with_only_all_starred(self, mock_get_url, mock_fetch):
|
def test_non_starred_repo_not_cloned_with_only_all_starred(self, mock_get_url, mock_fetch, create_args):
|
||||||
"""Non-starred repos should NOT be cloned when only --all-starred is set."""
|
"""Non-starred repos should NOT be cloned when only --all-starred is set."""
|
||||||
args = self._create_mock_args(all_starred=True)
|
args = create_args(all_starred=True)
|
||||||
mock_get_url.return_value = "https://github.com/testuser/my-project.git"
|
mock_get_url.return_value = "https://github.com/testuser/my-project.git"
|
||||||
|
|
||||||
# A regular (non-starred) repository
|
# A regular (non-starred) repository
|
||||||
@@ -134,9 +91,9 @@ class TestAllStarredCloning:
|
|||||||
|
|
||||||
@patch('github_backup.github_backup.fetch_repository')
|
@patch('github_backup.github_backup.fetch_repository')
|
||||||
@patch('github_backup.github_backup.get_github_repo_url')
|
@patch('github_backup.github_backup.get_github_repo_url')
|
||||||
def test_repositories_flag_still_works(self, mock_get_url, mock_fetch):
|
def test_repositories_flag_still_works(self, mock_get_url, mock_fetch, create_args):
|
||||||
"""--repositories flag should still clone repos as before."""
|
"""--repositories flag should still clone repos as before."""
|
||||||
args = self._create_mock_args(include_repository=True)
|
args = create_args(include_repository=True)
|
||||||
mock_get_url.return_value = "https://github.com/testuser/my-project.git"
|
mock_get_url.return_value = "https://github.com/testuser/my-project.git"
|
||||||
|
|
||||||
regular_repo = {
|
regular_repo = {
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import json
|
|||||||
import os
|
import os
|
||||||
import tempfile
|
import tempfile
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from unittest.mock import Mock
|
from unittest.mock import Mock, patch
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
@@ -12,22 +12,13 @@ from github_backup import github_backup
|
|||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def attachment_test_setup(tmp_path):
|
def attachment_test_setup(tmp_path, create_args):
|
||||||
"""Fixture providing setup and helper for attachment download tests."""
|
"""Fixture providing setup and helper for attachment download tests."""
|
||||||
from unittest.mock import patch
|
|
||||||
|
|
||||||
issue_cwd = tmp_path / "issues"
|
issue_cwd = tmp_path / "issues"
|
||||||
issue_cwd.mkdir()
|
issue_cwd.mkdir()
|
||||||
|
|
||||||
# Mock args
|
# Create args using shared fixture
|
||||||
args = Mock()
|
args = create_args(user="testuser", repository="testrepo")
|
||||||
args.as_app = False
|
|
||||||
args.token_fine = None
|
|
||||||
args.token_classic = None
|
|
||||||
args.osx_keychain_item_name = None
|
|
||||||
args.osx_keychain_item_account = None
|
|
||||||
args.user = "testuser"
|
|
||||||
args.repository = "testrepo"
|
|
||||||
|
|
||||||
repository = {"full_name": "testuser/testrepo"}
|
repository = {"full_name": "testuser/testrepo"}
|
||||||
|
|
||||||
@@ -349,3 +340,146 @@ class TestManifestDuplicatePrevention:
|
|||||||
downloaded_urls[0]
|
downloaded_urls[0]
|
||||||
== "https://github.com/user-attachments/assets/unavailable"
|
== "https://github.com/user-attachments/assets/unavailable"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestJWTWorkaround:
|
||||||
|
"""Test JWT workaround for fine-grained tokens on private repos (issue #477)."""
|
||||||
|
|
||||||
|
def test_markdown_api_extracts_jwt_url(self):
|
||||||
|
"""Markdown API response with JWT URL is extracted correctly."""
|
||||||
|
html_response = (
|
||||||
|
'<p><a href="https://private-user-images.githubusercontent.com'
|
||||||
|
'/123/abc.png?jwt=eyJhbGciOiJ"><img src="https://private-user-'
|
||||||
|
'images.githubusercontent.com/123/abc.png?jwt=eyJhbGciOiJ" '
|
||||||
|
'alt="img"></a></p>'
|
||||||
|
)
|
||||||
|
|
||||||
|
mock_response = Mock()
|
||||||
|
mock_response.read.return_value = html_response.encode("utf-8")
|
||||||
|
|
||||||
|
with patch("github_backup.github_backup.urlopen", return_value=mock_response):
|
||||||
|
result = github_backup.get_jwt_signed_url_via_markdown_api(
|
||||||
|
"https://github.com/user-attachments/assets/abc123",
|
||||||
|
"github_pat_token",
|
||||||
|
"owner/repo"
|
||||||
|
)
|
||||||
|
|
||||||
|
expected = (
|
||||||
|
"https://private-user-images.githubusercontent.com"
|
||||||
|
"/123/abc.png?jwt=eyJhbGciOiJ"
|
||||||
|
)
|
||||||
|
assert result == expected
|
||||||
|
|
||||||
|
def test_markdown_api_returns_none_on_http_error(self):
|
||||||
|
"""HTTP errors return None."""
|
||||||
|
from urllib.error import HTTPError
|
||||||
|
|
||||||
|
error = HTTPError("http://test", 403, "Forbidden", {}, None)
|
||||||
|
with patch("github_backup.github_backup.urlopen", side_effect=error):
|
||||||
|
result = github_backup.get_jwt_signed_url_via_markdown_api(
|
||||||
|
"https://github.com/user-attachments/assets/abc123",
|
||||||
|
"github_pat_token",
|
||||||
|
"owner/repo"
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
def test_markdown_api_returns_none_when_no_jwt_url(self):
|
||||||
|
"""Response without JWT URL returns None."""
|
||||||
|
mock_response = Mock()
|
||||||
|
mock_response.read.return_value = b"<p>No image here</p>"
|
||||||
|
|
||||||
|
with patch("github_backup.github_backup.urlopen", return_value=mock_response):
|
||||||
|
result = github_backup.get_jwt_signed_url_via_markdown_api(
|
||||||
|
"https://github.com/user-attachments/assets/abc123",
|
||||||
|
"github_pat_token",
|
||||||
|
"owner/repo"
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
def test_needs_jwt_only_for_fine_grained_private_assets(self):
|
||||||
|
"""needs_jwt is True only for fine-grained + private + /assets/ URL."""
|
||||||
|
assets_url = "https://github.com/user-attachments/assets/abc123"
|
||||||
|
files_url = "https://github.com/user-attachments/files/123/doc.pdf"
|
||||||
|
token_fine = "github_pat_test"
|
||||||
|
private = True
|
||||||
|
public = False
|
||||||
|
|
||||||
|
# Fine-grained + private + assets = True
|
||||||
|
needs_jwt = (
|
||||||
|
token_fine is not None
|
||||||
|
and private
|
||||||
|
and "github.com/user-attachments/assets/" in assets_url
|
||||||
|
)
|
||||||
|
assert needs_jwt is True
|
||||||
|
|
||||||
|
# Fine-grained + private + files = False
|
||||||
|
needs_jwt = (
|
||||||
|
token_fine is not None
|
||||||
|
and private
|
||||||
|
and "github.com/user-attachments/assets/" in files_url
|
||||||
|
)
|
||||||
|
assert needs_jwt is False
|
||||||
|
|
||||||
|
# Fine-grained + public + assets = False
|
||||||
|
needs_jwt = (
|
||||||
|
token_fine is not None
|
||||||
|
and public
|
||||||
|
and "github.com/user-attachments/assets/" in assets_url
|
||||||
|
)
|
||||||
|
assert needs_jwt is False
|
||||||
|
|
||||||
|
def test_jwt_workaround_sets_manifest_flag(self, attachment_test_setup):
|
||||||
|
"""Successful JWT workaround sets jwt_workaround flag in manifest."""
|
||||||
|
setup = attachment_test_setup
|
||||||
|
setup["args"].token_fine = "github_pat_test"
|
||||||
|
setup["repository"]["private"] = True
|
||||||
|
|
||||||
|
issue_data = {"body": "https://github.com/user-attachments/assets/abc123"}
|
||||||
|
|
||||||
|
jwt_url = "https://private-user-images.githubusercontent.com/123/abc.png?jwt=token"
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"github_backup.github_backup.get_jwt_signed_url_via_markdown_api",
|
||||||
|
return_value=jwt_url
|
||||||
|
), patch(
|
||||||
|
"github_backup.github_backup.download_attachment_file",
|
||||||
|
return_value={"success": True, "http_status": 200, "url": jwt_url}
|
||||||
|
):
|
||||||
|
github_backup.download_attachments(
|
||||||
|
setup["args"], setup["issue_cwd"], issue_data, 123, setup["repository"]
|
||||||
|
)
|
||||||
|
|
||||||
|
manifest_path = os.path.join(setup["issue_cwd"], "attachments", "123", "manifest.json")
|
||||||
|
with open(manifest_path) as f:
|
||||||
|
manifest = json.load(f)
|
||||||
|
|
||||||
|
assert manifest["attachments"][0]["jwt_workaround"] is True
|
||||||
|
assert manifest["attachments"][0]["url"] == "https://github.com/user-attachments/assets/abc123"
|
||||||
|
|
||||||
|
def test_jwt_workaround_failure_uses_skipped_at(self, attachment_test_setup):
|
||||||
|
"""Failed JWT workaround uses skipped_at instead of downloaded_at."""
|
||||||
|
setup = attachment_test_setup
|
||||||
|
setup["args"].token_fine = "github_pat_test"
|
||||||
|
setup["repository"]["private"] = True
|
||||||
|
|
||||||
|
issue_data = {"body": "https://github.com/user-attachments/assets/abc123"}
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"github_backup.github_backup.get_jwt_signed_url_via_markdown_api",
|
||||||
|
return_value=None # Markdown API failed
|
||||||
|
):
|
||||||
|
github_backup.download_attachments(
|
||||||
|
setup["args"], setup["issue_cwd"], issue_data, 123, setup["repository"]
|
||||||
|
)
|
||||||
|
|
||||||
|
manifest_path = os.path.join(setup["issue_cwd"], "attachments", "123", "manifest.json")
|
||||||
|
with open(manifest_path) as f:
|
||||||
|
manifest = json.load(f)
|
||||||
|
|
||||||
|
attachment = manifest["attachments"][0]
|
||||||
|
assert attachment["success"] is False
|
||||||
|
assert "skipped_at" in attachment
|
||||||
|
assert "downloaded_at" not in attachment
|
||||||
|
assert "Use --token-classic" in attachment["error"]
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
"""Tests for case-insensitive username/organization filtering."""
|
"""Tests for case-insensitive username/organization filtering."""
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from unittest.mock import Mock
|
|
||||||
|
|
||||||
from github_backup import github_backup
|
from github_backup import github_backup
|
||||||
|
|
||||||
@@ -9,25 +8,14 @@ from github_backup import github_backup
|
|||||||
class TestCaseSensitivity:
|
class TestCaseSensitivity:
|
||||||
"""Test suite for case-insensitive username matching in filter_repositories."""
|
"""Test suite for case-insensitive username matching in filter_repositories."""
|
||||||
|
|
||||||
def test_filter_repositories_case_insensitive_user(self):
|
def test_filter_repositories_case_insensitive_user(self, create_args):
|
||||||
"""Should filter repositories case-insensitively for usernames.
|
"""Should filter repositories case-insensitively for usernames.
|
||||||
|
|
||||||
Reproduces issue #198 where typing 'iamrodos' fails to match
|
Reproduces issue #198 where typing 'iamrodos' fails to match
|
||||||
repositories with owner.login='Iamrodos' (the canonical case from GitHub API).
|
repositories with owner.login='Iamrodos' (the canonical case from GitHub API).
|
||||||
"""
|
"""
|
||||||
# Simulate user typing lowercase username
|
# Simulate user typing lowercase username
|
||||||
args = Mock()
|
args = create_args(user="iamrodos")
|
||||||
args.user = "iamrodos" # lowercase (what user typed)
|
|
||||||
args.repository = None
|
|
||||||
args.name_regex = None
|
|
||||||
args.languages = None
|
|
||||||
args.exclude = None
|
|
||||||
args.fork = False
|
|
||||||
args.private = False
|
|
||||||
args.public = False
|
|
||||||
args.all = True
|
|
||||||
args.skip_archived = False
|
|
||||||
args.starred_skip_size_over = None
|
|
||||||
|
|
||||||
# Simulate GitHub API returning canonical case
|
# Simulate GitHub API returning canonical case
|
||||||
repos = [
|
repos = [
|
||||||
@@ -52,23 +40,12 @@ class TestCaseSensitivity:
|
|||||||
assert filtered[0]["name"] == "repo1"
|
assert filtered[0]["name"] == "repo1"
|
||||||
assert filtered[1]["name"] == "repo2"
|
assert filtered[1]["name"] == "repo2"
|
||||||
|
|
||||||
def test_filter_repositories_case_insensitive_org(self):
|
def test_filter_repositories_case_insensitive_org(self, create_args):
|
||||||
"""Should filter repositories case-insensitively for organizations.
|
"""Should filter repositories case-insensitively for organizations.
|
||||||
|
|
||||||
Tests the example from issue #198 where 'prai-org' doesn't match 'PRAI-Org'.
|
Tests the example from issue #198 where 'prai-org' doesn't match 'PRAI-Org'.
|
||||||
"""
|
"""
|
||||||
args = Mock()
|
args = create_args(user="prai-org")
|
||||||
args.user = "prai-org" # lowercase (what user typed)
|
|
||||||
args.repository = None
|
|
||||||
args.name_regex = None
|
|
||||||
args.languages = None
|
|
||||||
args.exclude = None
|
|
||||||
args.fork = False
|
|
||||||
args.private = False
|
|
||||||
args.public = False
|
|
||||||
args.all = True
|
|
||||||
args.skip_archived = False
|
|
||||||
args.starred_skip_size_over = None
|
|
||||||
|
|
||||||
repos = [
|
repos = [
|
||||||
{
|
{
|
||||||
@@ -85,20 +62,9 @@ class TestCaseSensitivity:
|
|||||||
assert len(filtered) == 1
|
assert len(filtered) == 1
|
||||||
assert filtered[0]["name"] == "repo1"
|
assert filtered[0]["name"] == "repo1"
|
||||||
|
|
||||||
def test_filter_repositories_case_variations(self):
|
def test_filter_repositories_case_variations(self, create_args):
|
||||||
"""Should handle various case combinations correctly."""
|
"""Should handle various case combinations correctly."""
|
||||||
args = Mock()
|
args = create_args(user="TeSt-UsEr")
|
||||||
args.user = "TeSt-UsEr" # Mixed case
|
|
||||||
args.repository = None
|
|
||||||
args.name_regex = None
|
|
||||||
args.languages = None
|
|
||||||
args.exclude = None
|
|
||||||
args.fork = False
|
|
||||||
args.private = False
|
|
||||||
args.public = False
|
|
||||||
args.all = True
|
|
||||||
args.skip_archived = False
|
|
||||||
args.starred_skip_size_over = None
|
|
||||||
|
|
||||||
repos = [
|
repos = [
|
||||||
{"name": "repo1", "owner": {"login": "test-user"}, "private": False, "fork": False},
|
{"name": "repo1", "owner": {"login": "test-user"}, "private": False, "fork": False},
|
||||||
|
|||||||
@@ -11,16 +11,9 @@ from github_backup import github_backup
|
|||||||
class TestHTTP451Exception:
|
class TestHTTP451Exception:
|
||||||
"""Test suite for HTTP 451 DMCA takedown exception handling."""
|
"""Test suite for HTTP 451 DMCA takedown exception handling."""
|
||||||
|
|
||||||
def test_repository_unavailable_error_raised(self):
|
def test_repository_unavailable_error_raised(self, create_args):
|
||||||
"""HTTP 451 should raise RepositoryUnavailableError with DMCA URL."""
|
"""HTTP 451 should raise RepositoryUnavailableError with DMCA URL."""
|
||||||
args = Mock()
|
args = create_args()
|
||||||
args.as_app = False
|
|
||||||
args.token_fine = None
|
|
||||||
args.token_classic = None
|
|
||||||
args.osx_keychain_item_name = None
|
|
||||||
args.osx_keychain_item_account = None
|
|
||||||
args.throttle_limit = None
|
|
||||||
args.throttle_pause = 0
|
|
||||||
|
|
||||||
mock_response = Mock()
|
mock_response = Mock()
|
||||||
mock_response.getcode.return_value = 451
|
mock_response.getcode.return_value = 451
|
||||||
@@ -30,30 +23,31 @@ class TestHTTP451Exception:
|
|||||||
"block": {
|
"block": {
|
||||||
"reason": "dmca",
|
"reason": "dmca",
|
||||||
"created_at": "2024-11-12T14:38:04Z",
|
"created_at": "2024-11-12T14:38:04Z",
|
||||||
"html_url": "https://github.com/github/dmca/blob/master/2024/11/2024-11-04-source-code.md"
|
"html_url": "https://github.com/github/dmca/blob/master/2024/11/2024-11-04-source-code.md",
|
||||||
}
|
},
|
||||||
}
|
}
|
||||||
mock_response.read.return_value = json.dumps(dmca_data).encode("utf-8")
|
mock_response.read.return_value = json.dumps(dmca_data).encode("utf-8")
|
||||||
mock_response.headers = {"x-ratelimit-remaining": "5000"}
|
mock_response.headers = {"x-ratelimit-remaining": "5000"}
|
||||||
mock_response.reason = "Unavailable For Legal Reasons"
|
mock_response.reason = "Unavailable For Legal Reasons"
|
||||||
|
|
||||||
with patch("github_backup.github_backup.make_request_with_retry", return_value=mock_response):
|
with patch(
|
||||||
|
"github_backup.github_backup.make_request_with_retry",
|
||||||
|
return_value=mock_response,
|
||||||
|
):
|
||||||
with pytest.raises(github_backup.RepositoryUnavailableError) as exc_info:
|
with pytest.raises(github_backup.RepositoryUnavailableError) as exc_info:
|
||||||
github_backup.retrieve_data(args, "https://api.github.com/repos/test/dmca/issues")
|
github_backup.retrieve_data(
|
||||||
|
args, "https://api.github.com/repos/test/dmca/issues"
|
||||||
|
)
|
||||||
|
|
||||||
assert exc_info.value.dmca_url == "https://github.com/github/dmca/blob/master/2024/11/2024-11-04-source-code.md"
|
assert (
|
||||||
|
exc_info.value.dmca_url
|
||||||
|
== "https://github.com/github/dmca/blob/master/2024/11/2024-11-04-source-code.md"
|
||||||
|
)
|
||||||
assert "451" in str(exc_info.value)
|
assert "451" in str(exc_info.value)
|
||||||
|
|
||||||
def test_repository_unavailable_error_without_dmca_url(self):
|
def test_repository_unavailable_error_without_dmca_url(self, create_args):
|
||||||
"""HTTP 451 without DMCA details should still raise exception."""
|
"""HTTP 451 without DMCA details should still raise exception."""
|
||||||
args = Mock()
|
args = create_args()
|
||||||
args.as_app = False
|
|
||||||
args.token_fine = None
|
|
||||||
args.token_classic = None
|
|
||||||
args.osx_keychain_item_name = None
|
|
||||||
args.osx_keychain_item_account = None
|
|
||||||
args.throttle_limit = None
|
|
||||||
args.throttle_pause = 0
|
|
||||||
|
|
||||||
mock_response = Mock()
|
mock_response = Mock()
|
||||||
mock_response.getcode.return_value = 451
|
mock_response.getcode.return_value = 451
|
||||||
@@ -61,23 +55,21 @@ class TestHTTP451Exception:
|
|||||||
mock_response.headers = {"x-ratelimit-remaining": "5000"}
|
mock_response.headers = {"x-ratelimit-remaining": "5000"}
|
||||||
mock_response.reason = "Unavailable For Legal Reasons"
|
mock_response.reason = "Unavailable For Legal Reasons"
|
||||||
|
|
||||||
with patch("github_backup.github_backup.make_request_with_retry", return_value=mock_response):
|
with patch(
|
||||||
|
"github_backup.github_backup.make_request_with_retry",
|
||||||
|
return_value=mock_response,
|
||||||
|
):
|
||||||
with pytest.raises(github_backup.RepositoryUnavailableError) as exc_info:
|
with pytest.raises(github_backup.RepositoryUnavailableError) as exc_info:
|
||||||
github_backup.retrieve_data(args, "https://api.github.com/repos/test/dmca/issues")
|
github_backup.retrieve_data(
|
||||||
|
args, "https://api.github.com/repos/test/dmca/issues"
|
||||||
|
)
|
||||||
|
|
||||||
assert exc_info.value.dmca_url is None
|
assert exc_info.value.dmca_url is None
|
||||||
assert "451" in str(exc_info.value)
|
assert "451" in str(exc_info.value)
|
||||||
|
|
||||||
def test_repository_unavailable_error_with_malformed_json(self):
|
def test_repository_unavailable_error_with_malformed_json(self, create_args):
|
||||||
"""HTTP 451 with malformed JSON should still raise exception."""
|
"""HTTP 451 with malformed JSON should still raise exception."""
|
||||||
args = Mock()
|
args = create_args()
|
||||||
args.as_app = False
|
|
||||||
args.token_fine = None
|
|
||||||
args.token_classic = None
|
|
||||||
args.osx_keychain_item_name = None
|
|
||||||
args.osx_keychain_item_account = None
|
|
||||||
args.throttle_limit = None
|
|
||||||
args.throttle_pause = 0
|
|
||||||
|
|
||||||
mock_response = Mock()
|
mock_response = Mock()
|
||||||
mock_response.getcode.return_value = 451
|
mock_response.getcode.return_value = 451
|
||||||
@@ -85,9 +77,14 @@ class TestHTTP451Exception:
|
|||||||
mock_response.headers = {"x-ratelimit-remaining": "5000"}
|
mock_response.headers = {"x-ratelimit-remaining": "5000"}
|
||||||
mock_response.reason = "Unavailable For Legal Reasons"
|
mock_response.reason = "Unavailable For Legal Reasons"
|
||||||
|
|
||||||
with patch("github_backup.github_backup.make_request_with_retry", return_value=mock_response):
|
with patch(
|
||||||
|
"github_backup.github_backup.make_request_with_retry",
|
||||||
|
return_value=mock_response,
|
||||||
|
):
|
||||||
with pytest.raises(github_backup.RepositoryUnavailableError):
|
with pytest.raises(github_backup.RepositoryUnavailableError):
|
||||||
github_backup.retrieve_data(args, "https://api.github.com/repos/test/dmca/issues")
|
github_backup.retrieve_data(
|
||||||
|
args, "https://api.github.com/repos/test/dmca/issues"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -1,9 +1,7 @@
|
|||||||
"""Tests for Link header pagination handling."""
|
"""Tests for Link header pagination handling."""
|
||||||
|
|
||||||
import json
|
import json
|
||||||
from unittest.mock import Mock, patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
from github_backup import github_backup
|
from github_backup import github_backup
|
||||||
|
|
||||||
@@ -38,22 +36,9 @@ class MockHTTPResponse:
|
|||||||
return headers
|
return headers
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
def test_cursor_based_pagination(create_args):
|
||||||
def mock_args():
|
|
||||||
"""Mock args for retrieve_data."""
|
|
||||||
args = Mock()
|
|
||||||
args.as_app = False
|
|
||||||
args.token_fine = None
|
|
||||||
args.token_classic = "fake_token"
|
|
||||||
args.osx_keychain_item_name = None
|
|
||||||
args.osx_keychain_item_account = None
|
|
||||||
args.throttle_limit = None
|
|
||||||
args.throttle_pause = 0
|
|
||||||
return args
|
|
||||||
|
|
||||||
|
|
||||||
def test_cursor_based_pagination(mock_args):
|
|
||||||
"""Link header with 'after' cursor parameter works correctly."""
|
"""Link header with 'after' cursor parameter works correctly."""
|
||||||
|
args = create_args(token_classic="fake_token")
|
||||||
|
|
||||||
# Simulate issues endpoint behavior: returns cursor in Link header
|
# Simulate issues endpoint behavior: returns cursor in Link header
|
||||||
responses = [
|
responses = [
|
||||||
@@ -76,7 +61,7 @@ def test_cursor_based_pagination(mock_args):
|
|||||||
|
|
||||||
with patch("github_backup.github_backup.urlopen", side_effect=mock_urlopen):
|
with patch("github_backup.github_backup.urlopen", side_effect=mock_urlopen):
|
||||||
results = github_backup.retrieve_data(
|
results = github_backup.retrieve_data(
|
||||||
mock_args, "https://api.github.com/repos/owner/repo/issues"
|
args, "https://api.github.com/repos/owner/repo/issues"
|
||||||
)
|
)
|
||||||
|
|
||||||
# Verify all items retrieved and cursor was used in second request
|
# Verify all items retrieved and cursor was used in second request
|
||||||
@@ -85,8 +70,9 @@ def test_cursor_based_pagination(mock_args):
|
|||||||
assert "after=ABC123" in requests_made[1]
|
assert "after=ABC123" in requests_made[1]
|
||||||
|
|
||||||
|
|
||||||
def test_page_based_pagination(mock_args):
|
def test_page_based_pagination(create_args):
|
||||||
"""Link header with 'page' parameter works correctly."""
|
"""Link header with 'page' parameter works correctly."""
|
||||||
|
args = create_args(token_classic="fake_token")
|
||||||
|
|
||||||
# Simulate pulls/repos endpoint behavior: returns page numbers in Link header
|
# Simulate pulls/repos endpoint behavior: returns page numbers in Link header
|
||||||
responses = [
|
responses = [
|
||||||
@@ -109,7 +95,7 @@ def test_page_based_pagination(mock_args):
|
|||||||
|
|
||||||
with patch("github_backup.github_backup.urlopen", side_effect=mock_urlopen):
|
with patch("github_backup.github_backup.urlopen", side_effect=mock_urlopen):
|
||||||
results = github_backup.retrieve_data(
|
results = github_backup.retrieve_data(
|
||||||
mock_args, "https://api.github.com/repos/owner/repo/pulls"
|
args, "https://api.github.com/repos/owner/repo/pulls"
|
||||||
)
|
)
|
||||||
|
|
||||||
# Verify all items retrieved and page parameter was used (not cursor)
|
# Verify all items retrieved and page parameter was used (not cursor)
|
||||||
@@ -119,8 +105,9 @@ def test_page_based_pagination(mock_args):
|
|||||||
assert "after" not in requests_made[1]
|
assert "after" not in requests_made[1]
|
||||||
|
|
||||||
|
|
||||||
def test_no_link_header_stops_pagination(mock_args):
|
def test_no_link_header_stops_pagination(create_args):
|
||||||
"""Pagination stops when Link header is absent."""
|
"""Pagination stops when Link header is absent."""
|
||||||
|
args = create_args(token_classic="fake_token")
|
||||||
|
|
||||||
# Simulate endpoint with results that fit in a single page
|
# Simulate endpoint with results that fit in a single page
|
||||||
responses = [
|
responses = [
|
||||||
@@ -137,7 +124,7 @@ def test_no_link_header_stops_pagination(mock_args):
|
|||||||
|
|
||||||
with patch("github_backup.github_backup.urlopen", side_effect=mock_urlopen):
|
with patch("github_backup.github_backup.urlopen", side_effect=mock_urlopen):
|
||||||
results = github_backup.retrieve_data(
|
results = github_backup.retrieve_data(
|
||||||
mock_args, "https://api.github.com/repos/owner/repo/labels"
|
args, "https://api.github.com/repos/owner/repo/labels"
|
||||||
)
|
)
|
||||||
|
|
||||||
# Verify pagination stopped after first request
|
# Verify pagination stopped after first request
|
||||||
|
|||||||
@@ -9,26 +9,27 @@ import pytest
|
|||||||
|
|
||||||
from github_backup import github_backup
|
from github_backup import github_backup
|
||||||
from github_backup.github_backup import (
|
from github_backup.github_backup import (
|
||||||
MAX_RETRIES,
|
|
||||||
calculate_retry_delay,
|
calculate_retry_delay,
|
||||||
make_request_with_retry,
|
make_request_with_retry,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Default retry count used in tests (matches argparse default)
|
||||||
|
# With max_retries=5, total attempts = 6 (1 initial + 5 retries)
|
||||||
|
DEFAULT_MAX_RETRIES = 5
|
||||||
|
|
||||||
|
|
||||||
class TestCalculateRetryDelay:
|
class TestCalculateRetryDelay:
|
||||||
def test_respects_retry_after_header(self):
|
def test_respects_retry_after_header(self):
|
||||||
headers = {'retry-after': '30'}
|
headers = {"retry-after": "30"}
|
||||||
assert calculate_retry_delay(0, headers) == 30
|
assert calculate_retry_delay(0, headers) == 30
|
||||||
|
|
||||||
def test_respects_rate_limit_reset(self):
|
def test_respects_rate_limit_reset(self):
|
||||||
import time
|
import time
|
||||||
import calendar
|
import calendar
|
||||||
|
|
||||||
# Set reset time 60 seconds in the future
|
# Set reset time 60 seconds in the future
|
||||||
future_reset = calendar.timegm(time.gmtime()) + 60
|
future_reset = calendar.timegm(time.gmtime()) + 60
|
||||||
headers = {
|
headers = {"x-ratelimit-remaining": "0", "x-ratelimit-reset": str(future_reset)}
|
||||||
'x-ratelimit-remaining': '0',
|
|
||||||
'x-ratelimit-reset': str(future_reset)
|
|
||||||
}
|
|
||||||
delay = calculate_retry_delay(0, headers)
|
delay = calculate_retry_delay(0, headers)
|
||||||
# Should be approximately 60 seconds (with some tolerance for execution time)
|
# Should be approximately 60 seconds (with some tolerance for execution time)
|
||||||
assert 55 <= delay <= 65
|
assert 55 <= delay <= 65
|
||||||
@@ -50,12 +51,10 @@ class TestCalculateRetryDelay:
|
|||||||
def test_minimum_rate_limit_delay(self):
|
def test_minimum_rate_limit_delay(self):
|
||||||
import time
|
import time
|
||||||
import calendar
|
import calendar
|
||||||
|
|
||||||
# Set reset time in the past (already reset)
|
# Set reset time in the past (already reset)
|
||||||
past_reset = calendar.timegm(time.gmtime()) - 100
|
past_reset = calendar.timegm(time.gmtime()) - 100
|
||||||
headers = {
|
headers = {"x-ratelimit-remaining": "0", "x-ratelimit-reset": str(past_reset)}
|
||||||
'x-ratelimit-remaining': '0',
|
|
||||||
'x-ratelimit-reset': str(past_reset)
|
|
||||||
}
|
|
||||||
delay = calculate_retry_delay(0, headers)
|
delay = calculate_retry_delay(0, headers)
|
||||||
# Should be minimum 10 seconds even if reset time is in past
|
# Should be minimum 10 seconds even if reset time is in past
|
||||||
assert delay >= 10
|
assert delay >= 10
|
||||||
@@ -64,20 +63,9 @@ class TestCalculateRetryDelay:
|
|||||||
class TestRetrieveDataRetry:
|
class TestRetrieveDataRetry:
|
||||||
"""Tests for retry behavior in retrieve_data."""
|
"""Tests for retry behavior in retrieve_data."""
|
||||||
|
|
||||||
@pytest.fixture
|
def test_json_parse_error_retries_and_fails(self, create_args):
|
||||||
def mock_args(self):
|
|
||||||
args = Mock()
|
|
||||||
args.as_app = False
|
|
||||||
args.token_fine = None
|
|
||||||
args.token_classic = "fake_token"
|
|
||||||
args.osx_keychain_item_name = None
|
|
||||||
args.osx_keychain_item_account = None
|
|
||||||
args.throttle_limit = None
|
|
||||||
args.throttle_pause = 0
|
|
||||||
return args
|
|
||||||
|
|
||||||
def test_json_parse_error_retries_and_fails(self, mock_args):
|
|
||||||
"""HTTP 200 with invalid JSON should retry and eventually fail."""
|
"""HTTP 200 with invalid JSON should retry and eventually fail."""
|
||||||
|
args = create_args(token_classic="fake_token")
|
||||||
mock_response = Mock()
|
mock_response = Mock()
|
||||||
mock_response.getcode.return_value = 200
|
mock_response.getcode.return_value = 200
|
||||||
mock_response.read.return_value = b"not valid json {"
|
mock_response.read.return_value = b"not valid json {"
|
||||||
@@ -85,21 +73,31 @@ class TestRetrieveDataRetry:
|
|||||||
|
|
||||||
call_count = 0
|
call_count = 0
|
||||||
|
|
||||||
def mock_make_request(*args, **kwargs):
|
def mock_make_request(*a, **kw):
|
||||||
nonlocal call_count
|
nonlocal call_count
|
||||||
call_count += 1
|
call_count += 1
|
||||||
return mock_response
|
return mock_response
|
||||||
|
|
||||||
with patch("github_backup.github_backup.make_request_with_retry", side_effect=mock_make_request):
|
with patch(
|
||||||
with patch("github_backup.github_backup.calculate_retry_delay", return_value=0): # No delay in tests
|
"github_backup.github_backup.make_request_with_retry",
|
||||||
|
side_effect=mock_make_request,
|
||||||
|
):
|
||||||
|
with patch(
|
||||||
|
"github_backup.github_backup.calculate_retry_delay", return_value=0
|
||||||
|
): # No delay in tests
|
||||||
with pytest.raises(Exception) as exc_info:
|
with pytest.raises(Exception) as exc_info:
|
||||||
github_backup.retrieve_data(mock_args, "https://api.github.com/repos/test/repo/issues")
|
github_backup.retrieve_data(
|
||||||
|
args, "https://api.github.com/repos/test/repo/issues"
|
||||||
|
)
|
||||||
|
|
||||||
assert "Failed to read response after" in str(exc_info.value)
|
assert "Failed to read response after" in str(exc_info.value)
|
||||||
assert call_count == MAX_RETRIES
|
assert (
|
||||||
|
call_count == DEFAULT_MAX_RETRIES + 1
|
||||||
|
) # 1 initial + 5 retries = 6 attempts
|
||||||
|
|
||||||
def test_json_parse_error_recovers_on_retry(self, mock_args):
|
def test_json_parse_error_recovers_on_retry(self, create_args):
|
||||||
"""HTTP 200 with invalid JSON should succeed if retry returns valid JSON."""
|
"""HTTP 200 with invalid JSON should succeed if retry returns valid JSON."""
|
||||||
|
args = create_args(token_classic="fake_token")
|
||||||
bad_response = Mock()
|
bad_response = Mock()
|
||||||
bad_response.getcode.return_value = 200
|
bad_response.getcode.return_value = 200
|
||||||
bad_response.read.return_value = b"not valid json {"
|
bad_response.read.return_value = b"not valid json {"
|
||||||
@@ -113,32 +111,47 @@ class TestRetrieveDataRetry:
|
|||||||
responses = [bad_response, bad_response, good_response]
|
responses = [bad_response, bad_response, good_response]
|
||||||
call_count = 0
|
call_count = 0
|
||||||
|
|
||||||
def mock_make_request(*args, **kwargs):
|
def mock_make_request(*a, **kw):
|
||||||
nonlocal call_count
|
nonlocal call_count
|
||||||
result = responses[call_count]
|
result = responses[call_count]
|
||||||
call_count += 1
|
call_count += 1
|
||||||
return result
|
return result
|
||||||
|
|
||||||
with patch("github_backup.github_backup.make_request_with_retry", side_effect=mock_make_request):
|
with patch(
|
||||||
with patch("github_backup.github_backup.calculate_retry_delay", return_value=0):
|
"github_backup.github_backup.make_request_with_retry",
|
||||||
result = github_backup.retrieve_data(mock_args, "https://api.github.com/repos/test/repo/issues")
|
side_effect=mock_make_request,
|
||||||
|
):
|
||||||
|
with patch(
|
||||||
|
"github_backup.github_backup.calculate_retry_delay", return_value=0
|
||||||
|
):
|
||||||
|
result = github_backup.retrieve_data(
|
||||||
|
args, "https://api.github.com/repos/test/repo/issues"
|
||||||
|
)
|
||||||
|
|
||||||
assert result == [{"id": 1}]
|
assert result == [{"id": 1}]
|
||||||
assert call_count == 3 # Failed twice, succeeded on third
|
assert call_count == 3 # Failed twice, succeeded on third
|
||||||
|
|
||||||
def test_http_error_raises_exception(self, mock_args):
|
def test_http_error_raises_exception(self, create_args):
|
||||||
"""Non-success HTTP status codes should raise Exception."""
|
"""Non-success HTTP status codes should raise Exception."""
|
||||||
|
args = create_args(token_classic="fake_token")
|
||||||
mock_response = Mock()
|
mock_response = Mock()
|
||||||
mock_response.getcode.return_value = 404
|
mock_response.getcode.return_value = 404
|
||||||
mock_response.read.return_value = b'{"message": "Not Found"}'
|
mock_response.read.return_value = b'{"message": "Not Found"}'
|
||||||
mock_response.headers = {"x-ratelimit-remaining": "5000"}
|
mock_response.headers = {"x-ratelimit-remaining": "5000"}
|
||||||
mock_response.reason = "Not Found"
|
mock_response.reason = "Not Found"
|
||||||
|
|
||||||
with patch("github_backup.github_backup.make_request_with_retry", return_value=mock_response):
|
with patch(
|
||||||
|
"github_backup.github_backup.make_request_with_retry",
|
||||||
|
return_value=mock_response,
|
||||||
|
):
|
||||||
with pytest.raises(Exception) as exc_info:
|
with pytest.raises(Exception) as exc_info:
|
||||||
github_backup.retrieve_data(mock_args, "https://api.github.com/repos/test/notfound/issues")
|
github_backup.retrieve_data(
|
||||||
|
args, "https://api.github.com/repos/test/notfound/issues"
|
||||||
|
)
|
||||||
|
|
||||||
assert not isinstance(exc_info.value, github_backup.RepositoryUnavailableError)
|
assert not isinstance(
|
||||||
|
exc_info.value, github_backup.RepositoryUnavailableError
|
||||||
|
)
|
||||||
assert "404" in str(exc_info.value)
|
assert "404" in str(exc_info.value)
|
||||||
|
|
||||||
|
|
||||||
@@ -151,7 +164,7 @@ class TestMakeRequestWithRetry:
|
|||||||
good_response.read.return_value = b'{"ok": true}'
|
good_response.read.return_value = b'{"ok": true}'
|
||||||
|
|
||||||
call_count = 0
|
call_count = 0
|
||||||
fail_count = MAX_RETRIES - 1 # Fail all but last attempt
|
fail_count = DEFAULT_MAX_RETRIES # Fail all retries, succeed on last attempt
|
||||||
|
|
||||||
def mock_urlopen(*args, **kwargs):
|
def mock_urlopen(*args, **kwargs):
|
||||||
nonlocal call_count
|
nonlocal call_count
|
||||||
@@ -167,14 +180,18 @@ class TestMakeRequestWithRetry:
|
|||||||
return good_response
|
return good_response
|
||||||
|
|
||||||
with patch("github_backup.github_backup.urlopen", side_effect=mock_urlopen):
|
with patch("github_backup.github_backup.urlopen", side_effect=mock_urlopen):
|
||||||
with patch("github_backup.github_backup.calculate_retry_delay", return_value=0):
|
with patch(
|
||||||
|
"github_backup.github_backup.calculate_retry_delay", return_value=0
|
||||||
|
):
|
||||||
result = make_request_with_retry(Mock(), None)
|
result = make_request_with_retry(Mock(), None)
|
||||||
|
|
||||||
assert result == good_response
|
assert result == good_response
|
||||||
assert call_count == MAX_RETRIES
|
assert (
|
||||||
|
call_count == DEFAULT_MAX_RETRIES + 1
|
||||||
|
) # 1 initial + 5 retries = 6 attempts
|
||||||
|
|
||||||
def test_503_error_retries_until_exhausted(self):
|
def test_503_error_retries_until_exhausted(self):
|
||||||
"""HTTP 503 should retry MAX_RETRIES times then raise."""
|
"""HTTP 503 should make 1 initial + DEFAULT_MAX_RETRIES retry attempts then raise."""
|
||||||
call_count = 0
|
call_count = 0
|
||||||
|
|
||||||
def mock_urlopen(*args, **kwargs):
|
def mock_urlopen(*args, **kwargs):
|
||||||
@@ -189,12 +206,16 @@ class TestMakeRequestWithRetry:
|
|||||||
)
|
)
|
||||||
|
|
||||||
with patch("github_backup.github_backup.urlopen", side_effect=mock_urlopen):
|
with patch("github_backup.github_backup.urlopen", side_effect=mock_urlopen):
|
||||||
with patch("github_backup.github_backup.calculate_retry_delay", return_value=0):
|
with patch(
|
||||||
|
"github_backup.github_backup.calculate_retry_delay", return_value=0
|
||||||
|
):
|
||||||
with pytest.raises(HTTPError) as exc_info:
|
with pytest.raises(HTTPError) as exc_info:
|
||||||
make_request_with_retry(Mock(), None)
|
make_request_with_retry(Mock(), None)
|
||||||
|
|
||||||
assert exc_info.value.code == 503
|
assert exc_info.value.code == 503
|
||||||
assert call_count == MAX_RETRIES
|
assert (
|
||||||
|
call_count == DEFAULT_MAX_RETRIES + 1
|
||||||
|
) # 1 initial + 5 retries = 6 attempts
|
||||||
|
|
||||||
def test_404_error_not_retried(self):
|
def test_404_error_not_retried(self):
|
||||||
"""HTTP 404 should not be retried - raise immediately."""
|
"""HTTP 404 should not be retried - raise immediately."""
|
||||||
@@ -237,7 +258,9 @@ class TestMakeRequestWithRetry:
|
|||||||
return good_response
|
return good_response
|
||||||
|
|
||||||
with patch("github_backup.github_backup.urlopen", side_effect=mock_urlopen):
|
with patch("github_backup.github_backup.urlopen", side_effect=mock_urlopen):
|
||||||
with patch("github_backup.github_backup.calculate_retry_delay", return_value=0):
|
with patch(
|
||||||
|
"github_backup.github_backup.calculate_retry_delay", return_value=0
|
||||||
|
):
|
||||||
result = make_request_with_retry(Mock(), None)
|
result = make_request_with_retry(Mock(), None)
|
||||||
|
|
||||||
assert result == good_response
|
assert result == good_response
|
||||||
@@ -269,7 +292,7 @@ class TestMakeRequestWithRetry:
|
|||||||
"""URLError (connection error) should retry and succeed if subsequent request works."""
|
"""URLError (connection error) should retry and succeed if subsequent request works."""
|
||||||
good_response = Mock()
|
good_response = Mock()
|
||||||
call_count = 0
|
call_count = 0
|
||||||
fail_count = MAX_RETRIES - 1 # Fail all but last attempt
|
fail_count = DEFAULT_MAX_RETRIES # Fail all retries, succeed on last attempt
|
||||||
|
|
||||||
def mock_urlopen(*args, **kwargs):
|
def mock_urlopen(*args, **kwargs):
|
||||||
nonlocal call_count
|
nonlocal call_count
|
||||||
@@ -279,14 +302,18 @@ class TestMakeRequestWithRetry:
|
|||||||
return good_response
|
return good_response
|
||||||
|
|
||||||
with patch("github_backup.github_backup.urlopen", side_effect=mock_urlopen):
|
with patch("github_backup.github_backup.urlopen", side_effect=mock_urlopen):
|
||||||
with patch("github_backup.github_backup.calculate_retry_delay", return_value=0):
|
with patch(
|
||||||
|
"github_backup.github_backup.calculate_retry_delay", return_value=0
|
||||||
|
):
|
||||||
result = make_request_with_retry(Mock(), None)
|
result = make_request_with_retry(Mock(), None)
|
||||||
|
|
||||||
assert result == good_response
|
assert result == good_response
|
||||||
assert call_count == MAX_RETRIES
|
assert (
|
||||||
|
call_count == DEFAULT_MAX_RETRIES + 1
|
||||||
|
) # 1 initial + 5 retries = 6 attempts
|
||||||
|
|
||||||
def test_socket_error_retries_until_exhausted(self):
|
def test_socket_error_retries_until_exhausted(self):
|
||||||
"""socket.error should retry MAX_RETRIES times then raise."""
|
"""socket.error should make 1 initial + DEFAULT_MAX_RETRIES retry attempts then raise."""
|
||||||
call_count = 0
|
call_count = 0
|
||||||
|
|
||||||
def mock_urlopen(*args, **kwargs):
|
def mock_urlopen(*args, **kwargs):
|
||||||
@@ -295,38 +322,43 @@ class TestMakeRequestWithRetry:
|
|||||||
raise socket.error("Connection reset by peer")
|
raise socket.error("Connection reset by peer")
|
||||||
|
|
||||||
with patch("github_backup.github_backup.urlopen", side_effect=mock_urlopen):
|
with patch("github_backup.github_backup.urlopen", side_effect=mock_urlopen):
|
||||||
with patch("github_backup.github_backup.calculate_retry_delay", return_value=0):
|
with patch(
|
||||||
|
"github_backup.github_backup.calculate_retry_delay", return_value=0
|
||||||
|
):
|
||||||
with pytest.raises(socket.error):
|
with pytest.raises(socket.error):
|
||||||
make_request_with_retry(Mock(), None)
|
make_request_with_retry(Mock(), None)
|
||||||
|
|
||||||
assert call_count == MAX_RETRIES
|
assert (
|
||||||
|
call_count == DEFAULT_MAX_RETRIES + 1
|
||||||
|
) # 1 initial + 5 retries = 6 attempts
|
||||||
|
|
||||||
|
|
||||||
class TestRetrieveDataThrottling:
|
class TestRetrieveDataThrottling:
|
||||||
"""Tests for throttling behavior in retrieve_data."""
|
"""Tests for throttling behavior in retrieve_data."""
|
||||||
|
|
||||||
@pytest.fixture
|
def test_throttling_pauses_when_rate_limit_low(self, create_args):
|
||||||
def mock_args(self):
|
|
||||||
args = Mock()
|
|
||||||
args.as_app = False
|
|
||||||
args.token_fine = None
|
|
||||||
args.token_classic = "fake_token"
|
|
||||||
args.osx_keychain_item_name = None
|
|
||||||
args.osx_keychain_item_account = None
|
|
||||||
args.throttle_limit = 10 # Throttle when remaining <= 10
|
|
||||||
args.throttle_pause = 5 # Pause 5 seconds
|
|
||||||
return args
|
|
||||||
|
|
||||||
def test_throttling_pauses_when_rate_limit_low(self, mock_args):
|
|
||||||
"""Should pause when x-ratelimit-remaining is at or below throttle_limit."""
|
"""Should pause when x-ratelimit-remaining is at or below throttle_limit."""
|
||||||
|
args = create_args(
|
||||||
|
token_classic="fake_token",
|
||||||
|
throttle_limit=10,
|
||||||
|
throttle_pause=5,
|
||||||
|
)
|
||||||
mock_response = Mock()
|
mock_response = Mock()
|
||||||
mock_response.getcode.return_value = 200
|
mock_response.getcode.return_value = 200
|
||||||
mock_response.read.return_value = json.dumps([{"id": 1}]).encode("utf-8")
|
mock_response.read.return_value = json.dumps([{"id": 1}]).encode("utf-8")
|
||||||
mock_response.headers = {"x-ratelimit-remaining": "5", "Link": ""} # Below throttle_limit
|
mock_response.headers = {
|
||||||
|
"x-ratelimit-remaining": "5",
|
||||||
|
"Link": "",
|
||||||
|
} # Below throttle_limit
|
||||||
|
|
||||||
with patch("github_backup.github_backup.make_request_with_retry", return_value=mock_response):
|
with patch(
|
||||||
|
"github_backup.github_backup.make_request_with_retry",
|
||||||
|
return_value=mock_response,
|
||||||
|
):
|
||||||
with patch("github_backup.github_backup.time.sleep") as mock_sleep:
|
with patch("github_backup.github_backup.time.sleep") as mock_sleep:
|
||||||
github_backup.retrieve_data(mock_args, "https://api.github.com/repos/test/repo/issues")
|
github_backup.retrieve_data(
|
||||||
|
args, "https://api.github.com/repos/test/repo/issues"
|
||||||
|
)
|
||||||
|
|
||||||
mock_sleep.assert_called_once_with(5) # throttle_pause value
|
mock_sleep.assert_called_once_with(5) # throttle_pause value
|
||||||
|
|
||||||
@@ -334,26 +366,114 @@ class TestRetrieveDataThrottling:
|
|||||||
class TestRetrieveDataSingleItem:
|
class TestRetrieveDataSingleItem:
|
||||||
"""Tests for single item (dict) responses in retrieve_data."""
|
"""Tests for single item (dict) responses in retrieve_data."""
|
||||||
|
|
||||||
@pytest.fixture
|
def test_dict_response_returned_as_list(self, create_args):
|
||||||
def mock_args(self):
|
|
||||||
args = Mock()
|
|
||||||
args.as_app = False
|
|
||||||
args.token_fine = None
|
|
||||||
args.token_classic = "fake_token"
|
|
||||||
args.osx_keychain_item_name = None
|
|
||||||
args.osx_keychain_item_account = None
|
|
||||||
args.throttle_limit = None
|
|
||||||
args.throttle_pause = 0
|
|
||||||
return args
|
|
||||||
|
|
||||||
def test_dict_response_returned_as_list(self, mock_args):
|
|
||||||
"""Single dict response should be returned as a list with one item."""
|
"""Single dict response should be returned as a list with one item."""
|
||||||
|
args = create_args(token_classic="fake_token")
|
||||||
mock_response = Mock()
|
mock_response = Mock()
|
||||||
mock_response.getcode.return_value = 200
|
mock_response.getcode.return_value = 200
|
||||||
mock_response.read.return_value = json.dumps({"login": "testuser", "id": 123}).encode("utf-8")
|
mock_response.read.return_value = json.dumps(
|
||||||
|
{"login": "testuser", "id": 123}
|
||||||
|
).encode("utf-8")
|
||||||
mock_response.headers = {"x-ratelimit-remaining": "5000", "Link": ""}
|
mock_response.headers = {"x-ratelimit-remaining": "5000", "Link": ""}
|
||||||
|
|
||||||
with patch("github_backup.github_backup.make_request_with_retry", return_value=mock_response):
|
with patch(
|
||||||
result = github_backup.retrieve_data(mock_args, "https://api.github.com/user")
|
"github_backup.github_backup.make_request_with_retry",
|
||||||
|
return_value=mock_response,
|
||||||
|
):
|
||||||
|
result = github_backup.retrieve_data(
|
||||||
|
args, "https://api.github.com/user"
|
||||||
|
)
|
||||||
|
|
||||||
assert result == [{"login": "testuser", "id": 123}]
|
assert result == [{"login": "testuser", "id": 123}]
|
||||||
|
|
||||||
|
|
||||||
|
class TestRetriesCliArgument:
|
||||||
|
"""Tests for --retries CLI argument validation and behavior."""
|
||||||
|
|
||||||
|
def test_retries_argument_accepted(self):
|
||||||
|
"""--retries flag should be accepted and parsed correctly."""
|
||||||
|
args = github_backup.parse_args(["--retries", "3", "testuser"])
|
||||||
|
assert args.max_retries == 3
|
||||||
|
|
||||||
|
def test_retries_default_value(self):
|
||||||
|
"""--retries should default to 5 if not specified."""
|
||||||
|
args = github_backup.parse_args(["testuser"])
|
||||||
|
assert args.max_retries == 5
|
||||||
|
|
||||||
|
def test_retries_zero_is_valid(self):
|
||||||
|
"""--retries 0 should be valid and mean 1 attempt (no retries)."""
|
||||||
|
args = github_backup.parse_args(["--retries", "0", "testuser"])
|
||||||
|
assert args.max_retries == 0
|
||||||
|
|
||||||
|
def test_retries_negative_rejected(self):
|
||||||
|
"""--retries with negative value should be rejected by argparse."""
|
||||||
|
with pytest.raises(SystemExit):
|
||||||
|
github_backup.parse_args(["--retries", "-1", "testuser"])
|
||||||
|
|
||||||
|
def test_retries_non_integer_rejected(self):
|
||||||
|
"""--retries with non-integer value should be rejected by argparse."""
|
||||||
|
with pytest.raises(SystemExit):
|
||||||
|
github_backup.parse_args(["--retries", "abc", "testuser"])
|
||||||
|
|
||||||
|
def test_retries_one_with_transient_error_succeeds(self):
|
||||||
|
"""--retries 1 should allow one retry after initial failure."""
|
||||||
|
good_response = Mock()
|
||||||
|
good_response.read.return_value = b'{"ok": true}'
|
||||||
|
|
||||||
|
call_count = 0
|
||||||
|
|
||||||
|
def mock_urlopen(*args, **kwargs):
|
||||||
|
nonlocal call_count
|
||||||
|
call_count += 1
|
||||||
|
if call_count == 1:
|
||||||
|
raise HTTPError(
|
||||||
|
url="https://api.github.com/test",
|
||||||
|
code=502,
|
||||||
|
msg="Bad Gateway",
|
||||||
|
hdrs={"x-ratelimit-remaining": "5000"},
|
||||||
|
fp=None,
|
||||||
|
)
|
||||||
|
return good_response
|
||||||
|
|
||||||
|
with patch("github_backup.github_backup.urlopen", side_effect=mock_urlopen):
|
||||||
|
with patch(
|
||||||
|
"github_backup.github_backup.calculate_retry_delay", return_value=0
|
||||||
|
):
|
||||||
|
result = make_request_with_retry(Mock(), None, max_retries=1)
|
||||||
|
|
||||||
|
assert result == good_response
|
||||||
|
assert call_count == 2 # 1 initial + 1 retry = 2 attempts
|
||||||
|
|
||||||
|
def test_custom_retry_count_limits_attempts(self, create_args):
|
||||||
|
"""Custom --retries value should limit actual retry attempts."""
|
||||||
|
args = create_args(
|
||||||
|
token_classic="fake_token",
|
||||||
|
max_retries=2, # 2 retries = 3 total attempts (1 initial + 2 retries)
|
||||||
|
)
|
||||||
|
|
||||||
|
mock_response = Mock()
|
||||||
|
mock_response.getcode.return_value = 200
|
||||||
|
mock_response.read.return_value = b"not valid json {"
|
||||||
|
mock_response.headers = {"x-ratelimit-remaining": "5000"}
|
||||||
|
|
||||||
|
call_count = 0
|
||||||
|
|
||||||
|
def mock_make_request(*args, **kwargs):
|
||||||
|
nonlocal call_count
|
||||||
|
call_count += 1
|
||||||
|
return mock_response
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"github_backup.github_backup.make_request_with_retry",
|
||||||
|
side_effect=mock_make_request,
|
||||||
|
):
|
||||||
|
with patch(
|
||||||
|
"github_backup.github_backup.calculate_retry_delay", return_value=0
|
||||||
|
):
|
||||||
|
with pytest.raises(Exception) as exc_info:
|
||||||
|
github_backup.retrieve_data(
|
||||||
|
args, "https://api.github.com/repos/test/repo/issues"
|
||||||
|
)
|
||||||
|
|
||||||
|
assert "Failed to read response after 3 attempts" in str(exc_info.value)
|
||||||
|
assert call_count == 3 # 1 initial + 2 retries = 3 attempts
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
"""Tests for --skip-assets-on flag behavior (issue #135)."""
|
"""Tests for --skip-assets-on flag behavior (issue #135)."""
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from unittest.mock import Mock, patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
from github_backup import github_backup
|
from github_backup import github_backup
|
||||||
|
|
||||||
@@ -13,52 +13,6 @@ class TestSkipAssetsOn:
|
|||||||
while still backing up release metadata.
|
while still backing up release metadata.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def _create_mock_args(self, **overrides):
|
|
||||||
"""Create a mock args object with sensible defaults."""
|
|
||||||
args = Mock()
|
|
||||||
args.user = "testuser"
|
|
||||||
args.output_directory = "/tmp/backup"
|
|
||||||
args.include_repository = False
|
|
||||||
args.include_everything = False
|
|
||||||
args.include_gists = False
|
|
||||||
args.include_starred_gists = False
|
|
||||||
args.all_starred = False
|
|
||||||
args.skip_existing = False
|
|
||||||
args.bare_clone = False
|
|
||||||
args.lfs_clone = False
|
|
||||||
args.no_prune = False
|
|
||||||
args.include_wiki = False
|
|
||||||
args.include_issues = False
|
|
||||||
args.include_issue_comments = False
|
|
||||||
args.include_issue_events = False
|
|
||||||
args.include_pulls = False
|
|
||||||
args.include_pull_comments = False
|
|
||||||
args.include_pull_commits = False
|
|
||||||
args.include_pull_details = False
|
|
||||||
args.include_labels = False
|
|
||||||
args.include_hooks = False
|
|
||||||
args.include_milestones = False
|
|
||||||
args.include_releases = True
|
|
||||||
args.include_assets = True
|
|
||||||
args.skip_assets_on = []
|
|
||||||
args.include_attachments = False
|
|
||||||
args.incremental = False
|
|
||||||
args.incremental_by_files = False
|
|
||||||
args.github_host = None
|
|
||||||
args.prefer_ssh = False
|
|
||||||
args.token_classic = "test-token"
|
|
||||||
args.token_fine = None
|
|
||||||
args.as_app = False
|
|
||||||
args.osx_keychain_item_name = None
|
|
||||||
args.osx_keychain_item_account = None
|
|
||||||
args.skip_prerelease = False
|
|
||||||
args.number_of_latest_releases = None
|
|
||||||
|
|
||||||
for key, value in overrides.items():
|
|
||||||
setattr(args, key, value)
|
|
||||||
|
|
||||||
return args
|
|
||||||
|
|
||||||
def _create_mock_repository(self, name="test-repo", owner="testuser"):
|
def _create_mock_repository(self, name="test-repo", owner="testuser"):
|
||||||
"""Create a mock repository object."""
|
"""Create a mock repository object."""
|
||||||
return {
|
return {
|
||||||
@@ -123,10 +77,10 @@ class TestSkipAssetsOnBehavior(TestSkipAssetsOn):
|
|||||||
@patch("github_backup.github_backup.mkdir_p")
|
@patch("github_backup.github_backup.mkdir_p")
|
||||||
@patch("github_backup.github_backup.json_dump_if_changed")
|
@patch("github_backup.github_backup.json_dump_if_changed")
|
||||||
def test_assets_downloaded_when_not_skipped(
|
def test_assets_downloaded_when_not_skipped(
|
||||||
self, mock_json_dump, mock_mkdir, mock_retrieve, mock_download
|
self, mock_json_dump, mock_mkdir, mock_retrieve, mock_download, create_args
|
||||||
):
|
):
|
||||||
"""Assets should be downloaded when repo is not in skip list."""
|
"""Assets should be downloaded when repo is not in skip list."""
|
||||||
args = self._create_mock_args(skip_assets_on=[])
|
args = create_args(skip_assets_on=[])
|
||||||
repository = self._create_mock_repository(name="normal-repo")
|
repository = self._create_mock_repository(name="normal-repo")
|
||||||
release = self._create_mock_release()
|
release = self._create_mock_release()
|
||||||
asset = self._create_mock_asset()
|
asset = self._create_mock_asset()
|
||||||
@@ -154,10 +108,10 @@ class TestSkipAssetsOnBehavior(TestSkipAssetsOn):
|
|||||||
@patch("github_backup.github_backup.mkdir_p")
|
@patch("github_backup.github_backup.mkdir_p")
|
||||||
@patch("github_backup.github_backup.json_dump_if_changed")
|
@patch("github_backup.github_backup.json_dump_if_changed")
|
||||||
def test_assets_skipped_when_repo_name_matches(
|
def test_assets_skipped_when_repo_name_matches(
|
||||||
self, mock_json_dump, mock_mkdir, mock_retrieve, mock_download
|
self, mock_json_dump, mock_mkdir, mock_retrieve, mock_download, create_args
|
||||||
):
|
):
|
||||||
"""Assets should be skipped when repo name is in skip list."""
|
"""Assets should be skipped when repo name is in skip list."""
|
||||||
args = self._create_mock_args(skip_assets_on=["big-repo"])
|
args = create_args(skip_assets_on=["big-repo"])
|
||||||
repository = self._create_mock_repository(name="big-repo")
|
repository = self._create_mock_repository(name="big-repo")
|
||||||
release = self._create_mock_release()
|
release = self._create_mock_release()
|
||||||
|
|
||||||
@@ -180,10 +134,10 @@ class TestSkipAssetsOnBehavior(TestSkipAssetsOn):
|
|||||||
@patch("github_backup.github_backup.mkdir_p")
|
@patch("github_backup.github_backup.mkdir_p")
|
||||||
@patch("github_backup.github_backup.json_dump_if_changed")
|
@patch("github_backup.github_backup.json_dump_if_changed")
|
||||||
def test_assets_skipped_when_full_name_matches(
|
def test_assets_skipped_when_full_name_matches(
|
||||||
self, mock_json_dump, mock_mkdir, mock_retrieve, mock_download
|
self, mock_json_dump, mock_mkdir, mock_retrieve, mock_download, create_args
|
||||||
):
|
):
|
||||||
"""Assets should be skipped when owner/repo format matches."""
|
"""Assets should be skipped when owner/repo format matches."""
|
||||||
args = self._create_mock_args(skip_assets_on=["otheruser/big-repo"])
|
args = create_args(skip_assets_on=["otheruser/big-repo"])
|
||||||
repository = self._create_mock_repository(name="big-repo", owner="otheruser")
|
repository = self._create_mock_repository(name="big-repo", owner="otheruser")
|
||||||
release = self._create_mock_release()
|
release = self._create_mock_release()
|
||||||
|
|
||||||
@@ -206,11 +160,11 @@ class TestSkipAssetsOnBehavior(TestSkipAssetsOn):
|
|||||||
@patch("github_backup.github_backup.mkdir_p")
|
@patch("github_backup.github_backup.mkdir_p")
|
||||||
@patch("github_backup.github_backup.json_dump_if_changed")
|
@patch("github_backup.github_backup.json_dump_if_changed")
|
||||||
def test_case_insensitive_matching(
|
def test_case_insensitive_matching(
|
||||||
self, mock_json_dump, mock_mkdir, mock_retrieve, mock_download
|
self, mock_json_dump, mock_mkdir, mock_retrieve, mock_download, create_args
|
||||||
):
|
):
|
||||||
"""Skip matching should be case-insensitive."""
|
"""Skip matching should be case-insensitive."""
|
||||||
# User types uppercase, repo name is lowercase
|
# User types uppercase, repo name is lowercase
|
||||||
args = self._create_mock_args(skip_assets_on=["BIG-REPO"])
|
args = create_args(skip_assets_on=["BIG-REPO"])
|
||||||
repository = self._create_mock_repository(name="big-repo")
|
repository = self._create_mock_repository(name="big-repo")
|
||||||
release = self._create_mock_release()
|
release = self._create_mock_release()
|
||||||
|
|
||||||
@@ -233,10 +187,10 @@ class TestSkipAssetsOnBehavior(TestSkipAssetsOn):
|
|||||||
@patch("github_backup.github_backup.mkdir_p")
|
@patch("github_backup.github_backup.mkdir_p")
|
||||||
@patch("github_backup.github_backup.json_dump_if_changed")
|
@patch("github_backup.github_backup.json_dump_if_changed")
|
||||||
def test_multiple_skip_repos(
|
def test_multiple_skip_repos(
|
||||||
self, mock_json_dump, mock_mkdir, mock_retrieve, mock_download
|
self, mock_json_dump, mock_mkdir, mock_retrieve, mock_download, create_args
|
||||||
):
|
):
|
||||||
"""Multiple repos in skip list should all be skipped."""
|
"""Multiple repos in skip list should all be skipped."""
|
||||||
args = self._create_mock_args(skip_assets_on=["repo1", "repo2", "repo3"])
|
args = create_args(skip_assets_on=["repo1", "repo2", "repo3"])
|
||||||
repository = self._create_mock_repository(name="repo2")
|
repository = self._create_mock_repository(name="repo2")
|
||||||
release = self._create_mock_release()
|
release = self._create_mock_release()
|
||||||
|
|
||||||
@@ -259,10 +213,10 @@ class TestSkipAssetsOnBehavior(TestSkipAssetsOn):
|
|||||||
@patch("github_backup.github_backup.mkdir_p")
|
@patch("github_backup.github_backup.mkdir_p")
|
||||||
@patch("github_backup.github_backup.json_dump_if_changed")
|
@patch("github_backup.github_backup.json_dump_if_changed")
|
||||||
def test_release_metadata_still_saved_when_assets_skipped(
|
def test_release_metadata_still_saved_when_assets_skipped(
|
||||||
self, mock_json_dump, mock_mkdir, mock_retrieve, mock_download
|
self, mock_json_dump, mock_mkdir, mock_retrieve, mock_download, create_args
|
||||||
):
|
):
|
||||||
"""Release JSON should still be saved even when assets are skipped."""
|
"""Release JSON should still be saved even when assets are skipped."""
|
||||||
args = self._create_mock_args(skip_assets_on=["big-repo"])
|
args = create_args(skip_assets_on=["big-repo"])
|
||||||
repository = self._create_mock_repository(name="big-repo")
|
repository = self._create_mock_repository(name="big-repo")
|
||||||
release = self._create_mock_release()
|
release = self._create_mock_release()
|
||||||
|
|
||||||
@@ -287,10 +241,10 @@ class TestSkipAssetsOnBehavior(TestSkipAssetsOn):
|
|||||||
@patch("github_backup.github_backup.mkdir_p")
|
@patch("github_backup.github_backup.mkdir_p")
|
||||||
@patch("github_backup.github_backup.json_dump_if_changed")
|
@patch("github_backup.github_backup.json_dump_if_changed")
|
||||||
def test_non_matching_repo_still_downloads_assets(
|
def test_non_matching_repo_still_downloads_assets(
|
||||||
self, mock_json_dump, mock_mkdir, mock_retrieve, mock_download
|
self, mock_json_dump, mock_mkdir, mock_retrieve, mock_download, create_args
|
||||||
):
|
):
|
||||||
"""Repos not in skip list should still download assets."""
|
"""Repos not in skip list should still download assets."""
|
||||||
args = self._create_mock_args(skip_assets_on=["other-repo"])
|
args = create_args(skip_assets_on=["other-repo"])
|
||||||
repository = self._create_mock_repository(name="normal-repo")
|
repository = self._create_mock_repository(name="normal-repo")
|
||||||
release = self._create_mock_release()
|
release = self._create_mock_release()
|
||||||
asset = self._create_mock_asset()
|
asset = self._create_mock_asset()
|
||||||
|
|||||||
@@ -1,39 +1,11 @@
|
|||||||
"""Tests for --starred-skip-size-over flag behavior (issue #108)."""
|
"""Tests for --starred-skip-size-over flag behavior (issue #108)."""
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from unittest.mock import Mock
|
|
||||||
|
|
||||||
from github_backup import github_backup
|
from github_backup import github_backup
|
||||||
|
|
||||||
|
|
||||||
class TestStarredSkipSizeOver:
|
class TestStarredSkipSizeOverArgumentParsing:
|
||||||
"""Test suite for --starred-skip-size-over flag.
|
|
||||||
|
|
||||||
Issue #108: Allow restricting size of starred repositories before cloning.
|
|
||||||
The size is based on the GitHub API's 'size' field (in KB), but the CLI
|
|
||||||
argument accepts MB for user convenience.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def _create_mock_args(self, **overrides):
|
|
||||||
"""Create a mock args object with sensible defaults."""
|
|
||||||
args = Mock()
|
|
||||||
args.user = "testuser"
|
|
||||||
args.repository = None
|
|
||||||
args.name_regex = None
|
|
||||||
args.languages = None
|
|
||||||
args.fork = False
|
|
||||||
args.private = False
|
|
||||||
args.skip_archived = False
|
|
||||||
args.starred_skip_size_over = None
|
|
||||||
args.exclude = None
|
|
||||||
|
|
||||||
for key, value in overrides.items():
|
|
||||||
setattr(args, key, value)
|
|
||||||
|
|
||||||
return args
|
|
||||||
|
|
||||||
|
|
||||||
class TestStarredSkipSizeOverArgumentParsing(TestStarredSkipSizeOver):
|
|
||||||
"""Tests for --starred-skip-size-over argument parsing."""
|
"""Tests for --starred-skip-size-over argument parsing."""
|
||||||
|
|
||||||
def test_starred_skip_size_over_not_set_defaults_to_none(self):
|
def test_starred_skip_size_over_not_set_defaults_to_none(self):
|
||||||
@@ -52,12 +24,17 @@ class TestStarredSkipSizeOverArgumentParsing(TestStarredSkipSizeOver):
|
|||||||
github_backup.parse_args(["testuser", "--starred-skip-size-over", "abc"])
|
github_backup.parse_args(["testuser", "--starred-skip-size-over", "abc"])
|
||||||
|
|
||||||
|
|
||||||
class TestStarredSkipSizeOverFiltering(TestStarredSkipSizeOver):
|
class TestStarredSkipSizeOverFiltering:
|
||||||
"""Tests for --starred-skip-size-over filtering behavior."""
|
"""Tests for --starred-skip-size-over filtering behavior.
|
||||||
|
|
||||||
def test_starred_repo_under_limit_is_kept(self):
|
Issue #108: Allow restricting size of starred repositories before cloning.
|
||||||
|
The size is based on the GitHub API's 'size' field (in KB), but the CLI
|
||||||
|
argument accepts MB for user convenience.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def test_starred_repo_under_limit_is_kept(self, create_args):
|
||||||
"""Starred repos under the size limit should be kept."""
|
"""Starred repos under the size limit should be kept."""
|
||||||
args = self._create_mock_args(starred_skip_size_over=500)
|
args = create_args(starred_skip_size_over=500)
|
||||||
|
|
||||||
repos = [
|
repos = [
|
||||||
{
|
{
|
||||||
@@ -72,9 +49,9 @@ class TestStarredSkipSizeOverFiltering(TestStarredSkipSizeOver):
|
|||||||
assert len(result) == 1
|
assert len(result) == 1
|
||||||
assert result[0]["name"] == "small-repo"
|
assert result[0]["name"] == "small-repo"
|
||||||
|
|
||||||
def test_starred_repo_over_limit_is_filtered(self):
|
def test_starred_repo_over_limit_is_filtered(self, create_args):
|
||||||
"""Starred repos over the size limit should be filtered out."""
|
"""Starred repos over the size limit should be filtered out."""
|
||||||
args = self._create_mock_args(starred_skip_size_over=500)
|
args = create_args(starred_skip_size_over=500)
|
||||||
|
|
||||||
repos = [
|
repos = [
|
||||||
{
|
{
|
||||||
@@ -88,9 +65,9 @@ class TestStarredSkipSizeOverFiltering(TestStarredSkipSizeOver):
|
|||||||
result = github_backup.filter_repositories(args, repos)
|
result = github_backup.filter_repositories(args, repos)
|
||||||
assert len(result) == 0
|
assert len(result) == 0
|
||||||
|
|
||||||
def test_own_repo_over_limit_is_kept(self):
|
def test_own_repo_over_limit_is_kept(self, create_args):
|
||||||
"""User's own repos should not be affected by the size limit."""
|
"""User's own repos should not be affected by the size limit."""
|
||||||
args = self._create_mock_args(starred_skip_size_over=500)
|
args = create_args(starred_skip_size_over=500)
|
||||||
|
|
||||||
repos = [
|
repos = [
|
||||||
{
|
{
|
||||||
@@ -105,9 +82,9 @@ class TestStarredSkipSizeOverFiltering(TestStarredSkipSizeOver):
|
|||||||
assert len(result) == 1
|
assert len(result) == 1
|
||||||
assert result[0]["name"] == "my-huge-repo"
|
assert result[0]["name"] == "my-huge-repo"
|
||||||
|
|
||||||
def test_starred_repo_at_exact_limit_is_kept(self):
|
def test_starred_repo_at_exact_limit_is_kept(self, create_args):
|
||||||
"""Starred repos at exactly the size limit should be kept."""
|
"""Starred repos at exactly the size limit should be kept."""
|
||||||
args = self._create_mock_args(starred_skip_size_over=500)
|
args = create_args(starred_skip_size_over=500)
|
||||||
|
|
||||||
repos = [
|
repos = [
|
||||||
{
|
{
|
||||||
@@ -122,9 +99,9 @@ class TestStarredSkipSizeOverFiltering(TestStarredSkipSizeOver):
|
|||||||
assert len(result) == 1
|
assert len(result) == 1
|
||||||
assert result[0]["name"] == "exact-limit-repo"
|
assert result[0]["name"] == "exact-limit-repo"
|
||||||
|
|
||||||
def test_mixed_repos_filtered_correctly(self):
|
def test_mixed_repos_filtered_correctly(self, create_args):
|
||||||
"""Mix of own and starred repos should be filtered correctly."""
|
"""Mix of own and starred repos should be filtered correctly."""
|
||||||
args = self._create_mock_args(starred_skip_size_over=500)
|
args = create_args(starred_skip_size_over=500)
|
||||||
|
|
||||||
repos = [
|
repos = [
|
||||||
{
|
{
|
||||||
@@ -153,9 +130,9 @@ class TestStarredSkipSizeOverFiltering(TestStarredSkipSizeOver):
|
|||||||
assert "starred-small" in names
|
assert "starred-small" in names
|
||||||
assert "starred-huge" not in names
|
assert "starred-huge" not in names
|
||||||
|
|
||||||
def test_no_size_limit_keeps_all_starred(self):
|
def test_no_size_limit_keeps_all_starred(self, create_args):
|
||||||
"""When no size limit is set, all starred repos should be kept."""
|
"""When no size limit is set, all starred repos should be kept."""
|
||||||
args = self._create_mock_args(starred_skip_size_over=None)
|
args = create_args(starred_skip_size_over=None)
|
||||||
|
|
||||||
repos = [
|
repos = [
|
||||||
{
|
{
|
||||||
@@ -169,9 +146,9 @@ class TestStarredSkipSizeOverFiltering(TestStarredSkipSizeOver):
|
|||||||
result = github_backup.filter_repositories(args, repos)
|
result = github_backup.filter_repositories(args, repos)
|
||||||
assert len(result) == 1
|
assert len(result) == 1
|
||||||
|
|
||||||
def test_repo_without_size_field_is_kept(self):
|
def test_repo_without_size_field_is_kept(self, create_args):
|
||||||
"""Repos without a size field should be kept (size defaults to 0)."""
|
"""Repos without a size field should be kept (size defaults to 0)."""
|
||||||
args = self._create_mock_args(starred_skip_size_over=500)
|
args = create_args(starred_skip_size_over=500)
|
||||||
|
|
||||||
repos = [
|
repos = [
|
||||||
{
|
{
|
||||||
@@ -185,9 +162,9 @@ class TestStarredSkipSizeOverFiltering(TestStarredSkipSizeOver):
|
|||||||
result = github_backup.filter_repositories(args, repos)
|
result = github_backup.filter_repositories(args, repos)
|
||||||
assert len(result) == 1
|
assert len(result) == 1
|
||||||
|
|
||||||
def test_zero_value_warns_and_is_ignored(self, caplog):
|
def test_zero_value_warns_and_is_ignored(self, create_args, caplog):
|
||||||
"""Zero value should warn and keep all repos."""
|
"""Zero value should warn and keep all repos."""
|
||||||
args = self._create_mock_args(starred_skip_size_over=0)
|
args = create_args(starred_skip_size_over=0)
|
||||||
|
|
||||||
repos = [
|
repos = [
|
||||||
{
|
{
|
||||||
@@ -202,9 +179,9 @@ class TestStarredSkipSizeOverFiltering(TestStarredSkipSizeOver):
|
|||||||
assert len(result) == 1
|
assert len(result) == 1
|
||||||
assert "must be greater than 0" in caplog.text
|
assert "must be greater than 0" in caplog.text
|
||||||
|
|
||||||
def test_negative_value_warns_and_is_ignored(self, caplog):
|
def test_negative_value_warns_and_is_ignored(self, create_args, caplog):
|
||||||
"""Negative value should warn and keep all repos."""
|
"""Negative value should warn and keep all repos."""
|
||||||
args = self._create_mock_args(starred_skip_size_over=-5)
|
args = create_args(starred_skip_size_over=-5)
|
||||||
|
|
||||||
repos = [
|
repos = [
|
||||||
{
|
{
|
||||||
|
|||||||
Reference in New Issue
Block a user