From eb88def888df288d9427e28ad9ec5778180753b7 Mon Sep 17 00:00:00 2001 From: BrOleg5 Date: Tue, 2 Jan 2024 12:54:06 +0800 Subject: [PATCH 1/4] Add option to include certain number of the latest releases. --- README.rst | 6 +++++- github_backup/github_backup.py | 16 +++++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 22d3062..0944536 100644 --- a/README.rst +++ b/README.rst @@ -49,7 +49,8 @@ CLI Help output:: [-P] [-F] [--prefer-ssh] [-v] [--keychain-name OSX_KEYCHAIN_ITEM_NAME] [--keychain-account OSX_KEYCHAIN_ITEM_ACCOUNT] - [--releases] [--assets] [--exclude [REPOSITORY [REPOSITORY ...]] + [--releases] [--assets] [--latest-releases INCLUDE_LATEST_RELEASES] + [--exclude [REPOSITORY [REPOSITORY ...]] [--throttle-limit THROTTLE_LIMIT] [--throttle-pause THROTTLE_PAUSE] USER @@ -126,6 +127,9 @@ CLI Help output:: binaries --assets include assets alongside release information; only applies if including releases + --latest-releases INCLUDE_LATEST_RELEASES + include certain number of the latest releases; only + applies if including releases --exclude [REPOSITORY [REPOSITORY ...]] names of repositories to exclude from backup. --throttle-limit THROTTLE_LIMIT diff --git a/github_backup/github_backup.py b/github_backup/github_backup.py index 860b73a..27c357b 100644 --- a/github_backup/github_backup.py +++ b/github_backup/github_backup.py @@ -23,6 +23,7 @@ from urllib.error import HTTPError, URLError from urllib.parse import quote as urlquote from urllib.parse import urlencode, urlparse from urllib.request import HTTPRedirectHandler, Request, build_opener, urlopen +from operator import itemgetter try: from . import __version__ @@ -382,6 +383,13 @@ def parse_args(args=None): dest="include_assets", help="include assets alongside release information; only applies if including releases", ) + parser.add_argument( + "--latest-releases", + type=int, + default=0, + dest="include_latest_releases", + help="include certain number of the latest releases; only applies if including releases", + ) parser.add_argument( "--throttle-limit", dest="throttle_limit", @@ -1206,8 +1214,14 @@ def backup_releases(args, repo_cwd, repository, repos_template, include_assets=F release_template = "{0}/{1}/releases".format(repos_template, repository_fullname) releases = retrieve_data(args, release_template, query_args=query_args) + if args.include_latest_releases and args.include_latest_releases < len(releases): + releases = sorted(releases, key=itemgetter('tag_name'), reverse=True) + releases = releases[:args.include_latest_releases] + logger.info("Saving the latest {0} releases to disk".format(len(releases))) + else: + logger.info("Saving {0} releases to disk".format(len(releases))) + # for each release, store it - logger.info("Saving {0} releases to disk".format(len(releases))) for release in releases: release_name = release["tag_name"] release_name_safe = release_name.replace("/", "__") From ab18e96ea802e9b0aafa5dace54eb72b26ca3387 Mon Sep 17 00:00:00 2001 From: BrOleg5 Date: Fri, 5 Jan 2024 20:26:08 +0800 Subject: [PATCH 2/4] Add option to skip prerelease versions. Replace release sorting by tag with release sorting by creation date. --- README.rst | 7 ++++--- github_backup/github_backup.py | 26 ++++++++++++++++++-------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/README.rst b/README.rst index 0944536..22c39d7 100644 --- a/README.rst +++ b/README.rst @@ -125,10 +125,11 @@ CLI Help output:: keychain that holds the personal access or OAuth token --releases include release information, not including assets or binaries - --assets include assets alongside release information; only - applies if including releases --latest-releases INCLUDE_LATEST_RELEASES - include certain number of the latest releases; only + include certain number of the latest releases; + only applies if including releases + --skip-prerelease skip prerelease and draft versions; only applies if including releases + --assets include assets alongside release information; only applies if including releases --exclude [REPOSITORY [REPOSITORY ...]] names of repositories to exclude from backup. diff --git a/github_backup/github_backup.py b/github_backup/github_backup.py index 27c357b..2761336 100644 --- a/github_backup/github_backup.py +++ b/github_backup/github_backup.py @@ -23,7 +23,7 @@ from urllib.error import HTTPError, URLError from urllib.parse import quote as urlquote from urllib.parse import urlencode, urlparse from urllib.request import HTTPRedirectHandler, Request, build_opener, urlopen -from operator import itemgetter +from datetime import datetime try: from . import __version__ @@ -377,12 +377,6 @@ def parse_args(args=None): dest="include_releases", help="include release information, not including assets or binaries", ) - parser.add_argument( - "--assets", - action="store_true", - dest="include_assets", - help="include assets alongside release information; only applies if including releases", - ) parser.add_argument( "--latest-releases", type=int, @@ -390,6 +384,18 @@ def parse_args(args=None): dest="include_latest_releases", help="include certain number of the latest releases; only applies if including releases", ) + parser.add_argument( + "--skip-prerelease", + action="store_true", + dest="skip_prerelease", + help="skip prerelease and draft versions; only applies if including releases", + ) + parser.add_argument( + "--assets", + action="store_true", + dest="include_assets", + help="include assets alongside release information; only applies if including releases", + ) parser.add_argument( "--throttle-limit", dest="throttle_limit", @@ -1214,8 +1220,12 @@ def backup_releases(args, repo_cwd, repository, repos_template, include_assets=F release_template = "{0}/{1}/releases".format(repos_template, repository_fullname) releases = retrieve_data(args, release_template, query_args=query_args) + if args.skip_prerelease: + releases = [r for r in releases if r["prerelease"] == False and r["draft"] == False] + if args.include_latest_releases and args.include_latest_releases < len(releases): - releases = sorted(releases, key=itemgetter('tag_name'), reverse=True) + releases.sort(key=lambda item: datetime.strptime(item["created_at"], "%Y-%m-%dT%H:%M:%SZ"), \ + reverse=True) releases = releases[:args.include_latest_releases] logger.info("Saving the latest {0} releases to disk".format(len(releases))) else: From 3e9a4fa0d885ae0e348df62d40f8c36c16b4076a Mon Sep 17 00:00:00 2001 From: BrOleg5 Date: Fri, 5 Jan 2024 20:35:37 +0800 Subject: [PATCH 3/4] Update CLI help output in README. Rename destination of latest-releases argument. --- README.rst | 5 +++-- github_backup/github_backup.py | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 22c39d7..82dc479 100644 --- a/README.rst +++ b/README.rst @@ -49,7 +49,8 @@ CLI Help output:: [-P] [-F] [--prefer-ssh] [-v] [--keychain-name OSX_KEYCHAIN_ITEM_NAME] [--keychain-account OSX_KEYCHAIN_ITEM_ACCOUNT] - [--releases] [--assets] [--latest-releases INCLUDE_LATEST_RELEASES] + [--releases] [--latest-releases NUMBER_OF_LATEST_RELEASES] + [--skip-prerelease] [--assets] [--exclude [REPOSITORY [REPOSITORY ...]] [--throttle-limit THROTTLE_LIMIT] [--throttle-pause THROTTLE_PAUSE] USER @@ -125,7 +126,7 @@ CLI Help output:: keychain that holds the personal access or OAuth token --releases include release information, not including assets or binaries - --latest-releases INCLUDE_LATEST_RELEASES + --latest-releases NUMBER_OF_LATEST_RELEASES include certain number of the latest releases; only applies if including releases --skip-prerelease skip prerelease and draft versions; only applies if including releases diff --git a/github_backup/github_backup.py b/github_backup/github_backup.py index 2761336..13fef00 100644 --- a/github_backup/github_backup.py +++ b/github_backup/github_backup.py @@ -381,7 +381,7 @@ def parse_args(args=None): "--latest-releases", type=int, default=0, - dest="include_latest_releases", + dest="number_of_latest_releases", help="include certain number of the latest releases; only applies if including releases", ) parser.add_argument( @@ -1223,10 +1223,10 @@ def backup_releases(args, repo_cwd, repository, repos_template, include_assets=F if args.skip_prerelease: releases = [r for r in releases if r["prerelease"] == False and r["draft"] == False] - if args.include_latest_releases and args.include_latest_releases < len(releases): + if args.number_of_latest_releases and args.number_of_latest_releases < len(releases): releases.sort(key=lambda item: datetime.strptime(item["created_at"], "%Y-%m-%dT%H:%M:%SZ"), \ reverse=True) - releases = releases[:args.include_latest_releases] + releases = releases[:args.number_of_latest_releases] logger.info("Saving the latest {0} releases to disk".format(len(releases))) else: logger.info("Saving {0} releases to disk".format(len(releases))) From 09f4168db67c90dd9093b8c799ac95d8881a1345 Mon Sep 17 00:00:00 2001 From: BrOleg5 Date: Fri, 5 Jan 2024 20:46:02 +0800 Subject: [PATCH 4/4] Fix code style. --- github_backup/github_backup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/github_backup/github_backup.py b/github_backup/github_backup.py index 13fef00..2690b49 100644 --- a/github_backup/github_backup.py +++ b/github_backup/github_backup.py @@ -511,7 +511,7 @@ def get_github_host(args): def read_file_contents(file_uri): - return open(file_uri[len(FILE_URI_PREFIX) :], "rt").readline().strip() + return open(file_uri[len(FILE_URI_PREFIX):], "rt").readline().strip() def get_github_repo_url(args, repository): @@ -1221,10 +1221,10 @@ def backup_releases(args, repo_cwd, repository, repos_template, include_assets=F releases = retrieve_data(args, release_template, query_args=query_args) if args.skip_prerelease: - releases = [r for r in releases if r["prerelease"] == False and r["draft"] == False] + releases = [r for r in releases if not r["prerelease"] and not r["draft"]] if args.number_of_latest_releases and args.number_of_latest_releases < len(releases): - releases.sort(key=lambda item: datetime.strptime(item["created_at"], "%Y-%m-%dT%H:%M:%SZ"), \ + releases.sort(key=lambda item: datetime.strptime(item["created_at"], "%Y-%m-%dT%H:%M:%SZ"), reverse=True) releases = releases[:args.number_of_latest_releases] logger.info("Saving the latest {0} releases to disk".format(len(releases)))