From 60cb484a19448542b7cf7c523cd01dac56f30074 Mon Sep 17 00:00:00 2001 From: froggleston Date: Wed, 22 Mar 2023 14:53:07 +0000 Subject: [PATCH 1/6] Add support for fine-grained tokens --- github_backup/github_backup.py | 59 ++++++++++++++++++++++------------ 1 file changed, 39 insertions(+), 20 deletions(-) diff --git a/github_backup/github_backup.py b/github_backup/github_backup.py index 738831b..a024cc5 100644 --- a/github_backup/github_backup.py +++ b/github_backup/github_backup.py @@ -150,9 +150,13 @@ def parse_args(args=None): 'If a username is given but not a password, the ' 'password will be prompted for.') parser.add_argument('-t', - '--token', - dest='token', + '--token-classic', + dest='token_classic', help='personal access, OAuth, or JSON Web token, or path to token (file://...)') # noqa + parser.add_argument('-f', + '--token-fine', + dest='token_fine', + help='fine-grained personal access token (github_pat_....)') # noqa parser.add_argument('--as-app', action='store_true', dest='as_app', @@ -357,18 +361,23 @@ def get_auth(args, encode=True, for_git_cli=False): raise Exception('No password item matching the provided name and account could be found in the osx keychain.') elif args.osx_keychain_item_account: raise Exception('You must specify both name and account fields for osx keychain password items') - elif args.token: + elif args.token_fine: + if args.token_fine.startswith("github_pat_"): + auth = args.token_fine + else: + raise Exception("Fine-grained token supplied does not look like a GitHub PAT") + elif args.token_classic: _path_specifier = 'file://' - if args.token.startswith(_path_specifier): - args.token = open(args.token[len(_path_specifier):], - 'rt').readline().strip() + if args.token_classic.startswith(_path_specifier): + args.token_classic = open(args.token_classic[len(_path_specifier):], + 'rt').readline().strip() if not args.as_app: - auth = args.token + ':' + 'x-oauth-basic' + auth = args.token_classic + ':' + 'x-oauth-basic' else: if not for_git_cli: - auth = args.token + auth = args.token_classic else: - auth = 'x-access-token:' + args.token + auth = 'x-access-token:' + args.token_classic elif args.username: if not args.password: args.password = getpass.getpass() @@ -383,7 +392,7 @@ def get_auth(args, encode=True, for_git_cli=False): if not auth: return None - if not encode: + if not encode or args.token_fine is not None: return auth return base64.b64encode(auth.encode('ascii')) @@ -421,12 +430,19 @@ def get_github_repo_url(args, repository): return repository['ssh_url'] auth = get_auth(args, encode=False, for_git_cli=True) - if auth: - repo_url = 'https://{0}@{1}/{2}/{3}.git'.format( - auth, - get_github_host(args), - repository['owner']['login'], - repository['name']) + if auth: + if args.token_fine is None: + repo_url = 'https://{0}@{1}/{2}/{3}.git'.format( + auth, + get_github_host(args), + repository['owner']['login'], + repository['name']) + else: + repo_url = 'https://{0}@{1}/{2}/{3}.git'.format( + "oauth2:"+auth, + get_github_host(args), + repository['owner']['login'], + repository['name']) else: repo_url = repository['clone_url'] @@ -441,7 +457,7 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False): while True: page = page + 1 - request = _construct_request(per_page, page, query_args, template, auth, as_app=args.as_app) # noqa + request = _construct_request(per_page, page, query_args, template, auth, as_app=args.as_app, fine=True if args.token_fine is not None else False) # noqa r, errors = _get_response(request, auth, template) status_code = int(r.getcode()) @@ -474,7 +490,7 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False): log_warning('API request failed. Retrying in 5 seconds') retries += 1 time.sleep(5) - request = _construct_request(per_page, page, query_args, template, auth, as_app=args.as_app) # noqa + request = _construct_request(per_page, page, query_args, template, auth, as_app=args.as_app, fine=True if args.token_fine is not None else False) # noqa r, errors = _get_response(request, auth, template) status_code = int(r.getcode()) @@ -557,7 +573,7 @@ def _get_response(request, auth, template): return r, errors -def _construct_request(per_page, page, query_args, template, auth, as_app=None): +def _construct_request(per_page, page, query_args, template, auth, as_app=None, fine=False): querystring = urlencode(dict(list({ 'per_page': per_page, 'page': page @@ -566,7 +582,10 @@ def _construct_request(per_page, page, query_args, template, auth, as_app=None): request = Request(template + '?' + querystring) if auth is not None: if not as_app: - request.add_header('Authorization', 'Basic '.encode('ascii') + auth) + if fine: + request.add_header('Authorization', 'token ' + auth) + else: + request.add_header('Authorization', 'Basic '.encode('ascii') + auth) else: auth = auth.encode('ascii') request.add_header('Authorization', 'token '.encode('ascii') + auth) From 61275c61b246452dc2867b9ee83aa9da2c746530 Mon Sep 17 00:00:00 2001 From: Robert Davey Date: Tue, 28 Mar 2023 16:52:48 +0100 Subject: [PATCH 2/6] Update README.rst Add flags and example for fine-grained tokens --- README.rst | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 689c0b6..aadd645 100644 --- a/README.rst +++ b/README.rst @@ -29,7 +29,8 @@ Usage CLI Usage is as follows:: - github-backup [-h] [-u USERNAME] [-p PASSWORD] [-t TOKEN] [--as-app] + github-backup [-h] [-u USERNAME] [-p PASSWORD] + [-t TOKEN_CLASSIC] [-f TOKEN_FINE] [--as-app] [-o OUTPUT_DIRECTORY] [-i] [--starred] [--all-starred] [--watched] [--followers] [--following] [--all] [--issues] [--issue-comments] [--issue-events] [--pulls] @@ -57,7 +58,9 @@ CLI Usage is as follows:: -p PASSWORD, --password PASSWORD password for basic auth. If a username is given but not a password, the password will be prompted for. - -t TOKEN, --token TOKEN + -f TOKEN_FINE, --token-fine TOKEN + fine-grained personal access token + -t TOKEN_CLASSIC, --token-classic TOKEN personal access, OAuth, or JSON Web token, or path to token (file://...) --as-app authenticate as github app instead of as a user. @@ -160,13 +163,13 @@ Backup all repositories, including private ones:: export ACCESS_TOKEN=SOME-GITHUB-TOKEN github-backup WhiteHouse --token $ACCESS_TOKEN --organization --output-directory /tmp/white-house --repositories --private -Backup a single organization repository with everything else (wiki, pull requests, comments, issues etc):: +Use a fine-grained access token to backup a single organization repository with everything else (wiki, pull requests, comments, issues etc):: export ACCESS_TOKEN=SOME-GITHUB-TOKEN ORGANIZATION=docker REPO=cli # e.g. git@github.com:docker/cli.git - github-backup $ORGANIZATION -P -t $ACCESS_TOKEN -o . --all -O -R $REPO + github-backup $ORGANIZATION -P -f $ACCESS_TOKEN -o . --all -O -R $REPO Testing ======= From f12b8775093630b1acc088b349d8dcaca1955d4c Mon Sep 17 00:00:00 2001 From: Halvor Holsten Strand Date: Fri, 29 Sep 2023 14:01:53 +0200 Subject: [PATCH 3/6] Keep backwards compatability by going back to "--token" for classic. Allow "file://" uri for "--token-fine". --- README.rst | 7 ++++--- github_backup/github_backup.py | 8 +++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index aadd645..7e9e592 100644 --- a/README.rst +++ b/README.rst @@ -58,9 +58,10 @@ CLI Usage is as follows:: -p PASSWORD, --password PASSWORD password for basic auth. If a username is given but not a password, the password will be prompted for. - -f TOKEN_FINE, --token-fine TOKEN - fine-grained personal access token - -t TOKEN_CLASSIC, --token-classic TOKEN + -f TOKEN_FINE, --token-fine TOKEN_FINE + fine-grained personal access token or path to token + (file://...) + -t TOKEN_CLASSIC, --token TOKEN_CLASSIC personal access, OAuth, or JSON Web token, or path to token (file://...) --as-app authenticate as github app instead of as a user. diff --git a/github_backup/github_backup.py b/github_backup/github_backup.py index a024cc5..bc1387b 100644 --- a/github_backup/github_backup.py +++ b/github_backup/github_backup.py @@ -150,7 +150,7 @@ def parse_args(args=None): 'If a username is given but not a password, the ' 'password will be prompted for.') parser.add_argument('-t', - '--token-classic', + '--token', dest='token_classic', help='personal access, OAuth, or JSON Web token, or path to token (file://...)') # noqa parser.add_argument('-f', @@ -362,6 +362,11 @@ def get_auth(args, encode=True, for_git_cli=False): elif args.osx_keychain_item_account: raise Exception('You must specify both name and account fields for osx keychain password items') elif args.token_fine: + _path_specifier = 'file://' + if args.token_fine.startswith(_path_specifier): + args.token_fine = open(args.token_fine[len(_path_specifier):], + 'rt').readline().strip() + if args.token_fine.startswith("github_pat_"): auth = args.token_fine else: @@ -371,6 +376,7 @@ def get_auth(args, encode=True, for_git_cli=False): if args.token_classic.startswith(_path_specifier): args.token_classic = open(args.token_classic[len(_path_specifier):], 'rt').readline().strip() + if not args.as_app: auth = args.token_classic + ':' + 'x-oauth-basic' else: From a9d35c0fd52659ead22446bffe22567784444ac5 Mon Sep 17 00:00:00 2001 From: Halvor Holsten Strand Date: Fri, 29 Sep 2023 14:40:16 +0200 Subject: [PATCH 4/6] Ran black. --- github_backup/github_backup.py | 51 ++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/github_backup/github_backup.py b/github_backup/github_backup.py index 7337783..ec03d3f 100644 --- a/github_backup/github_backup.py +++ b/github_backup/github_backup.py @@ -127,13 +127,13 @@ def parse_args(args=None): "-t", "--token", dest="token_classic", - help="personal access, OAuth, or JSON Web token, or path to token (file://...)" + help="personal access, OAuth, or JSON Web token, or path to token (file://...)", ) # noqa parser.add_argument( "-f", - '--token-fine', + "--token-fine", dest="token_fine", - help="fine-grained personal access token (github_pat_....), or path to token (file://...)" + help="fine-grained personal access token (github_pat_....), or path to token (file://...)", ) # noqa parser.add_argument( "--as-app", @@ -436,22 +436,28 @@ def get_auth(args, encode=True, for_git_cli=False): elif args.osx_keychain_item_account: raise Exception( "You must specify both name and account fields for osx keychain password items" - ) + ) elif args.token_fine: _path_specifier = "file://" if args.token_fine.startswith(_path_specifier): - args.token_fine = open(args.token_fine[len(_path_specifier):], - "rt").readline().strip() + args.token_fine = ( + open(args.token_fine[len(_path_specifier) :], "rt").readline().strip() + ) if args.token_fine.startswith("github_pat_"): auth = args.token_fine else: - raise Exception("Fine-grained token supplied does not look like a GitHub PAT") + raise Exception( + "Fine-grained token supplied does not look like a GitHub PAT" + ) elif args.token_classic: _path_specifier = "file://" if args.token_classic.startswith(_path_specifier): - args.token_classic = open(args.token_classic[len(_path_specifier):], - "rt").readline().strip() + args.token_classic = ( + open(args.token_classic[len(_path_specifier) :], "rt") + .readline() + .strip() + ) if not args.as_app: auth = args.token_classic + ":" + "x-oauth-basic" @@ -518,7 +524,7 @@ def get_github_repo_url(args, repository): return repository["ssh_url"] auth = get_auth(args, encode=False, for_git_cli=True) - if auth: + if auth: if args.token_fine is None: repo_url = "https://{0}@{1}/{2}/{3}.git".format( auth, @@ -528,7 +534,7 @@ def get_github_repo_url(args, repository): ) else: repo_url = "https://{0}@{1}/{2}/{3}.git".format( - "oauth2:"+auth, + "oauth2:" + auth, get_github_host(args), repository["owner"]["login"], repository["name"], @@ -548,7 +554,13 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False): while True: page = page + 1 request = _construct_request( - per_page, page, query_args, template, auth, as_app=args.as_app, fine=True if args.token_fine is not None else False + per_page, + page, + query_args, + template, + auth, + as_app=args.as_app, + fine=True if args.token_fine is not None else False, ) # noqa r, errors = _get_response(request, auth, template) @@ -584,7 +596,13 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False): retries += 1 time.sleep(5) request = _construct_request( - per_page, page, query_args, template, auth, as_app=args.as_app, fine=True if args.token_fine is not None else False + per_page, + page, + query_args, + template, + auth, + as_app=args.as_app, + fine=True if args.token_fine is not None else False, ) # noqa r, errors = _get_response(request, auth, template) @@ -668,10 +686,13 @@ def _get_response(request, auth, template): return r, errors -def _construct_request(per_page, page, query_args, template, auth, as_app=None, fine=False): +def _construct_request( + per_page, page, query_args, template, auth, as_app=None, fine=False +): querystring = urlencode( dict( - list({"per_page": per_page, "page": page}.items()) + list(query_args.items()) + list({"per_page": per_page, "page": page}.items()) + + list(query_args.items()) ) ) From 15de769d674647afe0aecf4cdd0b021b40533f03 Mon Sep 17 00:00:00 2001 From: Halvor Holsten Strand Date: Sun, 1 Oct 2023 22:22:15 +0200 Subject: [PATCH 5/6] Simplified one if/elif scenario. Extracted file reading of another if/elif scenario. --- github_backup/github_backup.py | 41 +++++++++++++--------------------- 1 file changed, 15 insertions(+), 26 deletions(-) diff --git a/github_backup/github_backup.py b/github_backup/github_backup.py index ec03d3f..b05d1fb 100644 --- a/github_backup/github_backup.py +++ b/github_backup/github_backup.py @@ -36,6 +36,7 @@ except ImportError: VERSION = "unknown" FNULL = open(os.devnull, "w") +FILE_URI_PREFIX = "file://" logger = logging.getLogger(__name__) @@ -438,11 +439,8 @@ def get_auth(args, encode=True, for_git_cli=False): "You must specify both name and account fields for osx keychain password items" ) elif args.token_fine: - _path_specifier = "file://" - if args.token_fine.startswith(_path_specifier): - args.token_fine = ( - open(args.token_fine[len(_path_specifier) :], "rt").readline().strip() - ) + if args.token_fine.startswith(FILE_URI_PREFIX): + args.token_fine = read_file_contents(args.token_fine) if args.token_fine.startswith("github_pat_"): auth = args.token_fine @@ -451,13 +449,8 @@ def get_auth(args, encode=True, for_git_cli=False): "Fine-grained token supplied does not look like a GitHub PAT" ) elif args.token_classic: - _path_specifier = "file://" - if args.token_classic.startswith(_path_specifier): - args.token_classic = ( - open(args.token_classic[len(_path_specifier) :], "rt") - .readline() - .strip() - ) + if args.token_classic.startswith(FILE_URI_PREFIX): + args.token_classic = read_file_contents(args.token_classic) if not args.as_app: auth = args.token_classic + ":" + "x-oauth-basic" @@ -504,6 +497,10 @@ def get_github_host(args): return host +def read_file_contents(file_uri): + result = open(file_uri[len(FILE_URI_PREFIX) :], "rt").readline().strip() + + def get_github_repo_url(args, repository): if repository.get("is_gist"): if args.prefer_ssh: @@ -525,20 +522,12 @@ def get_github_repo_url(args, repository): auth = get_auth(args, encode=False, for_git_cli=True) if auth: - if args.token_fine is None: - repo_url = "https://{0}@{1}/{2}/{3}.git".format( - auth, - get_github_host(args), - repository["owner"]["login"], - repository["name"], - ) - else: - repo_url = "https://{0}@{1}/{2}/{3}.git".format( - "oauth2:" + auth, - get_github_host(args), - repository["owner"]["login"], - repository["name"], - ) + repo_url = "https://{0}@{1}/{2}/{3}.git".format( + auth if args.token_fine is None else "oauth2:" + auth, + get_github_host(args), + repository["owner"]["login"], + repository["name"], + ) else: repo_url = repository["clone_url"] From b277baa6ea50144e80841c52b7812ccbf90a4fd5 Mon Sep 17 00:00:00 2001 From: Halvor Holsten Strand Date: Mon, 2 Oct 2023 09:14:40 +0200 Subject: [PATCH 6/6] Update github_backup.py --- github_backup/github_backup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/github_backup/github_backup.py b/github_backup/github_backup.py index b05d1fb..bc42a20 100644 --- a/github_backup/github_backup.py +++ b/github_backup/github_backup.py @@ -498,7 +498,7 @@ def get_github_host(args): def read_file_contents(file_uri): - result = open(file_uri[len(FILE_URI_PREFIX) :], "rt").readline().strip() + return open(file_uri[len(FILE_URI_PREFIX) :], "rt").readline().strip() def get_github_repo_url(args, repository):