diff --git a/README.rst b/README.rst index f7ce874..2e4dfa4 100644 --- a/README.rst +++ b/README.rst @@ -29,16 +29,17 @@ Usage CLI Usage is as follows:: - github-backup [-h] [-u USERNAME] [-p PASSWORD] [-t TOKEN] [--as-app] - [-o OUTPUT_DIRECTORY] [-l LOG_LEVEL] [-i] [--starred] - [--all-starred] [--watched] [--followers] [--following] - [--all] [--issues] [--issue-comments] [--issue-events] - [--pulls] [--pull-comments] [--pull-commits] - [--pull-details] [--labels] [--hooks] [--milestones] - [--repositories] [--bare] [--lfs] [--wikis] [--gists] - [--starred-gists] [--skip-archived] [--skip-existing] - [-L [LANGUAGES ...]] [-N NAME_REGEX] [-H GITHUB_HOST] - [-O] [-R REPOSITORY] [-P] [-F] [--prefer-ssh] [-v] + github-backup [-h] [-u USERNAME] [-p PASSWORD] [-t TOKEN_CLASSIC] + [-f TOKEN_FINE] [--as-app] [-o OUTPUT_DIRECTORY] + [-l LOG_LEVEL] [-i] [--starred] [--all-starred] + [--watched] [--followers] [--following] [--all] [--issues] + [--issue-comments] [--issue-events] [--pulls] + [--pull-comments] [--pull-commits] [--pull-details] + [--labels] [--hooks] [--milestones] [--repositories] + [--bare] [--lfs] [--wikis] [--gists] [--starred-gists] + [--skip-archived] [--skip-existing] [-L [LANGUAGES ...]] + [-N NAME_REGEX] [-H GITHUB_HOST] [-O] [-R REPOSITORY] + [-P] [-F] [--prefer-ssh] [-v] [--keychain-name OSX_KEYCHAIN_ITEM_NAME] [--keychain-account OSX_KEYCHAIN_ITEM_ACCOUNT] [--releases] [--assets] [--exclude [REPOSITORY [REPOSITORY ...]] @@ -57,7 +58,10 @@ CLI Usage is as follows:: -p PASSWORD, --password PASSWORD password for basic auth. If a username is given but not a password, the password will be prompted for. - -t TOKEN, --token TOKEN + -f TOKEN_FINE, --token-fine TOKEN_FINE + fine-grained personal access token or path to token + (file://...) + -t TOKEN_CLASSIC, --token TOKEN_CLASSIC personal access, OAuth, or JSON Web token, or path to token (file://...) --as-app authenticate as github app instead of as a user. @@ -163,13 +167,13 @@ Backup all repositories, including private ones:: export ACCESS_TOKEN=SOME-GITHUB-TOKEN github-backup WhiteHouse --token $ACCESS_TOKEN --organization --output-directory /tmp/white-house --repositories --private -Backup a single organization repository with everything else (wiki, pull requests, comments, issues etc):: +Use a fine-grained access token to backup a single organization repository with everything else (wiki, pull requests, comments, issues etc):: export ACCESS_TOKEN=SOME-GITHUB-TOKEN ORGANIZATION=docker REPO=cli # e.g. git@github.com:docker/cli.git - github-backup $ORGANIZATION -P -t $ACCESS_TOKEN -o . --all -O -R $REPO + github-backup $ORGANIZATION -P -f $ACCESS_TOKEN -o . --all -O -R $REPO Testing ======= diff --git a/github_backup/github_backup.py b/github_backup/github_backup.py index 4558504..bc42a20 100644 --- a/github_backup/github_backup.py +++ b/github_backup/github_backup.py @@ -36,6 +36,7 @@ except ImportError: VERSION = "unknown" FNULL = open(os.devnull, "w") +FILE_URI_PREFIX = "file://" logger = logging.getLogger(__name__) @@ -126,9 +127,15 @@ def parse_args(args=None): parser.add_argument( "-t", "--token", - dest="token", + dest="token_classic", help="personal access, OAuth, or JSON Web token, or path to token (file://...)", ) # noqa + parser.add_argument( + "-f", + "--token-fine", + dest="token_fine", + help="fine-grained personal access token (github_pat_....), or path to token (file://...)", + ) # noqa parser.add_argument( "--as-app", action="store_true", @@ -431,18 +438,27 @@ def get_auth(args, encode=True, for_git_cli=False): raise Exception( "You must specify both name and account fields for osx keychain password items" ) - elif args.token: - _path_specifier = "file://" - if args.token.startswith(_path_specifier): - path_specifier_len = len(_path_specifier) - args.token = open(args.token[path_specifier_len:], "rt").readline().strip() + elif args.token_fine: + if args.token_fine.startswith(FILE_URI_PREFIX): + args.token_fine = read_file_contents(args.token_fine) + + if args.token_fine.startswith("github_pat_"): + auth = args.token_fine + else: + raise Exception( + "Fine-grained token supplied does not look like a GitHub PAT" + ) + elif args.token_classic: + if args.token_classic.startswith(FILE_URI_PREFIX): + args.token_classic = read_file_contents(args.token_classic) + if not args.as_app: - auth = args.token + ":" + "x-oauth-basic" + auth = args.token_classic + ":" + "x-oauth-basic" else: if not for_git_cli: - auth = args.token + auth = args.token_classic else: - auth = "x-access-token:" + args.token + auth = "x-access-token:" + args.token_classic elif args.username: if not args.password: args.password = getpass.getpass() @@ -457,7 +473,7 @@ def get_auth(args, encode=True, for_git_cli=False): if not auth: return None - if not encode: + if not encode or args.token_fine is not None: return auth return base64.b64encode(auth.encode("ascii")) @@ -481,6 +497,10 @@ def get_github_host(args): return host +def read_file_contents(file_uri): + return open(file_uri[len(FILE_URI_PREFIX) :], "rt").readline().strip() + + def get_github_repo_url(args, repository): if repository.get("is_gist"): if args.prefer_ssh: @@ -503,7 +523,7 @@ def get_github_repo_url(args, repository): auth = get_auth(args, encode=False, for_git_cli=True) if auth: repo_url = "https://{0}@{1}/{2}/{3}.git".format( - auth, + auth if args.token_fine is None else "oauth2:" + auth, get_github_host(args), repository["owner"]["login"], repository["name"], @@ -523,7 +543,13 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False): while True: page = page + 1 request = _construct_request( - per_page, page, query_args, template, auth, as_app=args.as_app + per_page, + page, + query_args, + template, + auth, + as_app=args.as_app, + fine=True if args.token_fine is not None else False, ) # noqa r, errors = _get_response(request, auth, template) @@ -559,7 +585,13 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False): retries += 1 time.sleep(5) request = _construct_request( - per_page, page, query_args, template, auth, as_app=args.as_app + per_page, + page, + query_args, + template, + auth, + as_app=args.as_app, + fine=True if args.token_fine is not None else False, ) # noqa r, errors = _get_response(request, auth, template) @@ -643,17 +675,23 @@ def _get_response(request, auth, template): return r, errors -def _construct_request(per_page, page, query_args, template, auth, as_app=None): +def _construct_request( + per_page, page, query_args, template, auth, as_app=None, fine=False +): querystring = urlencode( dict( - list({"per_page": per_page, "page": page}.items()) + list(query_args.items()) + list({"per_page": per_page, "page": page}.items()) + + list(query_args.items()) ) ) request = Request(template + "?" + querystring) if auth is not None: if not as_app: - request.add_header("Authorization", "Basic ".encode("ascii") + auth) + if fine: + request.add_header("Authorization", "token " + auth) + else: + request.add_header("Authorization", "Basic ".encode("ascii") + auth) else: auth = auth.encode("ascii") request.add_header("Authorization", "token ".encode("ascii") + auth)