Merge pull request #216 from Ondkloss/feature/fine_grained

Add support for fine-grained tokens (continued)

README.rst

@@ -29,16 +29,17 @@ Usage
 
 CLI Usage is as follows::
 
-    github-backup [-h] [-u USERNAME] [-p PASSWORD] [-t TOKEN] [--as-app]
-                  [-o OUTPUT_DIRECTORY] [-l LOG_LEVEL] [-i] [--starred]
-                  [--all-starred] [--watched] [--followers] [--following]
-                  [--all] [--issues] [--issue-comments] [--issue-events]
-                  [--pulls] [--pull-comments] [--pull-commits]
-                  [--pull-details] [--labels] [--hooks] [--milestones]
-                  [--repositories] [--bare] [--lfs] [--wikis] [--gists]
-                  [--starred-gists] [--skip-archived] [--skip-existing]
-                  [-L [LANGUAGES ...]] [-N NAME_REGEX] [-H GITHUB_HOST]
-                  [-O] [-R REPOSITORY] [-P] [-F] [--prefer-ssh] [-v]
+    github-backup [-h] [-u USERNAME] [-p PASSWORD] [-t TOKEN_CLASSIC]
+                  [-f TOKEN_FINE] [--as-app] [-o OUTPUT_DIRECTORY]
+                  [-l LOG_LEVEL] [-i] [--starred] [--all-starred]
+                  [--watched] [--followers] [--following] [--all] [--issues]
+                  [--issue-comments] [--issue-events] [--pulls]
+                  [--pull-comments] [--pull-commits] [--pull-details]
+                  [--labels] [--hooks] [--milestones] [--repositories]
+                  [--bare] [--lfs] [--wikis] [--gists] [--starred-gists]
+                  [--skip-archived] [--skip-existing] [-L [LANGUAGES ...]]
+                  [-N NAME_REGEX] [-H GITHUB_HOST] [-O] [-R REPOSITORY]
+                  [-P] [-F] [--prefer-ssh] [-v]
                   [--keychain-name OSX_KEYCHAIN_ITEM_NAME]
                   [--keychain-account OSX_KEYCHAIN_ITEM_ACCOUNT]
                   [--releases] [--assets] [--exclude [REPOSITORY [REPOSITORY ...]]
@@ -57,7 +58,10 @@ CLI Usage is as follows::
       -p PASSWORD, --password PASSWORD
                             password for basic auth. If a username is given but
                             not a password, the password will be prompted for.
-      -t TOKEN, --token TOKEN
+      -f TOKEN_FINE, --token-fine TOKEN_FINE
+                            fine-grained personal access token or path to token
+                            (file://...)
+      -t TOKEN_CLASSIC, --token TOKEN_CLASSIC
                             personal access, OAuth, or JSON Web token, or path to
                             token (file://...)
       --as-app              authenticate as github app instead of as a user.
@@ -163,13 +167,13 @@ Backup all repositories, including private ones::
     export ACCESS_TOKEN=SOME-GITHUB-TOKEN
     github-backup WhiteHouse --token $ACCESS_TOKEN --organization --output-directory /tmp/white-house --repositories --private
 
-Backup a single organization repository with everything else (wiki, pull requests, comments, issues etc)::
+Use a fine-grained access token to backup a single organization repository with everything else (wiki, pull requests, comments, issues etc)::
 
     export ACCESS_TOKEN=SOME-GITHUB-TOKEN
     ORGANIZATION=docker
     REPO=cli
     # e.g. git@github.com:docker/cli.git
-    github-backup $ORGANIZATION -P -t $ACCESS_TOKEN -o . --all -O -R $REPO
+    github-backup $ORGANIZATION -P -f $ACCESS_TOKEN -o . --all -O -R $REPO
 
 Testing
 =======

github_backup/github_backup.py

@@ -36,6 +36,7 @@ except ImportError:
     VERSION = "unknown"
 
 FNULL = open(os.devnull, "w")
+FILE_URI_PREFIX = "file://"
 logger = logging.getLogger(__name__)
 
 
@@ -126,9 +127,15 @@ def parse_args(args=None):
     parser.add_argument(
         "-t",
         "--token",
-        dest="token",
+        dest="token_classic",
         help="personal access, OAuth, or JSON Web token, or path to token (file://...)",
     )  # noqa
+    parser.add_argument(
+        "-f",
+        "--token-fine",
+        dest="token_fine",
+        help="fine-grained personal access token (github_pat_....), or path to token (file://...)",
+    )  # noqa
     parser.add_argument(
         "--as-app",
         action="store_true",
@@ -431,18 +438,27 @@ def get_auth(args, encode=True, for_git_cli=False):
         raise Exception(
             "You must specify both name and account fields for osx keychain password items"
         )
-    elif args.token:
-        _path_specifier = "file://"
-        if args.token.startswith(_path_specifier):
-            path_specifier_len = len(_path_specifier)
-            args.token = open(args.token[path_specifier_len:], "rt").readline().strip()
+    elif args.token_fine:
+        if args.token_fine.startswith(FILE_URI_PREFIX):
+            args.token_fine = read_file_contents(args.token_fine)
+
+        if args.token_fine.startswith("github_pat_"):
+            auth = args.token_fine
+        else:
+            raise Exception(
+                "Fine-grained token supplied does not look like a GitHub PAT"
+            )
+    elif args.token_classic:
+        if args.token_classic.startswith(FILE_URI_PREFIX):
+            args.token_classic = read_file_contents(args.token_classic)
+
         if not args.as_app:
-            auth = args.token + ":" + "x-oauth-basic"
+            auth = args.token_classic + ":" + "x-oauth-basic"
         else:
             if not for_git_cli:
-                auth = args.token
+                auth = args.token_classic
             else:
-                auth = "x-access-token:" + args.token
+                auth = "x-access-token:" + args.token_classic
     elif args.username:
         if not args.password:
             args.password = getpass.getpass()
@@ -457,7 +473,7 @@ def get_auth(args, encode=True, for_git_cli=False):
     if not auth:
         return None
 
-    if not encode:
+    if not encode or args.token_fine is not None:
         return auth
 
     return base64.b64encode(auth.encode("ascii"))
@@ -481,6 +497,10 @@ def get_github_host(args):
     return host
 
 
+def read_file_contents(file_uri):
+    return open(file_uri[len(FILE_URI_PREFIX) :], "rt").readline().strip()
+
+
 def get_github_repo_url(args, repository):
     if repository.get("is_gist"):
         if args.prefer_ssh:
@@ -503,7 +523,7 @@ def get_github_repo_url(args, repository):
     auth = get_auth(args, encode=False, for_git_cli=True)
     if auth:
         repo_url = "https://{0}@{1}/{2}/{3}.git".format(
-            auth,
+            auth if args.token_fine is None else "oauth2:" + auth,
             get_github_host(args),
             repository["owner"]["login"],
             repository["name"],
@@ -523,7 +543,13 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False):
     while True:
         page = page + 1
         request = _construct_request(
-            per_page, page, query_args, template, auth, as_app=args.as_app
+            per_page,
+            page,
+            query_args,
+            template,
+            auth,
+            as_app=args.as_app,
+            fine=True if args.token_fine is not None else False,
         )  # noqa
         r, errors = _get_response(request, auth, template)
 
@@ -559,7 +585,13 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False):
             retries += 1
             time.sleep(5)
             request = _construct_request(
-                per_page, page, query_args, template, auth, as_app=args.as_app
+                per_page,
+                page,
+                query_args,
+                template,
+                auth,
+                as_app=args.as_app,
+                fine=True if args.token_fine is not None else False,
             )  # noqa
             r, errors = _get_response(request, auth, template)
 
@@ -643,16 +675,22 @@ def _get_response(request, auth, template):
     return r, errors
 
 
-def _construct_request(per_page, page, query_args, template, auth, as_app=None):
+def _construct_request(
+    per_page, page, query_args, template, auth, as_app=None, fine=False
+):
     querystring = urlencode(
         dict(
-            list({"per_page": per_page, "page": page}.items()) + list(query_args.items())
+            list({"per_page": per_page, "page": page}.items())
+            + list(query_args.items())
         )
     )
 
     request = Request(template + "?" + querystring)
     if auth is not None:
         if not as_app:
+            if fine:
+                request.add_header("Authorization", "token " + auth)
+            else:
                 request.add_header("Authorization", "Basic ".encode("ascii") + auth)
         else:
             auth = auth.encode("ascii")