Merge pull request #449 from 0x2b3bfa0/patch-1

Use cursor based pagination

CHANGES.rst

@@ -1,10 +1,38 @@
 Changelog
 =========
 
-0.51.1 (2025-11-16)
+0.51.2 (2025-11-16)
 -------------------
 ------------------------
 
+Fix
+~~~
+- Improve CA certificate detection with fallback chain. [Rodos]
+
+  The previous implementation incorrectly assumed empty get_ca_certs()
+  meant broken SSL, causing false failures in GitHub Codespaces and other
+  directory-based cert systems where certificates exist but aren't pre-loaded.
+  It would then attempt to import certifi as a workaround, but certifi wasn't
+  listed in requirements.txt, causing the fallback to fail with ImportError
+  even though the system certificates would have worked fine.
+
+  This commit replaces the naive check with a layered fallback approach that
+  checks multiple certificate sources. First it checks for pre-loaded system
+  certs (file-based systems). Then it verifies system cert paths exist
+  (directory-based systems like Ubuntu/Debian/Codespaces). Finally it attempts
+  to use certifi as an optional fallback only if needed.
+
+  This approach eliminates hard dependencies (certifi is now optional), works
+  in GitHub Codespaces without any setup, and fails gracefully with clear hints
+  for resolution when SSL is actually broken rather than failing with
+  ModuleNotFoundError.
+
+  Fixes #444
+
+
+0.51.1 (2025-11-16)
+-------------------
+
 Fix
 ~~~
 - Prevent duplicate attachment downloads. [Rodos]

github_backup/__init__.py

@@ -1 +1 @@
-__version__ = "0.51.1"
+__version__ = "0.51.2"

github_backup/github_backup.py

@@ -37,22 +37,33 @@ FNULL = open(os.devnull, "w")
 FILE_URI_PREFIX = "file://"
 logger = logging.getLogger(__name__)
 
+# Setup SSL context with fallback chain
 https_ctx = ssl.create_default_context()
-if not https_ctx.get_ca_certs():
-    import warnings
+if https_ctx.get_ca_certs():
+    # Layer 1: Certificates pre-loaded from system (file-based)
+    pass
+else:
+    paths = ssl.get_default_verify_paths()
+    if (paths.cafile and os.path.exists(paths.cafile)) or (
+        paths.capath and os.path.exists(paths.capath)
+    ):
+        # Layer 2: Cert paths exist, will be lazy-loaded on first use (directory-based)
+        pass
+    else:
+        # Layer 3: Try certifi package as optional fallback
+        try:
+            import certifi
 
-    warnings.warn(
-        "\n\nYOUR DEFAULT CA CERTS ARE EMPTY.\n"
-        + "PLEASE POPULATE ANY OF:"
-        + "".join(
-            ["\n - " + x for x in ssl.get_default_verify_paths() if type(x) is str]
-        )
-        + "\n",
-        stacklevel=2,
-    )
-    import certifi
-
-    https_ctx = ssl.create_default_context(cafile=certifi.where())
+            https_ctx = ssl.create_default_context(cafile=certifi.where())
+        except ImportError:
+            # All layers failed - no certificates available anywhere
+            sys.exit(
+                "\nERROR: No CA certificates found. Cannot connect to GitHub over SSL.\n\n"
+                "Solutions you can explore:\n"
+                "  1. pip install certifi\n"
+                "  2. Alpine: apk add ca-certificates\n"
+                "  3. Debian/Ubuntu: apt-get install ca-certificates\n\n"
+            )
 
 
 def logging_subprocess(
@@ -581,27 +592,26 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False):
     auth = get_auth(args, encode=not args.as_app)
     query_args = get_query_args(query_args)
     per_page = 100
-    page = 0
+    next_url = None
 
     while True:
         if single_request:
-            request_page, request_per_page = None, None
+            request_per_page = None
         else:
-            page = page + 1
-            request_page, request_per_page = page, per_page
+            request_per_page = per_page
 
         request = _construct_request(
             request_per_page,
-            request_page,
             query_args,
-            template,
+            next_url or template,
             auth,
             as_app=args.as_app,
             fine=True if args.token_fine is not None else False,
         )  # noqa
-        r, errors = _get_response(request, auth, template)
+        r, errors = _get_response(request, auth, next_url or template)
 
         status_code = int(r.getcode())
+
         # Check if we got correct data
         try:
             response = json.loads(r.read().decode("utf-8"))
@@ -633,15 +643,14 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False):
             retries += 1
             time.sleep(5)
             request = _construct_request(
-                per_page,
-                page,
+                request_per_page,
                 query_args,
-                template,
+                next_url or template,
                 auth,
                 as_app=args.as_app,
                 fine=True if args.token_fine is not None else False,
             )  # noqa
-            r, errors = _get_response(request, auth, template)
+            r, errors = _get_response(request, auth, next_url or template)
 
             status_code = int(r.getcode())
             try:
@@ -671,7 +680,16 @@ def retrieve_data_gen(args, template, query_args=None, single_request=False):
             if type(response) is list:
                 for resp in response:
                     yield resp
-                if len(response) < per_page:
+                # Parse Link header for next page URL (cursor-based pagination)
+                link_header = r.headers.get("Link", "")
+                next_url = None
+                if link_header:
+                    # Parse Link header: <https://api.github.com/...?per_page=100&after=cursor>; rel="next"
+                    for link in link_header.split(","):
+                        if 'rel="next"' in link:
+                            next_url = link[link.find("<") + 1:link.find(">")]
+                            break
+                if not next_url:
                     break
             elif type(response) is dict and single_request:
                 yield response
@@ -724,22 +742,27 @@ def _get_response(request, auth, template):
 
 
 def _construct_request(
-    per_page, page, query_args, template, auth, as_app=None, fine=False
+    per_page, query_args, template, auth, as_app=None, fine=False
 ):
-    all_query_args = {}
-    if per_page:
-        all_query_args["per_page"] = per_page
-    if page:
-        all_query_args["page"] = page
-    if query_args:
-        all_query_args.update(query_args)
-
-    request_url = template
-    if all_query_args:
-        querystring = urlencode(all_query_args)
-        request_url = template + "?" + querystring
+    # If template is already a full URL with query params (from Link header), use it directly
+    if "?" in template and template.startswith("http"):
+        request_url = template
+        # Extract query string for logging
+        querystring = template.split("?", 1)[1]
     else:
-        querystring = ""
+        # Build URL with query parameters
+        all_query_args = {}
+        if per_page:
+            all_query_args["per_page"] = per_page
+        if query_args:
+            all_query_args.update(query_args)
+
+        request_url = template
+        if all_query_args:
+            querystring = urlencode(all_query_args)
+            request_url = template + "?" + querystring
+        else:
+            querystring = ""
 
     request = Request(request_url)
     if auth is not None:
@@ -755,7 +778,7 @@ def _construct_request(
                 "Accept", "application/vnd.github.machine-man-preview+json"
             )
 
-    log_url = template
+    log_url = template if "?" not in template else template.split("?")[0]
     if querystring:
         log_url += "?" + querystring
     logger.info("Requesting {}".format(log_url))
@@ -832,8 +855,7 @@ def download_file(url, path, auth, as_app=False, fine=False):
         return
 
     request = _construct_request(
-        per_page=100,
-        page=1,
+        per_page=None,
         query_args={},
         template=url,
         auth=auth,

requirements.txt

@@ -1 +0,0 @@
-