Bump version 3.3.5 (#902)

This reverts commit 0edc50e271.

CHANGELOG.md

@@ -1,5 +1,30 @@
 # Changelog
 
+## v3.3.5 (2023/09/13)
+
+* Change checkbox selector, support languages other than English
+
+## v3.3.4 (2023/09/02)
+
+* Update checkbox selector
+
+## v3.3.3 (2023/08/31)
+
+* Update undetected_chromedriver to v3.5.3
+
+## v3.3.2 (2023/08/03)
+
+* Fix URL domain in Prometheus exporter
+
+## v3.3.1 (2023/08/03)
+
+* Fix for Cloudflare verify checkbox
+* Fix HEADLESS=false in Windows binary
+* Fix Prometheus exporter for management and health endpoints
+* Remove misleading stack trace when the verify checkbox is not found
+* Revert "Update base Docker image to Debian Bookworm" #849
+* Revert "Install Chromium 115 from Debian testing" #849
+
 ## v3.3.0 (2023/08/02)
 
 * Fix for new Cloudflare detection. Thanks @cedric-bour for #845

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.11-slim-bookworm as builder
+FROM python:3.11-slim-bullseye as builder
 
 # Build dummy packages to skip installing them and their dependencies
 RUN apt-get update \
@@ -12,7 +12,7 @@ RUN apt-get update \
     && equivs-build adwaita-icon-theme \
     && mv adwaita-icon-theme_*.deb /adwaita-icon-theme.deb
 
-FROM python:3.11-slim-bookworm
+FROM python:3.11-slim-bullseye
 
 # Copy dummy packages
 COPY --from=builder /*.deb /
@@ -27,16 +27,12 @@ WORKDIR /app
     # Install dummy packages
 RUN dpkg -i /libgl1-mesa-dri.deb \
     && dpkg -i /adwaita-icon-theme.deb \
-    # Use Testing packages. The latest version of Chromium is not available for ARM
-    && sed -i 's/bookworm-updates/bookworm-updates testing/g' /etc/apt/sources.list.d/debian.sources \
     # Install dependencies
     && apt-get update \
-    && apt-get install -y --no-install-recommends -t testing chromium chromium-common chromium-driver xvfb dumb-init \
-        procps curl vim-tiny xauth \
+    && apt-get install -y --no-install-recommends chromium chromium-common chromium-driver xvfb dumb-init \
+        procps curl vim xauth \
     # Remove temporary files and hardware decoding libraries
     && rm -rf /var/lib/apt/lists/* \
-    && rm -f /usr/lib/systemd/systemd* \
-    && rm -f /usr/lib/x86_64-linux-gnu/systemd/* \
     && rm -f /usr/lib/x86_64-linux-gnu/libmfxhw* \
     && rm -f /usr/lib/x86_64-linux-gnu/mfx/* \
     # Create flaresolverr user
@@ -66,17 +62,17 @@ ENTRYPOINT ["/usr/bin/dumb-init", "--"]
 CMD ["/usr/local/bin/python", "-u", "/app/flaresolverr.py"]
 
 # Local build
-# docker build -t ngosang/flaresolverr:3.3.0 .
-# docker run -p 8191:8191 ngosang/flaresolverr:3.3.0
+# docker build -t ngosang/flaresolverr:3.3.5 .
+# docker run -p 8191:8191 ngosang/flaresolverr:3.3.5
 
 # Multi-arch build
 # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
 # docker buildx create --use
-# docker buildx build -t ngosang/flaresolverr:3.3.0 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 .
+# docker buildx build -t ngosang/flaresolverr:3.3.5 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 .
 #   add --push to publish in DockerHub
 
 # Test multi-arch build
 # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
 # docker buildx create --use
-# docker buildx build -t ngosang/flaresolverr:3.3.0 --platform linux/arm/v7 --load .
-# docker run -p 8191:8191 --platform linux/arm/v7 ngosang/flaresolverr:3.3.0
+# docker buildx build -t ngosang/flaresolverr:3.3.5 --platform linux/arm/v7 --load .
+# docker run -p 8191:8191 --platform linux/arm/v7 ngosang/flaresolverr:3.3.5

package.json

@@ -1,6 +1,6 @@
 {
   "name": "flaresolverr",
-  "version": "3.3.0",
+  "version": "3.3.5",
   "description": "Proxy server to bypass Cloudflare protection",
   "author": "Diego Heras (ngosang / ngosang@hotmail.es)",
   "license": "MIT"

src/bottle_plugins/prometheus_plugin.py

@@ -2,7 +2,8 @@ import logging
 import os
 import urllib.parse
 
-from dtos import V1ResponseBase
+from bottle import request
+from dtos import V1RequestBase, V1ResponseBase
 from metrics import start_metrics_http_server, REQUEST_COUNTER, REQUEST_DURATION
 
 PROMETHEUS_ENABLED = os.environ.get('PROMETHEUS_ENABLED', 'false').lower() == 'true'
@@ -33,10 +34,18 @@ def prometheus_plugin(callback):
     def export_metrics(actual_response):
         res = V1ResponseBase(actual_response)
 
+        if res.startTimestamp is None or res.endTimestamp is None:
+            # skip management and healthcheck endpoints
+            return
+
         domain = "unknown"
         if res.solution and res.solution.url:
-            parsed_url = urllib.parse.urlparse(res.solution.url)
-            domain = parsed_url.hostname
+            domain = parse_domain_url(res.solution.url)
+        else:
+            # timeout error
+            req = V1RequestBase(request.json)
+            if req.url:
+                domain = parse_domain_url(req.url)
 
         run_time = (res.endTimestamp - res.startTimestamp) / 1000
         REQUEST_DURATION.labels(domain=domain).observe(run_time)
@@ -50,4 +59,8 @@ def prometheus_plugin(callback):
             result = "error"
         REQUEST_COUNTER.labels(domain=domain, result=result).inc()
 
+    def parse_domain_url(url):
+        parsed_url = urllib.parse.urlparse(url)
+        return parsed_url.hostname
+
     return wrapper

src/flaresolverr.py

@@ -62,6 +62,12 @@ if __name__ == "__main__":
     if sys.version_info < (3, 9):
         raise Exception("The Python version is less than 3.9, a version equal to or higher is required.")
 
+    # fix for HEADLESS=false in Windows binary
+    # https://stackoverflow.com/a/27694505
+    if os.name == 'nt':
+        import multiprocessing
+        multiprocessing.freeze_support()
+
     # fix ssl certificates for compiled binaries
     # https://github.com/pyinstaller/pyinstaller/issues/7229
     # https://stackoverflow.com/questions/55736855/how-to-change-the-cafile-argument-in-the-ssl-module-in-python3

src/flaresolverr_service.py

@@ -46,7 +46,7 @@ CHALLENGE_SELECTORS = [
     # Fairlane / pararius.com
     'div.vc div.text-box h2'
 ]
-SHORT_TIMEOUT = 10
+SHORT_TIMEOUT = 1
 SESSIONS_STORAGE = SessionsStorage()
 
 
@@ -252,11 +252,11 @@ def _resolve_challenge(req: V1RequestBase, method: str) -> ChallengeResolutionT:
 def click_verify(driver: WebDriver):
     try:
         logging.debug("Try to find the Cloudflare verify checkbox...")
-        iframe = driver.find_element(By.XPATH, "//iframe[@title='Widget containing a Cloudflare security challenge']")
+        iframe = driver.find_element(By.XPATH, "//iframe[starts-with(@id, 'cf-chl-widget-')]")
         driver.switch_to.frame(iframe)
         checkbox = driver.find_element(
             by=By.XPATH,
-            value='//*[@id="cf-stage"]//label[@class="ctp-checkbox-label"]/input',
+            value='//*[@id="challenge-stage"]/div/label/map/img',
         )
         if checkbox:
             actions = ActionChains(driver)
@@ -264,8 +264,8 @@ def click_verify(driver: WebDriver):
             actions.click(checkbox)
             actions.perform()
             logging.debug("Cloudflare verify checkbox found and clicked!")
-    except Exception as e:
-        logging.debug("Cloudflare verify checkbox not found on the page. Error: " + str(e))
+    except Exception:
+        logging.debug("Cloudflare verify checkbox not found on the page.")
     finally:
         driver.switch_to.default_content()
 
@@ -281,8 +281,8 @@ def click_verify(driver: WebDriver):
             actions.click(button)
             actions.perform()
             logging.debug("The Cloudflare 'Verify you are human' button found and clicked!")
-    except Exception as e:
-        logging.debug("The Cloudflare 'Verify you are human' button not found on the page. Error: " + str(e))
+    except Exception:
+        logging.debug("The Cloudflare 'Verify you are human' button not found on the page.")
 
     time.sleep(2)
 
@@ -297,8 +297,8 @@ def _evil_logic(req: V1RequestBase, driver: WebDriver, method: str) -> Challenge
     if method == 'POST':
         _post_request(req, driver)
     else:
-        with driver:
         driver.get(req.url)
+        driver.start_session()  # required to bypass Cloudflare
 
     # set cookies if required
     if req.cookies is not None and len(req.cookies) > 0:
@@ -310,8 +310,8 @@ def _evil_logic(req: V1RequestBase, driver: WebDriver, method: str) -> Challenge
         if method == 'POST':
             _post_request(req, driver)
         else:
-            with driver:
             driver.get(req.url)
+            driver.start_session()  # required to bypass Cloudflare
 
     # wait for the page
     if utils.get_config_log_html():
@@ -430,5 +430,5 @@ def _post_request(req: V1RequestBase, driver: WebDriver):
             <script>document.getElementById('hackForm').submit();</script>
         </body>
         </html>"""
-    with driver:
     driver.get("data:text/html;charset=utf-8," + html_content)
+    driver.start_session()  # required to bypass Cloudflare

src/undetected_chromedriver/__init__.py

@@ -17,11 +17,12 @@ by UltrafunkAmsterdam (https://github.com/ultrafunkamsterdam)
 from __future__ import annotations
 
 
-__version__ = "3.5.0"
+__version__ = "3.5.3"
 
 import json
 import logging
 import os
+import pathlib
 import re
 import shutil
 import subprocess
@@ -373,6 +374,18 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
                 browser_executable_path or find_chrome_executable()
             )
 
+        if not options.binary_location or not \
+                pathlib.Path(options.binary_location).exists():
+                raise FileNotFoundError(
+                    "\n---------------------\n"
+                    "Could not determine browser executable."
+                    "\n---------------------\n"
+                    "Make sure your browser is installed in the default location (path).\n"
+                    "If you are sure about the browser executable, you can specify it using\n"
+                    "the `browser_executable_path='{}` parameter.\n\n"
+                    .format("/path/to/browser/executable" if IS_POSIX else "c:/path/to/your/browser.exe")
+                )
+
         self._delay = 3
 
         self.user_data_dir = user_data_dir
@@ -895,8 +908,6 @@ def find_chrome_executable():
             if item is not None:
                 for subitem in (
                     "Google/Chrome/Application",
-                    "Google/Chrome Beta/Application",
-                    "Google/Chrome Canary/Application",
                 ):
                     candidates.add(os.sep.join((item, subitem, "chrome.exe")))
     for candidate in candidates:

src/undetected_chromedriver/patcher.py

@@ -3,9 +3,11 @@
 
 from distutils.version import LooseVersion
 import io
+import json
 import logging
 import os
 import pathlib
+import platform
 import random
 import re
 import shutil
@@ -24,21 +26,9 @@ IS_POSIX = sys.platform.startswith(("darwin", "cygwin", "linux", "linux2"))
 
 class Patcher(object):
     lock = Lock()
-    url_repo = "https://chromedriver.storage.googleapis.com"
-    zip_name = "chromedriver_%s.zip"
     exe_name = "chromedriver%s"
 
     platform = sys.platform
-    if platform.endswith("win32"):
-        zip_name %= "win32"
-        exe_name %= ".exe"
-    if platform.endswith(("linux", "linux2")):
-        zip_name %= "linux64"
-        exe_name %= ""
-    if platform.endswith("darwin"):
-        zip_name %= "mac64"
-        exe_name %= ""
-
     if platform.endswith("win32"):
         d = "~/appdata/roaming/undetected_chromedriver"
     elif "LAMBDA_TASK_ROOT" in os.environ:
@@ -72,6 +62,10 @@ class Patcher(object):
         prefix = "undetected"
         self.user_multi_procs = user_multi_procs
 
+        self.is_old_chromedriver = version_main and version_main <= 114
+        # Needs to be called before self.exe_name is accessed
+        self._set_platform_name()
+
         if not os.path.exists(self.data_path):
             os.makedirs(self.data_path, exist_ok=True)
 
@@ -97,9 +91,33 @@ class Patcher(object):
             self._custom_exe_path = True
             self.executable_path = executable_path
 
+        # Set the correct repository to download the Chromedriver from
+        if self.is_old_chromedriver:
+            self.url_repo = "https://chromedriver.storage.googleapis.com"
+        else:
+            self.url_repo = "https://googlechromelabs.github.io/chrome-for-testing"
+
         self.version_main = version_main
         self.version_full = None
 
+    def _set_platform_name(self):
+        """
+        Set the platform and exe name based on the platform undetected_chromedriver is running on
+        in order to download the correct chromedriver.
+        """
+        if self.platform.endswith("win32"):
+            self.platform_name = "win32"
+            self.exe_name %= ".exe"
+        if self.platform.endswith(("linux", "linux2")):
+            self.platform_name = "linux64"
+            self.exe_name %= ""
+        if self.platform.endswith("darwin"):
+            if self.is_old_chromedriver:
+                self.platform_name = "mac64"
+            else:
+                self.platform_name = "mac-x64"
+            self.exe_name %= ""
+
     def auto(self, executable_path=None, force=False, version_main=None, _=None):
         """
 
@@ -111,16 +129,15 @@ class Patcher(object):
         Returns:
 
         """
-        # if self.user_multi_procs and \
-        #         self.user_multi_procs != -1:
-        #     # -1 being a skip value used later in this block
-        #
         p = pathlib.Path(self.data_path)
+        if self.user_multi_procs:
             with Lock():
-            files = list(p.rglob("*chromedriver*?"))
-            for file in files:
-                if self.is_binary_patched(file):
-                    self.executable_path = str(file)
+                files = list(p.rglob("*chromedriver*"))
+                most_recent = max(files, key=lambda f: f.stat().st_mtime)
+                files.remove(most_recent)
+                list(map(lambda f: f.unlink(), files))
+                if self.is_binary_patched(most_recent):
+                    self.executable_path = str(most_recent)
                     return True
 
         if executable_path:
@@ -202,7 +219,11 @@ class Patcher(object):
     def cleanup_unused_files(self):
         p = pathlib.Path(self.data_path)
         items = list(p.glob("*undetected*"))
-        print(items)
+        for item in items:
+            try:
+                item.unlink()
+            except:
+                pass
 
     def patch(self):
         self.patch_exe()
@@ -214,13 +235,33 @@ class Patcher(object):
         :return: version string
         :rtype: LooseVersion
         """
-        path = "/latest_release"
-        if self.version_main:
-            path += f"_{self.version_main}"
+        # Endpoint for old versions of Chromedriver (114 and below)
+        if self.is_old_chromedriver:
+            path = f"/latest_release_{self.version_main}"
             path = path.upper()
             logger.debug("getting release number from %s" % path)
             return LooseVersion(urlopen(self.url_repo + path).read().decode())
 
+        # Endpoint for new versions of Chromedriver (115+)
+        if not self.version_main:
+            # Fetch the latest version
+            path = "/last-known-good-versions-with-downloads.json"
+            logger.debug("getting release number from %s" % path)
+            with urlopen(self.url_repo + path) as conn:
+                response = conn.read().decode()
+
+            last_versions = json.loads(response)
+            return LooseVersion(last_versions["channels"]["Stable"]["version"])
+
+        # Fetch the latest minor version of the major version provided
+        path = "/latest-versions-per-milestone-with-downloads.json"
+        logger.debug("getting release number from %s" % path)
+        with urlopen(self.url_repo + path) as conn:
+            response = conn.read().decode()
+
+        major_versions = json.loads(response)
+        return LooseVersion(major_versions["milestones"][str(self.version_main)]["version"])
+
     def parse_exe_version(self):
         with io.open(self.executable_path, "rb") as f:
             for line in iter(lambda: f.readline(), b""):
@@ -234,10 +275,16 @@ class Patcher(object):
 
         :return: path to downloaded file
         """
-        u = "%s/%s/%s" % (self.url_repo, self.version_full.vstring, self.zip_name)
-        logger.debug("downloading from %s" % u)
-        # return urlretrieve(u, filename=self.data_path)[0]
-        return urlretrieve(u)[0]
+        zip_name = f"chromedriver_{self.platform_name}.zip"
+        if self.is_old_chromedriver:
+            download_url = "%s/%s/%s" % (self.url_repo, self.version_full.vstring, zip_name)
+        else:
+            zip_name = zip_name.replace("_", "-", 1)
+            download_url = "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/%s/%s/%s"
+            download_url %= (self.version_full.vstring, self.platform_name, zip_name)
+
+        logger.debug("downloading from %s" % download_url)
+        return urlretrieve(download_url)[0]
 
     def unzip_package(self, fp):
         """
@@ -245,6 +292,12 @@ class Patcher(object):
 
         :return: path to unpacked executable
         """
+        exe_path = self.exe_name
+        if not self.is_old_chromedriver:
+            # The new chromedriver unzips into its own folder
+            zip_name = f"chromedriver-{self.platform_name}"
+            exe_path = os.path.join(zip_name, self.exe_name)
+
         logger.debug("unzipping %s" % fp)
         try:
             os.unlink(self.zip_path)
@@ -253,10 +306,10 @@ class Patcher(object):
 
         os.makedirs(self.zip_path, mode=0o755, exist_ok=True)
         with zipfile.ZipFile(fp, mode="r") as zf:
-            zf.extract(self.exe_name, self.zip_path)
-        os.rename(os.path.join(self.zip_path, self.exe_name), self.executable_path)
+            zf.extractall(self.zip_path)
+        os.rename(os.path.join(self.zip_path, exe_path), self.executable_path)
         os.remove(fp)
-        os.rmdir(self.zip_path)
+        shutil.rmtree
         os.chmod(self.executable_path, 0o755)
         return self.executable_path