Bump version 3.1.0

* Add support for sessions

* Add tests for sessions

* Missing return type

* Don't re-create an existing session

* Return success in case of session doesn't exists on destroy

* Create session if necessary on get request

* Add session TTL to the request.get method

When fetching some webpage with a predefined session id,
FlareSorverr is using existing instance of WebDriver.
That allows user to not manage cookies explicitly
and rely on WebDriver to maintain the session. However,
if session has been created long time ago, CloudFlare might
stop accepting the requests, so we want to recreate the session
time to time. From the user perspective the easiest way of doing it
is to define their expectation on the session duration.

These changes add an option to define Time-to-live (TTL) for the session
and FlareSorverr takes care about rotating the sessions.

* Update message for session destroy in tests

---------

Co-authored-by: Michel Roux <xefir@crystalyx.net>

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -32,7 +32,8 @@ body:
       - Operating system:
       - Are you using Docker: [yes/no]
       - FlareSolverr User-Agent (see log traces or / endpoint):
-      - Are you using a proxy or VPN: [yes/no]
+      - Are you using a VPN: [yes/no]
+      - Are you using a Proxy: [yes/no]
       - Are you using Captcha Solver: [yes/no]
       - If using captcha solver, which one:
       - URL to test this issue:

.github/workflows/release-docker.yml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       -
         name: Checkout

.github/workflows/release.yml

@@ -6,26 +6,15 @@ on:
       - 'v*.*.*'
 
 jobs:
-  build:
+  create-release:
     name: Create release
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
         with:
           fetch-depth: 0 # get all commits, branches and tags (required for the changelog)
 
-      - name: Setup Node
-        uses: actions/setup-node@v3
-        with:
-          node-version: '16'
-
-      - name: Build artifacts
-        run: |
-          npm install
-          npm run build
-          npm run package
-
       - name: Build changelog
         id: github_changelog
         run: |
@@ -47,9 +36,60 @@ jobs:
           draft: false
           prerelease: false
 
+  build-linux:
+    name: Build Linux binary
+    needs: create-release
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # get all commits, branches and tags (required for the changelog)
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Build artifacts
+        run: |
+          python -m pip install -r requirements.txt
+          python -m pip install pyinstaller==5.9.0
+          cd src
+          python build_package.py
+
       - name: Upload release artifacts
-        uses: alexellis/upload-assets@0.2.2
+        uses: alexellis/upload-assets@0.4.0
         env:
           GITHUB_TOKEN: ${{ secrets.GH_PAT }}
         with:
-          asset_paths: '["./bin/*.zip"]'
+          asset_paths: '["./dist/flaresolverr_*"]'
+
+  build-windows:
+    name: Build Windows binary
+    needs: create-release
+    runs-on: windows-2022
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # get all commits, branches and tags (required for the changelog)
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Build artifacts
+        run: |
+          python -m pip install -r requirements.txt
+          python -m pip install pyinstaller==5.9.0
+          cd src
+          python build_package.py
+
+      - name: Upload release artifacts
+        uses: alexellis/upload-assets@0.4.0
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_PAT }}
+        with:
+          asset_paths: '["./dist/flaresolverr_*"]'

.gitignore

@@ -25,6 +25,7 @@ __pycache__/
 build/
 develop-eggs/
 dist/
+dist_chrome/
 downloads/
 eggs/
 .eggs/

CHANGELOG.md

@@ -1,6 +1,31 @@
 # Changelog
 
-## v3.1.0 (upcoming)
+## v3.1.0 (2023/03/20)
+
+* Build binaries for Linux x64 and Windows x64
+* Sessions with auto-creation on fetch request and TTL
+* Fix error trace: Crash Reports/pending No such file or directory
+* Fix Waitress server error with asyncore_use_poll=true
+* Attempt to fix Docker ARM32 build
+* Print platform information on start up
+* Add Fairlane challenge selector
+* Update DDOS-GUARD title
+* Update dependencies
+
+## v3.0.4 (2023/03/07)
+
+* Click on the Cloudflare's 'Verify you are human' button if necessary
+
+## v3.0.3 (2023/03/06)
+
+* Update undetected_chromedriver version to 3.4.6
+
+## v3.0.2 (2023/01/08)
+
+* Detect Cloudflare blocked access
+* Check Chrome / Chromium web browser is installed correctly
+
+## v3.0.1 (2023/01/06)
 
 * Kill Chromium processes properly to avoid defunct/zombie processes
 * Update undetected-chromedriver

Dockerfile

@@ -30,7 +30,7 @@ RUN dpkg -i /libgl1-mesa-dri.deb \
     # Install dependencies
     && apt-get update \
     && apt-get install -y --no-install-recommends chromium chromium-common chromium-driver xvfb dumb-init \
-        procps curl vim \
+        procps curl vim xauth \
     # Remove temporary files and hardware decoding libraries
     && rm -rf /var/lib/apt/lists/* \
     && rm -f /usr/lib/x86_64-linux-gnu/libmfxhw* \
@@ -48,6 +48,8 @@ RUN pip install -r requirements.txt \
 
 USER flaresolverr
 
+RUN mkdir -p "/app/.config/chromium/Crash Reports/pending"
+
 COPY src .
 COPY package.json ../
 
@@ -67,3 +69,9 @@ CMD ["/usr/local/bin/python", "-u", "/app/flaresolverr.py"]
 # docker buildx create --use
 # docker buildx build -t ngosang/flaresolverr:3.0.0 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 .
 #   add --push to publish in DockerHub
+
+# Test multi-arch build
+# docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+# docker buildx create --use
+# docker buildx build -t ngosang/flaresolverr:3.0.0 --platform linux/arm/v7 --load .
+# docker run -p 8191:8191 --platform linux/arm/v7 ngosang/flaresolverr:3.0.0

README.md

@@ -65,15 +65,13 @@ Remember to restart the Docker daemon and the container after the update.
 ### Precompiled binaries
 
 This is the recommended way for Windows users.
-* Download the [FlareSolverr zip](https://github.com/FlareSolverr/FlareSolverr/releases) from the release's assets. It is available for Windows and Linux.
-* Extract the zip file. FlareSolverr executable and firefox folder must be in the same directory.
+* Download the [FlareSolverr executable](https://github.com/FlareSolverr/FlareSolverr/releases) from the release's page. It is available for Windows x64 and Linux x64.
 * Execute FlareSolverr binary. In the environment variables section you can find how to change the configuration.
 
 ### From source code
 
-This is the recommended way for macOS users and for developers.
-* Install [Python 3.10](https://www.python.org/downloads/).
-* Install [Chrome](https://www.google.com/intl/en_us/chrome/) or [Chromium](https://www.chromium.org/getting-involved/download-chromium/) web browser.
+* Install [Python 3.11](https://www.python.org/downloads/).
+* Install [Chrome](https://www.google.com/intl/en_us/chrome/) (all OS) or [Chromium](https://www.chromium.org/getting-involved/download-chromium/) (just Linux, it doesn't work in Windows) web browser.
 * (Only in Linux / macOS) Install [Xvfb](https://en.wikipedia.org/wiki/Xvfb) package.
 * Clone this repository and open a shell in that path.
 * Run `pip install -r requirements.txt` command to install FlareSolverr dependencies.
@@ -140,14 +138,15 @@ session. When you no longer need to use a session you should make sure to close
 
 #### + `request.get`
 
-| Parameter         | Notes                                                                                                                                                                                                                                                                                                                                        |
-|-------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| url               | Mandatory                                                                                                                                                                                                                                                                                                                                    |
-| session           | Optional. Will send the request from and existing browser instance. If one is not sent it will create a temporary instance that will be destroyed immediately after the request is completed.                                                                                                                                                |
-| maxTimeout        | Optional, default value 60000. Max timeout to solve the challenge in milliseconds.                                                                                                                                                                                                                                                           |
-| cookies           | Optional. Will be used by the headless browser. Follow [this](https://github.com/puppeteer/puppeteer/blob/v3.3.0/docs/api.md#pagesetcookiecookies) format.                                                                                                                                                                                   |
-| returnOnlyCookies | Optional, default false. Only returns the cookies. Response data, headers and other parts of the response are removed.                                                                                                                                                                                                                       |
-| proxy             | Optional, default disabled. Eg: `"proxy": {"url": "http://127.0.0.1:8888"}`. You must include the proxy schema in the URL: `http://`, `socks4://` or `socks5://`. Authorization (username/password) is not supported. (When the `session` parameter is set, the proxy is ignored; a session specific proxy can be set in `sessions.create`.) |
+| Parameter           | Notes                                                                                                                                                                                                                                                                                                                                        |
+|---------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| url                 | Mandatory                                                                                                                                                                                                                                                                                                                                    |
+| session             | Optional. Will send the request from and existing browser instance. If one is not sent it will create a temporary instance that will be destroyed immediately after the request is completed.                                                                                                                                                |
+| session_ttl_minutes | Optional. FlareSolverr will automatically rotate expired sessions based on the TTL provided in minutes.                                                                                                                                                                                                                                      |
+| maxTimeout          | Optional, default value 60000. Max timeout to solve the challenge in milliseconds.                                                                                                                                                                                                                                                           |
+| cookies             | Optional. Will be used by the headless browser. Follow [this](https://github.com/puppeteer/puppeteer/blob/v3.3.0/docs/api.md#pagesetcookiecookies) format.                                                                                                                                                                                   |
+| returnOnlyCookies   | Optional, default false. Only returns the cookies. Response data, headers and other parts of the response are removed.                                                                                                                                                                                                                       |
+| proxy               | Optional, default disabled. Eg: `"proxy": {"url": "http://127.0.0.1:8888"}`. You must include the proxy schema in the URL: `http://`, `socks4://` or `socks5://`. Authorization (username/password) is not supported. (When the `session` parameter is set, the proxy is ignored; a session specific proxy can be set in `sessions.create`.) |
 
 :warning: If you want to use Cloudflare clearance cookie in your scripts, make sure you use the FlareSolverr User-Agent too. If they don't match you will see the challenge.
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "flaresolverr",
-  "version": "3.0.1",
+  "version": "3.1.0",
   "description": "Proxy server to bypass Cloudflare protection",
   "author": "Diego Heras (ngosang / ngosang@hotmail.es)",
   "license": "MIT"

requirements.txt

@@ -1,9 +1,12 @@
-bottle==0.12.23
+bottle==0.12.25
 waitress==2.1.2
-selenium==4.7.2
+selenium==4.8.2
 func-timeout==4.3.5
 # required by undetected_chromedriver
-requests==2.28.1
+requests==2.28.2
+certifi==2022.12.7
 websockets==10.4
 # only required for linux
 xvfbwrapper==0.2.9
+# only required for windows
+pefile==2023.2.7

src/build_package.py

@@ -0,0 +1,86 @@
+import os
+import platform
+import shutil
+import subprocess
+import sys
+import zipfile
+
+import requests
+
+
+def clean_files():
+    try:
+        shutil.rmtree(os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'build'))
+    except Exception:
+        pass
+    try:
+        shutil.rmtree(os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'dist'))
+    except Exception:
+        pass
+    try:
+        shutil.rmtree(os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'dist_chrome'))
+    except Exception:
+        pass
+
+
+def download_chromium():
+    # https://commondatastorage.googleapis.com/chromium-browser-snapshots/index.html?prefix=Linux_x64/
+    revision = "1090006" if os.name == 'nt' else '1090007'
+    arch = 'Win' if os.name == 'nt' else 'Linux_x64'
+    dl_file = 'chrome-win' if os.name == 'nt' else 'chrome-linux'
+    dl_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'dist_chrome')
+    dl_path_folder = os.path.join(dl_path, dl_file)
+    dl_path_zip = dl_path_folder + '.zip'
+
+    # response = requests.get(
+    #     f'https://commondatastorage.googleapis.com/chromium-browser-snapshots/{arch}/LAST_CHANGE',
+    #     timeout=30)
+    # revision = response.text.strip()
+    print("Downloading revision: " + revision)
+
+    os.mkdir(dl_path)
+    with requests.get(
+            f'https://commondatastorage.googleapis.com/chromium-browser-snapshots/{arch}/{revision}/{dl_file}.zip',
+            stream=True) as r:
+        r.raise_for_status()
+        with open(dl_path_zip, 'wb') as f:
+            for chunk in r.iter_content(chunk_size=8192):
+                f.write(chunk)
+    print("File downloaded: " + dl_path_zip)
+    with zipfile.ZipFile(dl_path_zip, 'r') as zip_ref:
+        zip_ref.extractall(dl_path)
+    os.remove(dl_path_zip)
+    shutil.move(dl_path_folder, os.path.join(dl_path, "chrome"))
+
+
+def run_pyinstaller():
+    sep = ';' if os.name == 'nt' else ':'
+    subprocess.check_call([sys.executable, "-m", "PyInstaller",
+                           "--onefile",
+                           "--add-data", f"package.json{sep}.",
+                           "--add-data", f"{os.path.join('dist_chrome', 'chrome')}{sep}chrome",
+                           os.path.join("src", "flaresolverr.py")],
+                          cwd=os.pardir)
+    exe_folder = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'dist')
+    exe_name = 'flaresolverr.exe' if os.name == 'nt' else 'flaresolverr'
+    exe_new_name = 'flaresolverr_windows_x64.exe' if os.name == 'nt' else 'flaresolverr_linux_x64'
+    exe_path = os.path.join(exe_folder, exe_name)
+    exe_new_path = os.path.join(exe_folder, exe_new_name)
+    shutil.move(exe_path, exe_new_path)
+    print("Executable path: " + exe_new_path)
+
+
+if __name__ == "__main__":
+    print("Building package...")
+    print("Platform: " + platform.platform())
+
+    print("Cleaning previous build...")
+    clean_files()
+
+    print("Downloading Chromium...")
+    download_chromium()
+
+    print("Building pyinstaller executable... ")
+    run_pyinstaller()
+
+# NOTE: python -m pip install pyinstaller

src/dtos.py

@@ -33,6 +33,7 @@ class V1RequestBase(object):
     maxTimeout: int = None
     proxy: dict = None
     session: str = None
+    session_ttl_minutes: int = None
     headers: list = None  # deprecated v2.0.0, not used
     userAgent: str = None  # deprecated v2.0.0, not used
 
@@ -51,6 +52,8 @@ class V1ResponseBase(object):
     # V1ResponseBase
     status: str = None
     message: str = None
+    session: str = None
+    sessions: list[str] = None
     startTimestamp: int = None
     endTimestamp: int = None
     version: str = None

src/flaresolverr.py

@@ -3,11 +3,12 @@ import logging
 import os
 import sys
 
-from bottle import run, response, Bottle, request
+import certifi
+from bottle import run, response, Bottle, request, ServerAdapter
 
 from bottle_plugins.error_plugin import error_plugin
 from bottle_plugins.logger_plugin import logger_plugin
-from dtos import IndexResponse, V1RequestBase
+from dtos import V1RequestBase
 import flaresolverr_service
 import utils
 
@@ -60,6 +61,12 @@ def controller_v1():
 
 
 if __name__ == "__main__":
+    # fix ssl certificates for compiled binaries
+    # https://github.com/pyinstaller/pyinstaller/issues/7229
+    # https://stackoverflow.com/questions/55736855/how-to-change-the-cafile-argument-in-the-ssl-module-in-python3
+    os.environ["REQUESTS_CA_BUNDLE"] = certifi.where()
+    os.environ["SSL_CERT_FILE"] = certifi.where()
+
     # validate configuration
     log_level = os.environ.get('LOG_LEVEL', 'info').upper()
     log_html = utils.get_config_log_html()
@@ -92,4 +99,10 @@ if __name__ == "__main__":
 
     # start webserver
     # default server 'wsgiref' does not support concurrent requests
-    run(app, host=server_host, port=server_port, quiet=True, server='waitress')
+    # https://github.com/FlareSolverr/FlareSolverr/issues/680
+    # https://github.com/Pylons/waitress/issues/31
+    class WaitressServerPoll(ServerAdapter):
+        def run(self, handler):
+            from waitress import serve
+            serve(handler, host=self.host, port=self.port, asyncore_use_poll=True)
+    run(app, host=server_host, port=server_port, quiet=True, server=WaitressServerPoll)

src/flaresolverr_service.py

@@ -1,44 +1,77 @@
 import logging
+import platform
+import sys
 import time
+from datetime import timedelta
 from urllib.parse import unquote
 
-from func_timeout import func_timeout, FunctionTimedOut
+from func_timeout import FunctionTimedOut, func_timeout
 from selenium.common import TimeoutException
 from selenium.webdriver.chrome.webdriver import WebDriver
 from selenium.webdriver.common.by import By
+from selenium.webdriver.support.expected_conditions import (
+    presence_of_element_located, staleness_of, title_is)
+from selenium.webdriver.common.action_chains import ActionChains
 from selenium.webdriver.support.wait import WebDriverWait
-from selenium.webdriver.support.expected_conditions import presence_of_element_located, staleness_of, title_is
 
-from dtos import V1RequestBase, V1ResponseBase, ChallengeResolutionT, ChallengeResolutionResultT, IndexResponse, \
-    HealthResponse, STATUS_OK, STATUS_ERROR
 import utils
+from dtos import (STATUS_ERROR, STATUS_OK, ChallengeResolutionResultT,
+                  ChallengeResolutionT, HealthResponse, IndexResponse,
+                  V1RequestBase, V1ResponseBase)
+from sessions import SessionsStorage
 
+ACCESS_DENIED_TITLES = [
+    # Cloudflare
+    'Access denied',
+    # Cloudflare http://bitturk.net/ Firefox
+    'Attention Required! | Cloudflare'
+]
 ACCESS_DENIED_SELECTORS = [
     # Cloudflare
-    'div.cf-error-title span.cf-code-label span'
+    'div.cf-error-title span.cf-code-label span',
     # Cloudflare http://bitturk.net/ Firefox
     '#cf-error-details div.cf-error-overview h1'
 ]
-CHALLENGE_TITLE = [
+CHALLENGE_TITLES = [
     # Cloudflare
     'Just a moment...',
     # DDoS-GUARD
-    'DDOS-GUARD',
+    'DDoS-Guard'
 ]
 CHALLENGE_SELECTORS = [
     # Cloudflare
     '#cf-challenge-running', '.ray_id', '.attack-box', '#cf-please-wait', '#challenge-spinner', '#trk_jschal_js',
     # Custom CloudFlare for EbookParadijs, Film-Paleis, MuziekFabriek and Puur-Hollands
-    'td.info #js_info'
+    'td.info #js_info',
+    # Fairlane / pararius.com
+    'div.vc div.text-box h2'
 ]
 SHORT_TIMEOUT = 10
+SESSIONS_STORAGE = SessionsStorage()
 
 
 def test_browser_installation():
     logging.info("Testing web browser installation...")
+    logging.info("Platform: " + platform.platform())
+
+    chrome_exe_path = utils.get_chrome_exe_path()
+    if chrome_exe_path is None:
+        logging.error("Chrome / Chromium web browser not installed!")
+        sys.exit(1)
+    else:
+        logging.info("Chrome / Chromium path: " + chrome_exe_path)
+
+    chrome_major_version = utils.get_chrome_major_version()
+    if chrome_major_version == '':
+        logging.error("Chrome / Chromium version not detected!")
+        sys.exit(1)
+    else:
+        logging.info("Chrome / Chromium major version: " + chrome_major_version)
+
+    logging.info("Launching web browser...")
     user_agent = utils.get_user_agent()
     logging.info("FlareSolverr User-Agent: " + user_agent)
-    logging.info("Test successful")
+    logging.info("Test successful!")
 
 
 def index_endpoint() -> IndexResponse:
@@ -92,11 +125,11 @@ def _controller_v1_handler(req: V1RequestBase) -> V1ResponseBase:
     # execute the command
     res: V1ResponseBase
     if req.cmd == 'sessions.create':
-        raise Exception("Not implemented yet.")
+        res = _cmd_sessions_create(req)
     elif req.cmd == 'sessions.list':
-        raise Exception("Not implemented yet.")
+        res = _cmd_sessions_list(req)
     elif req.cmd == 'sessions.destroy':
-        raise Exception("Not implemented yet.")
+        res = _cmd_sessions_destroy(req)
     elif req.cmd == 'request.get':
         res = _cmd_request_get(req)
     elif req.cmd == 'request.post':
@@ -143,19 +176,116 @@ def _cmd_request_post(req: V1RequestBase) -> V1ResponseBase:
     return res
 
 
+def _cmd_sessions_create(req: V1RequestBase) -> V1ResponseBase:
+    logging.debug("Creating new session...")
+
+    session, fresh = SESSIONS_STORAGE.create(session_id=req.session)
+    session_id = session.session_id
+
+    if not fresh:
+        return V1ResponseBase({
+            "status": STATUS_OK,
+            "message": "Session already exists.",
+            "session": session_id
+        })
+
+    return V1ResponseBase({
+        "status": STATUS_OK,
+        "message": "Session created successfully.",
+        "session": session_id
+    })
+
+
+def _cmd_sessions_list(req: V1RequestBase) -> V1ResponseBase:
+    session_ids = SESSIONS_STORAGE.session_ids()
+
+    return V1ResponseBase({
+        "status": STATUS_OK,
+        "message": "",
+        "sessions": session_ids
+    })
+
+
+def _cmd_sessions_destroy(req: V1RequestBase) -> V1ResponseBase:
+    session_id = req.session
+    existed = SESSIONS_STORAGE.destroy(session_id)
+
+    if not existed:
+        raise Exception("The session doesn't exist.")
+
+    return V1ResponseBase({
+        "status": STATUS_OK,
+        "message": "The session has been removed."
+    })
+
+
 def _resolve_challenge(req: V1RequestBase, method: str) -> ChallengeResolutionT:
     timeout = req.maxTimeout / 1000
     driver = None
     try:
-        driver = utils.get_webdriver()
+        if req.session:
+            session_id = req.session
+            ttl = timedelta(minutes=req.session_ttl_minutes) if req.session_ttl_minutes else None
+            session, fresh = SESSIONS_STORAGE.get(session_id, ttl)
+
+            if fresh:
+                logging.debug(f"new session created to perform the request (session_id={session_id})")
+            else:
+                logging.debug(f"existing session is used to perform the request (session_id={session_id}, "
+                              f"lifetime={str(session.lifetime())}, ttl={str(ttl)})")
+
+            driver = session.driver
+        else:
+            driver = utils.get_webdriver()
+            logging.debug('New instance of webdriver has been created to perform the request')
         return func_timeout(timeout, _evil_logic, (req, driver, method))
     except FunctionTimedOut:
         raise Exception(f'Error solving the challenge. Timeout after {timeout} seconds.')
     except Exception as e:
         raise Exception('Error solving the challenge. ' + str(e))
     finally:
-        if driver is not None:
+        if not req.session and driver is not None:
             driver.quit()
+            logging.debug('A used instance of webdriver has been destroyed')
+
+
+def click_verify(driver: WebDriver):
+    try:
+        logging.debug("Try to find the Cloudflare verify checkbox")
+        iframe = driver.find_element(By.XPATH, "//iframe[@title='Widget containing a Cloudflare security challenge']")
+        driver.switch_to.frame(iframe)
+        checkbox = driver.find_element(
+            by=By.XPATH,
+            value='//*[@id="cf-stage"]//label[@class="ctp-checkbox-label"]/input',
+        )
+        if checkbox:
+            actions = ActionChains(driver)
+            actions.move_to_element_with_offset(checkbox, 5, 7)
+            actions.click(checkbox)
+            actions.perform()
+            logging.debug("Cloudflare verify checkbox found and clicked")
+    except Exception:
+        logging.debug("Cloudflare verify checkbox not found on the page")
+    finally:
+        driver.switch_to.default_content()
+
+    try:
+        logging.debug("Try to find the Cloudflare 'Verify you are human' button")
+        button = driver.find_element(
+            by=By.XPATH,
+            value="//input[@type='button' and @value='Verify you are human']",
+        )
+        if button:
+            actions = ActionChains(driver)
+            actions.move_to_element_with_offset(button, 5, 7)
+            actions.click(button)
+            actions.perform()
+            logging.debug("The Cloudflare 'Verify you are human' button found and clicked")
+    except Exception as e:
+        logging.debug("The Cloudflare 'Verify you are human' button not found on the page")
+        # print(e)
+
+    time.sleep(2)
 
 
 def _evil_logic(req: V1RequestBase, driver: WebDriver, method: str) -> ChallengeResolutionT:
@@ -174,7 +304,13 @@ def _evil_logic(req: V1RequestBase, driver: WebDriver, method: str) -> Challenge
 
     # wait for the page
     html_element = driver.find_element(By.TAG_NAME, "html")
+    page_title = driver.title
 
+    # find access denied titles
+    for title in ACCESS_DENIED_TITLES:
+        if title == page_title:
+            raise Exception('Cloudflare has blocked this request. '
+                            'Probably your IP is banned for this site, check in your web browser.')
     # find access denied selectors
     for selector in ACCESS_DENIED_SELECTORS:
         found_elements = driver.find_elements(By.CSS_SELECTOR, selector)
@@ -184,11 +320,10 @@ def _evil_logic(req: V1RequestBase, driver: WebDriver, method: str) -> Challenge
 
     # find challenge by title
     challenge_found = False
-    page_title = driver.title
-    for title in CHALLENGE_TITLE:
-        if title == page_title:
+    for title in CHALLENGE_TITLES:
+        if title.lower() == page_title.lower():
             challenge_found = True
-            logging.info("Challenge detected. Title found: " + title)
+            logging.info("Challenge detected. Title found: " + page_title)
             break
     if not challenge_found:
         # find challenge by selectors
@@ -199,17 +334,19 @@ def _evil_logic(req: V1RequestBase, driver: WebDriver, method: str) -> Challenge
                 logging.info("Challenge detected. Selector found: " + selector)
                 break
 
+    attempt = 0
     if challenge_found:
         while True:
             try:
-                # wait until the title change
-                for title in CHALLENGE_TITLE:
-                    logging.debug("Waiting for title: " + title)
+                attempt = attempt + 1
+                # wait until the title changes
+                for title in CHALLENGE_TITLES:
+                    logging.debug("Waiting for title (attempt " + str(attempt) + "): " + title)
                     WebDriverWait(driver, SHORT_TIMEOUT).until_not(title_is(title))
 
                 # then wait until all the selectors disappear
                 for selector in CHALLENGE_SELECTORS:
-                    logging.debug("Waiting for selector: " + selector)
+                    logging.debug("Waiting for selector (attempt " + str(attempt) + "): " + selector)
                     WebDriverWait(driver, SHORT_TIMEOUT).until_not(
                         presence_of_element_located((By.CSS_SELECTOR, selector)))
 
@@ -218,6 +355,9 @@ def _evil_logic(req: V1RequestBase, driver: WebDriver, method: str) -> Challenge
 
             except TimeoutException:
                 logging.debug("Timeout waiting for selector")
+
+                click_verify(driver)
+
                 # update the html (cloudflare reloads the page every 5 s)
                 html_element = driver.find_element(By.TAG_NAME, "html")
 

src/sessions.py

@@ -0,0 +1,81 @@
+import logging
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from typing import Optional, Tuple
+from uuid import uuid1
+
+from selenium.webdriver.chrome.webdriver import WebDriver
+
+import utils
+
+
+@dataclass
+class Session:
+    session_id: str
+    driver: WebDriver
+    created_at: datetime
+
+    def lifetime(self) -> timedelta:
+        return datetime.now() - self.created_at
+
+
+class SessionsStorage:
+    """SessionsStorage creates, stores and process all the sessions"""
+
+    def __init__(self):
+        self.sessions = {}
+
+    def create(self, session_id: Optional[str] = None, force_new: Optional[bool] = False) -> Tuple[Session, bool]:
+        """create creates new instance of WebDriver if necessary,
+        assign defined (or newly generated) session_id to the instance
+        and returns the session object. If a new session has been created
+        second argument is set to True.
+
+        Note: The function is idempotent, so in case if session_id
+        already exists in the storage a new instance of WebDriver won't be created
+        and existing session will be returned. Second argument defines if 
+        new session has been created (True) or an existing one was used (False).
+        """
+        session_id = session_id or str(uuid1())
+
+        if force_new:
+            self.destroy(session_id)
+
+        if self.exists(session_id):
+            return self.sessions[session_id], False
+
+        driver = utils.get_webdriver()
+        created_at = datetime.now()
+        session = Session(session_id, driver, created_at)
+
+        self.sessions[session_id] = session
+
+        return session, True
+
+    def exists(self, session_id: str) -> bool:
+        return session_id in self.sessions
+
+    def destroy(self, session_id: str) -> bool:
+        """destroy closes the driver instance and removes session from the storage.
+        The function is noop if session_id doesn't exist.
+        The function returns True if session was found and destroyed,
+        and False if session_id wasn't found.
+        """
+        if not self.exists(session_id):
+            return False
+
+        session = self.sessions.pop(session_id)
+        session.driver.quit()
+        return True
+
+    def get(self, session_id: str, ttl: Optional[timedelta] = None) -> Tuple[Session, bool]:
+        session, fresh = self.create(session_id)
+
+        if ttl is not None and not fresh and session.lifetime() > ttl:
+            logging.debug(f'session\'s lifetime has expired, so the session is recreated (session_id={session_id})')
+            session, fresh = self.create(session_id, force_new=True)
+
+        return session, fresh
+
+    def session_ids(self) -> list[str]:
+        return list(self.sessions.keys())

src/tests.py

@@ -23,6 +23,7 @@ class TestFlareSolverr(unittest.TestCase):
     cloudflare_url = "https://nowsecure.nl"
     cloudflare_url_2 = "https://idope.se/torrent-list/harry/"
     ddos_guard_url = "https://anidex.info/"
+    fairlane_url = "https://www.pararius.com/apartments/amsterdam"
     custom_cloudflare_url = "https://www.muziekfabriek.org"
     cloudflare_blocked_url = "https://cpasbiens3.fr/index.php?do=search&subaction=search"
 
@@ -166,6 +167,32 @@ class TestFlareSolverr(unittest.TestCase):
         self.assertIsNotNone(cf_cookie, "DDOS-Guard cookie not found")
         self.assertGreater(len(cf_cookie["value"]), 10)
 
+    def test_v1_endpoint_request_get_fairlane_js(self):
+        res = self.app.post_json('/v1', {
+            "cmd": "request.get",
+            "url": self.fairlane_url
+        })
+        self.assertEqual(res.status_code, 200)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_OK, body.status)
+        self.assertEqual("Challenge solved!", body.message)
+        self.assertGreater(body.startTimestamp, 10000)
+        self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
+        self.assertEqual(utils.get_flaresolverr_version(), body.version)
+
+        solution = body.solution
+        self.assertIn(self.fairlane_url, solution.url)
+        self.assertEqual(solution.status, 200)
+        self.assertIs(len(solution.headers), 0)
+        self.assertIn("<title>Rental Apartments Amsterdam</title>", solution.response)
+        self.assertGreater(len(solution.cookies), 0)
+        self.assertIn("Chrome/", solution.userAgent)
+
+        cf_cookie = _find_obj_by_key("name", "fl_pass_v2_b", solution.cookies)
+        self.assertIsNotNone(cf_cookie, "Fairlane cookie not found")
+        self.assertGreater(len(cf_cookie["value"]), 50)
+
     def test_v1_endpoint_request_get_custom_cloudflare_js(self):
         res = self.app.post_json('/v1', {
             "cmd": "request.get",
@@ -351,12 +378,85 @@ class TestFlareSolverr(unittest.TestCase):
         self.assertEqual(STATUS_OK, body.status)
         self.assertEqual("Challenge not detected!", body.message)
 
-    # todo: test Cmd 'sessions.create' should return OK
-    # todo: test Cmd 'sessions.create' should return OK with session
-    # todo: test Cmd 'sessions.list' should return OK
-    # todo: test Cmd 'sessions.destroy' should return OK
-    # todo: test Cmd 'sessions.destroy' should fail
-    # todo: test Cmd 'request.get' should use session
+    def test_v1_endpoint_sessions_create_without_session(self):
+        res = self.app.post_json('/v1', {
+            "cmd": "sessions.create"
+        })
+        self.assertEqual(res.status_code, 200)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_OK, body.status)
+        self.assertEqual("Session created successfully.", body.message)
+        self.assertIsNotNone(body.session)
+
+    def test_v1_endpoint_sessions_create_with_session(self):
+        res = self.app.post_json('/v1', {
+            "cmd": "sessions.create",
+            "session": "test_create_session"
+        })
+        self.assertEqual(res.status_code, 200)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_OK, body.status)
+        self.assertEqual("Session created successfully.", body.message)
+        self.assertEqual(body.session, "test_create_session")
+
+    def test_v1_endpoint_sessions_list(self):
+        self.app.post_json('/v1', {
+            "cmd": "sessions.create",
+            "session": "test_list_sessions"
+        })
+        res = self.app.post_json('/v1', {
+            "cmd": "sessions.list"
+        })
+        self.assertEqual(res.status_code, 200)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_OK, body.status)
+        self.assertEqual("", body.message)
+        self.assertGreaterEqual(len(body.sessions), 1)
+        self.assertIn("test_list_sessions", body.sessions)
+
+    def test_v1_endpoint_sessions_destroy_existing_session(self):
+        self.app.post_json('/v1', {
+            "cmd": "sessions.create",
+            "session": "test_destroy_sessions"
+        })
+        res = self.app.post_json('/v1', {
+            "cmd": "sessions.destroy",
+            "session": "test_destroy_sessions"
+        })
+        self.assertEqual(res.status_code, 200)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_OK, body.status)
+        self.assertEqual("The session has been removed.", body.message)
+
+    def test_v1_endpoint_sessions_destroy_non_existing_session(self):
+        res = self.app.post_json('/v1', {
+            "cmd": "sessions.destroy",
+            "session": "non_existing_session_name"
+        }, status=500)
+        self.assertEqual(res.status_code, 500)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_ERROR, body.status)
+        self.assertEqual("Error: The session doesn't exist.", body.message)
+
+    def test_v1_endpoint_request_get_with_session(self):
+        self.app.post_json('/v1', {
+            "cmd": "sessions.create",
+            "session": "test_request_sessions"
+        })
+        res = self.app.post_json('/v1', {
+            "cmd": "request.get",
+            "session": "test_request_sessions",
+            "url": self.google_url
+        })
+        self.assertEqual(res.status_code, 200)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_OK, body.status)
 
 
 if __name__ == '__main__':

src/undetected_chromedriver/__init__.py

@@ -17,7 +17,7 @@ by UltrafunkAmsterdam (https://github.com/ultrafunkamsterdam)
 from __future__ import annotations
 
 
-__version__ = "3.2.1"
+__version__ = "3.4.6"
 
 import json
 import logging
@@ -239,13 +239,13 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
 
         finalize(self, self._ensure_close, self)
         self.debug = debug
-        patcher = Patcher(
+        self.patcher = Patcher(
             executable_path=driver_executable_path,
             force=patcher_force_close,
             version_main=version_main,
         )
-        patcher.auto()
-        self.patcher = patcher
+        self.patcher.auto()
+        # self.patcher = patcher
         if not options:
             options = ChromeOptions()
 
@@ -287,6 +287,11 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
 
         # see if a custom user profile is specified in options
         for arg in options.arguments:
+
+            if any([_ in arg for _ in ("--headless", "headless")]):
+                options.arguments.remove(arg)
+                options.headless = True
+
             if "lang" in arg:
                 m = re.search("(?:--)?lang(?:[ =])?(.*)", arg)
                 try:
@@ -365,13 +370,18 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
             options.arguments.extend(["--no-default-browser-check", "--no-first-run"])
         if no_sandbox:
             options.arguments.extend(["--no-sandbox", "--test-type"])
+
         if headless or options.headless:
-            options.headless = True
-            options.add_argument("--window-size=1920,1080")
-            options.add_argument("--start-maximized")
-            options.add_argument("--no-sandbox")
-            # fixes "could not connect to chrome" error when running
-            # on linux using privileged user like root (which i don't recommend)
+            if self.patcher.version_main < 108:
+                options.add_argument("--headless=chrome")
+            elif self.patcher.version_main >= 108:
+                options.add_argument("--headless=new")
+
+        options.add_argument("--window-size=1920,1080")
+        options.add_argument("--start-maximized")
+        options.add_argument("--no-sandbox")
+        # fixes "could not connect to chrome" error when running
+        # on linux using privileged user like root (which i don't recommend)
 
         options.add_argument(
             "--log-level=%d" % log_level
@@ -424,7 +434,7 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
 
         if service_creationflags:
             service = selenium.webdriver.common.service.Service(
-                patcher.executable_path, port, service_args, service_log_path
+                self.patcher.executable_path, port, service_args, service_log_path
             )
             for attr_name in ("creationflags", "creation_flags"):
                 if hasattr(service, attr_name):
@@ -434,7 +444,7 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
             service = None
 
         super(Chrome, self).__init__(
-            executable_path=patcher.executable_path,
+            executable_path=self.patcher.executable_path,
             port=port,
             options=options,
             service_args=service_args,
@@ -475,18 +485,18 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
                     {
                         "source": """
 
-                            Object.defineProperty(window, 'navigator', {
-                                value: new Proxy(navigator, {
-                                        has: (target, key) => (key === 'webdriver' ? false : key in target),
-                                        get: (target, key) =>
-                                                key === 'webdriver' ?
-                                                false :
-                                                typeof target[key] === 'function' ?
-                                                target[key].bind(target) :
-                                                target[key]
-                                        })
-                            });
-
+                           Object.defineProperty(window, "navigator", {
+                                Object.defineProperty(window, "navigator", {
+                                  value: new Proxy(navigator, {
+                                    has: (target, key) => (key === "webdriver" ? false : key in target),
+                                    get: (target, key) =>
+                                      key === "webdriver"
+                                        ? false
+                                        : typeof target[key] === "function"
+                                        ? target[key].bind(target)
+                                        : target[key],
+                                  }),
+                                });
                     """
                     },
                 )
@@ -605,37 +615,38 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
 
         self.get = get_wrapped
 
-    def _get_cdc_props(self):
-        return self.execute_script(
-            """
-            let objectToInspect = window,
-                result = [];
-            while(objectToInspect !== null)
-            { result = result.concat(Object.getOwnPropertyNames(objectToInspect));
-              objectToInspect = Object.getPrototypeOf(objectToInspect); }
-            return result.filter(i => i.match(/.+_.+_(Array|Promise|Symbol)/ig))
-            """
-        )
-
-    def _hook_remove_cdc_props(self):
-        self.execute_cdp_cmd(
-            "Page.addScriptToEvaluateOnNewDocument",
-            {
-                "source": """
-                    let objectToInspect = window,
-                        result = [];
-                    while(objectToInspect !== null)
-                    { result = result.concat(Object.getOwnPropertyNames(objectToInspect));
-                      objectToInspect = Object.getPrototypeOf(objectToInspect); }
-                    result.forEach(p => p.match(/.+_.+_(Array|Promise|Symbol)/ig)
-                                        &&delete window[p]&&console.log('removed',p))
-                    """
-            },
-        )
+    # def _get_cdc_props(self):
+    #     return self.execute_script(
+    #         """
+    #         let objectToInspect = window,
+    #             result = [];
+    #         while(objectToInspect !== null)
+    #         { result = result.concat(Object.getOwnPropertyNames(objectToInspect));
+    #           objectToInspect = Object.getPrototypeOf(objectToInspect); }
+    #
+    #         return result.filter(i => i.match(/^([a-zA-Z]){27}(Array|Promise|Symbol)$/ig))
+    #         """
+    #     )
+    #
+    # def _hook_remove_cdc_props(self):
+    #     self.execute_cdp_cmd(
+    #         "Page.addScriptToEvaluateOnNewDocument",
+    #         {
+    #             "source": """
+    #                 let objectToInspect = window,
+    #                     result = [];
+    #                 while(objectToInspect !== null)
+    #                 { result = result.concat(Object.getOwnPropertyNames(objectToInspect));
+    #                   objectToInspect = Object.getPrototypeOf(objectToInspect); }
+    #                 result.forEach(p => p.match(/^([a-zA-Z]){27}(Array|Promise|Symbol)$/ig)
+    #                                     &&delete window[p]&&console.log('removed',p))
+    #                 """
+    #         },
+    #     )
 
     def get(self, url):
-        if self._get_cdc_props():
-            self._hook_remove_cdc_props()
+        # if self._get_cdc_props():
+        #     self._hook_remove_cdc_props()
         return super().get(url)
 
     def add_cdp_listener(self, event_name, callback):

src/undetected_chromedriver/_compat.py

@@ -1,262 +0,0 @@
-#!/usr/bin/env python3
-# this module is part of undetected_chromedriver
-
-
-"""
-
-         888                                                  888         d8b
-         888                                                  888         Y8P
-         888                                                  888
- .d8888b 88888b.  888d888 .d88b.  88888b.d88b.   .d88b.   .d88888 888d888 888 888  888  .d88b.  888d888
-d88P"    888 "88b 888P"  d88""88b 888 "888 "88b d8P  Y8b d88" 888 888P"   888 888  888 d8P  Y8b 888P"
-888      888  888 888    888  888 888  888  888 88888888 888  888 888     888 Y88  88P 88888888 888
-Y88b.    888  888 888    Y88..88P 888  888  888 Y8b.     Y88b 888 888     888  Y8bd8P  Y8b.     888
- "Y8888P 888  888 888     "Y88P"  888  888  888  "Y8888   "Y88888 888     888   Y88P    "Y8888  888   88888888
-
-by UltrafunkAmsterdam (https://github.com/ultrafunkamsterdam)
-
-"""
-
-from distutils.version import LooseVersion
-import io
-import logging
-import os
-import random
-import re
-import string
-import sys
-from urllib.request import urlopen
-from urllib.request import urlretrieve
-import zipfile
-
-from selenium.webdriver import Chrome as _Chrome
-from selenium.webdriver import ChromeOptions as _ChromeOptions
-
-
-TARGET_VERSION = 0
-logger = logging.getLogger("uc")
-
-
-class Chrome:
-    def __new__(cls, *args, emulate_touch=False, **kwargs):
-
-        if not ChromeDriverManager.installed:
-            ChromeDriverManager(*args, **kwargs).install()
-        if not ChromeDriverManager.selenium_patched:
-            ChromeDriverManager(*args, **kwargs).patch_selenium_webdriver()
-        if not kwargs.get("executable_path"):
-            kwargs["executable_path"] = "./{}".format(
-                ChromeDriverManager(*args, **kwargs).executable_path
-            )
-        if not kwargs.get("options"):
-            kwargs["options"] = ChromeOptions()
-        instance = object.__new__(_Chrome)
-        instance.__init__(*args, **kwargs)
-
-        instance._orig_get = instance.get
-
-        def _get_wrapped(*args, **kwargs):
-            if instance.execute_script("return navigator.webdriver"):
-                instance.execute_cdp_cmd(
-                    "Page.addScriptToEvaluateOnNewDocument",
-                    {
-                        "source": """
-
-                                   Object.defineProperty(window, 'navigator', {
-                                       value: new Proxy(navigator, {
-                                       has: (target, key) => (key === 'webdriver' ? false : key in target),
-                                       get: (target, key) =>
-                                           key === 'webdriver'
-                                           ? undefined
-                                           : typeof target[key] === 'function'
-                                           ? target[key].bind(target)
-                                           : target[key]
-                                       })
-                                   });
-                                    
-                                                            
-                        """
-                    },
-                )
-            return instance._orig_get(*args, **kwargs)
-
-        instance.get = _get_wrapped
-        instance.get = _get_wrapped
-        instance.get = _get_wrapped
-
-        original_user_agent_string = instance.execute_script(
-            "return navigator.userAgent"
-        )
-        instance.execute_cdp_cmd(
-            "Network.setUserAgentOverride",
-            {
-                "userAgent": original_user_agent_string.replace("Headless", ""),
-            },
-        )
-        if emulate_touch:
-            instance.execute_cdp_cmd(
-                "Page.addScriptToEvaluateOnNewDocument",
-                {
-                    "source": """
-                                   Object.defineProperty(navigator, 'maxTouchPoints', {
-                                       get: () => 1
-                               })"""
-                },
-            )
-        logger.info(f"starting undetected_chromedriver.Chrome({args}, {kwargs})")
-        return instance
-
-
-class ChromeOptions:
-    def __new__(cls, *args, **kwargs):
-        if not ChromeDriverManager.installed:
-            ChromeDriverManager(*args, **kwargs).install()
-        if not ChromeDriverManager.selenium_patched:
-            ChromeDriverManager(*args, **kwargs).patch_selenium_webdriver()
-
-        instance = object.__new__(_ChromeOptions)
-        instance.__init__()
-        instance.add_argument("start-maximized")
-        instance.add_experimental_option("excludeSwitches", ["enable-automation"])
-        instance.add_argument("--disable-blink-features=AutomationControlled")
-        return instance
-
-
-class ChromeDriverManager(object):
-    installed = False
-    selenium_patched = False
-    target_version = None
-
-    DL_BASE = "https://chromedriver.storage.googleapis.com/"
-
-    def __init__(self, executable_path=None, target_version=None, *args, **kwargs):
-
-        _platform = sys.platform
-
-        if TARGET_VERSION:
-            # use global if set
-            self.target_version = TARGET_VERSION
-
-        if target_version:
-            # use explicitly passed target
-            self.target_version = target_version  # user override
-
-        if not self.target_version:
-            # none of the above (default) and just get current version
-            self.target_version = self.get_release_version_number().version[
-                0
-            ]  # only major version int
-
-        self._base = base_ = "chromedriver{}"
-
-        exe_name = self._base
-        if _platform in ("win32",):
-            exe_name = base_.format(".exe")
-        if _platform in ("linux",):
-            _platform += "64"
-            exe_name = exe_name.format("")
-        if _platform in ("darwin",):
-            _platform = "mac64"
-            exe_name = exe_name.format("")
-        self.platform = _platform
-        self.executable_path = executable_path or exe_name
-        self._exe_name = exe_name
-
-    def patch_selenium_webdriver(self_):
-        """
-        Patches selenium package Chrome, ChromeOptions classes for current session
-
-        :return:
-        """
-        import selenium.webdriver.chrome.service
-        import selenium.webdriver
-
-        selenium.webdriver.Chrome = Chrome
-        selenium.webdriver.ChromeOptions = ChromeOptions
-        logger.info("Selenium patched. Safe to import Chrome / ChromeOptions")
-        self_.__class__.selenium_patched = True
-
-    def install(self, patch_selenium=True):
-        """
-        Initialize the patch
-
-        This will:
-         download chromedriver if not present
-         patch the downloaded chromedriver
-         patch selenium package if <patch_selenium> is True (default)
-
-        :param patch_selenium: patch selenium webdriver classes for Chrome and ChromeDriver (for current python session)
-        :return:
-        """
-        if not os.path.exists(self.executable_path):
-            self.fetch_chromedriver()
-            if not self.__class__.installed:
-                if self.patch_binary():
-                    self.__class__.installed = True
-
-        if patch_selenium:
-            self.patch_selenium_webdriver()
-
-    def get_release_version_number(self):
-        """
-        Gets the latest major version available, or the latest major version of self.target_version if set explicitly.
-
-        :return: version string
-        """
-        path = (
-            "LATEST_RELEASE"
-            if not self.target_version
-            else f"LATEST_RELEASE_{self.target_version}"
-        )
-        return LooseVersion(urlopen(self.__class__.DL_BASE + path).read().decode())
-
-    def fetch_chromedriver(self):
-        """
-        Downloads ChromeDriver from source and unpacks the executable
-
-        :return: on success, name of the unpacked executable
-        """
-        base_ = self._base
-        zip_name = base_.format(".zip")
-        ver = self.get_release_version_number().vstring
-        if os.path.exists(self.executable_path):
-            return self.executable_path
-        urlretrieve(
-            f"{self.__class__.DL_BASE}{ver}/{base_.format(f'_{self.platform}')}.zip",
-            filename=zip_name,
-        )
-        with zipfile.ZipFile(zip_name) as zf:
-            zf.extract(self._exe_name)
-        os.remove(zip_name)
-        if sys.platform != "win32":
-            os.chmod(self._exe_name, 0o755)
-        return self._exe_name
-
-    @staticmethod
-    def random_cdc():
-        cdc = random.choices(string.ascii_lowercase, k=26)
-        cdc[-6:-4] = map(str.upper, cdc[-6:-4])
-        cdc[2] = cdc[0]
-        cdc[3] = "_"
-        return "".join(cdc).encode()
-
-    def patch_binary(self):
-        """
-        Patches the ChromeDriver binary
-
-        :return: False on failure, binary name on success
-        """
-        linect = 0
-        replacement = self.random_cdc()
-        with io.open(self.executable_path, "r+b") as fh:
-            for line in iter(lambda: fh.readline(), b""):
-                if b"cdc_" in line:
-                    fh.seek(-len(line), 1)
-                    newline = re.sub(b"cdc_.{22}", replacement, line)
-                    fh.write(newline)
-                    linect += 1
-            return linect
-
-
-def install(executable_path=None, target_version=None, *args, **kwargs):
-    ChromeDriverManager(executable_path, target_version, *args, **kwargs).install()

src/undetected_chromedriver/dprocess.py

@@ -46,7 +46,6 @@ def start_detached(executable, *args):
 
 
 def _start_detached(executable, *args, writer: multiprocessing.Pipe = None):
-
     # configure launch
     kwargs = {}
     if platform.system() == "Windows":

src/undetected_chromedriver/options.py

@@ -56,7 +56,6 @@ class ChromeOptions(_ChromiumOptions):
     def handle_prefs(self, user_data_dir):
         prefs = self.experimental_options.get("prefs")
         if prefs:
-
             user_data_dir = user_data_dir or self._user_data_dir
             default_path = os.path.join(user_data_dir, "Default")
             os.makedirs(default_path, exist_ok=True)

src/undetected_chromedriver/patcher.py

@@ -7,7 +7,6 @@ import logging
 import os
 import random
 import re
-import secrets
 import string
 import sys
 import time
@@ -41,7 +40,7 @@ class Patcher(object):
         d = "~/appdata/roaming/undetected_chromedriver"
     elif "LAMBDA_TASK_ROOT" in os.environ:
         d = "/tmp/undetected_chromedriver"
-    elif platform.startswith(("linux","linux2")):
+    elif platform.startswith(("linux", "linux2")):
         d = "~/.local/share/undetected_chromedriver"
     elif platform.endswith("darwin"):
         d = "~/Library/Application Support/undetected_chromedriver"
@@ -51,7 +50,6 @@ class Patcher(object):
 
     def __init__(self, executable_path=None, force=False, version_main: int = 0):
         """
-
         Args:
             executable_path: None = automatic
                              a full file path to the chromedriver executable
@@ -60,10 +58,9 @@ class Patcher(object):
             version_main: 0 = auto
                 specify main chrome version (rounded, ex: 82)
         """
-
         self.force = force
-        self.executable_path = None
-        prefix = secrets.token_hex(8)
+        self._custom_exe_path = False
+        prefix = "undetected"
 
         if not os.path.exists(self.data_path):
             os.makedirs(self.data_path, exist_ok=True)
@@ -85,8 +82,6 @@ class Patcher(object):
                 os.path.join(".", self.executable_path)
             )
 
-        self._custom_exe_path = False
-
         if executable_path:
             self._custom_exe_path = True
             self.executable_path = executable_path
@@ -94,7 +89,6 @@ class Patcher(object):
         self.version_full = None
 
     def auto(self, executable_path=None, force=False, version_main=None):
-        """"""
         if executable_path:
             self.executable_path = executable_path
             self._custom_exe_path = True
@@ -206,43 +200,46 @@ class Patcher(object):
 
     @staticmethod
     def gen_random_cdc():
-        cdc = random.choices(string.ascii_lowercase, k=26)
-        cdc[-6:-4] = map(str.upper, cdc[-6:-4])
-        cdc[2] = cdc[0]
-        cdc[3] = "_"
+        cdc = random.choices(string.ascii_letters, k=27)
         return "".join(cdc).encode()
 
     def is_binary_patched(self, executable_path=None):
-        """simple check if executable is patched.
-
-        :return: False if not patched, else True
-        """
         executable_path = executable_path or self.executable_path
-        with io.open(executable_path, "rb") as fh:
-            for line in iter(lambda: fh.readline(), b""):
-                if b"cdc_" in line:
-                    return False
-            else:
-                return True
+        try:
+            with io.open(executable_path, "rb") as fh:
+                return fh.read().find(b"undetected chromedriver") != -1
+        except FileNotFoundError:
+            return False
 
     def patch_exe(self):
-        """
-        Patches the ChromeDriver binary
-
-        :return: False on failure, binary name on success
-        """
+        start = time.perf_counter()
         logger.info("patching driver executable %s" % self.executable_path)
-
-        linect = 0
-        replacement = self.gen_random_cdc()
         with io.open(self.executable_path, "r+b") as fh:
-            for line in iter(lambda: fh.readline(), b""):
-                if b"cdc_" in line:
-                    fh.seek(-len(line), 1)
-                    newline = re.sub(b"cdc_.{22}", replacement, line)
-                    fh.write(newline)
-                    linect += 1
-            return linect
+            content = fh.read()
+            # match_injected_codeblock = re.search(rb"{window.*;}", content)
+            match_injected_codeblock = re.search(rb"\{window\.cdc.*?;\}", content)
+            if match_injected_codeblock:
+                target_bytes = match_injected_codeblock[0]
+                new_target_bytes = (
+                    b'{console.log("undetected chromedriver 1337!")}'.ljust(
+                        len(target_bytes), b" "
+                    )
+                )
+                new_content = content.replace(target_bytes, new_target_bytes)
+                if new_content == content:
+                    logger.warning(
+                        "something went wrong patching the driver binary. could not find injection code block"
+                    )
+                else:
+                    logger.debug(
+                        "found block:\n%s\nreplacing with:\n%s"
+                        % (target_bytes, new_target_bytes)
+                    )
+                fh.seek(0)
+                fh.write(new_content)
+        logger.debug(
+            "patching took us {:.2f} seconds".format(time.perf_counter() - start)
+        )
 
     def __repr__(self):
         return "{0:s}({1:s})".format(
@@ -251,7 +248,6 @@ class Patcher(object):
         )
 
     def __del__(self):
-
         if self._custom_exe_path:
             # if the driver binary is specified by user
             # we assume it is important enough to not delete it

src/undetected_chromedriver/reactor.py

@@ -6,6 +6,7 @@ import json
 import logging
 import threading
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -63,9 +64,7 @@ class Reactor(threading.Thread):
                     break
 
     async def listen(self):
-
         while self.running:
-
             await self._wait_service_started()
             await asyncio.sleep(1)
 
@@ -74,9 +73,7 @@ class Reactor(threading.Thread):
                     log_entries = self.driver.get_log("performance")
 
                 for entry in log_entries:
-
                     try:
-
                         obj_serialized: str = entry.get("message")
                         obj = json.loads(obj_serialized)
                         message = obj.get("message")

src/undetected_chromedriver/v2.py

@@ -1,4 +0,0 @@
-# for backward compatibility
-import sys
-
-sys.modules[__name__] = sys.modules[__package__]

src/undetected_chromedriver/webelement.py

@@ -1,6 +1,7 @@
+from typing import List
+
 from selenium.webdriver.common.by import By
 import selenium.webdriver.remote.webelement
-from typing import List
 
 
 class WebElement(selenium.webdriver.remote.webelement.WebElement):

src/utils.py

@@ -8,6 +8,7 @@ from selenium.webdriver.chrome.webdriver import WebDriver
 import undetected_chromedriver as uc
 
 FLARESOLVERR_VERSION = None
+CHROME_EXE_PATH = None
 CHROME_MAJOR_VERSION = None
 USER_AGENT = None
 XVFB_DISPLAY = None
@@ -28,6 +29,8 @@ def get_flaresolverr_version() -> str:
         return FLARESOLVERR_VERSION
 
     package_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'package.json')
+    if not os.path.isfile(package_path):
+        package_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'package.json')
     with open(package_path) as f:
         FLARESOLVERR_VERSION = json.loads(f.read())['version']
         return FLARESOLVERR_VERSION
@@ -46,6 +49,11 @@ def get_webdriver() -> WebDriver:
     options.add_argument('--disable-dev-shm-usage')
     # this option removes the zygote sandbox (it seems that the resolution is a bit faster)
     options.add_argument('--no-zygote')
+    # attempt to fix Docker ARM32 build
+    options.add_argument('--disable-gpu-sandbox')
+    options.add_argument('--disable-software-rasterizer')
+    options.add_argument('--ignore-certificate-errors')
+    options.add_argument('--ignore-ssl-errors')
 
     # note: headless mode is detected (options.headless = True)
     # we launch the browser in head-full mode with the window hidden
@@ -67,9 +75,13 @@ def get_webdriver() -> WebDriver:
         if PATCHED_DRIVER_PATH is not None:
             driver_exe_path = PATCHED_DRIVER_PATH
 
+    # detect chrome path
+    browser_executable_path = get_chrome_exe_path()
+
     # downloads and patches the chromedriver
     # if we don't set driver_executable_path it downloads, patches, and deletes the driver each time
-    driver = uc.Chrome(options=options, driver_executable_path=driver_exe_path, version_main=version_main,
+    driver = uc.Chrome(options=options, browser_executable_path=browser_executable_path,
+                       driver_executable_path=driver_exe_path, version_main=version_main,
                        windows_headless=windows_headless)
 
     # save the patched driver to avoid re-downloads
@@ -88,23 +100,42 @@ def get_webdriver() -> WebDriver:
     return driver
 
 
+def get_chrome_exe_path() -> str:
+    global CHROME_EXE_PATH
+    if CHROME_EXE_PATH is not None:
+        return CHROME_EXE_PATH
+    # linux pyinstaller bundle
+    chrome_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'chrome', "chrome")
+    if os.path.exists(chrome_path) and os.access(chrome_path, os.X_OK):
+        CHROME_EXE_PATH = chrome_path
+        return CHROME_EXE_PATH
+    # windows pyinstaller bundle
+    chrome_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'chrome', "chrome.exe")
+    if os.path.exists(chrome_path) and os.access(chrome_path, os.X_OK):
+        CHROME_EXE_PATH = chrome_path
+        return CHROME_EXE_PATH
+    # system
+    CHROME_EXE_PATH = uc.find_chrome_executable()
+    return CHROME_EXE_PATH
+
+
 def get_chrome_major_version() -> str:
     global CHROME_MAJOR_VERSION
     if CHROME_MAJOR_VERSION is not None:
         return CHROME_MAJOR_VERSION
 
     if os.name == 'nt':
+        # Example: '104.0.5112.79'
         try:
-            stream = os.popen(
-                'reg query "HKLM\\SOFTWARE\\Wow6432Node\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\Google Chrome"')
-            output = stream.read()
-            # Example: '104.0.5112.79'
-            complete_version = extract_version_registry(output)
+            complete_version = extract_version_nt_executable(get_chrome_exe_path())
         except Exception:
-            # Example: '104.0.5112.79'
-            complete_version = extract_version_folder()
+            try:
+                complete_version = extract_version_nt_registry()
+            except Exception:
+                # Example: '104.0.5112.79'
+                complete_version = extract_version_nt_folder()
     else:
-        chrome_path = uc.find_chrome_executable()
+        chrome_path = get_chrome_exe_path()
         process = os.popen(f'"{chrome_path}" --version')
         # Example 1: 'Chromium 104.0.5112.79 Arch Linux\n'
         # Example 2: 'Google Chrome 104.0.5112.79 Arch Linux\n'
@@ -112,24 +143,32 @@ def get_chrome_major_version() -> str:
         process.close()
 
     CHROME_MAJOR_VERSION = complete_version.split('.')[0].split(' ')[-1]
-    logging.info(f"Chrome major version: {CHROME_MAJOR_VERSION}")
     return CHROME_MAJOR_VERSION
 
 
-def extract_version_registry(output) -> str:
-    try:
-        google_version = ''
-        for letter in output[output.rindex('DisplayVersion    REG_SZ') + 24:]:
-            if letter != '\n':
-                google_version += letter
-            else:
-                break
-        return google_version.strip()
-    except TypeError:
-        return ''
+def extract_version_nt_executable(exe_path: str) -> str:
+    import pefile
+    pe = pefile.PE(exe_path, fast_load=True)
+    pe.parse_data_directories(
+        directories=[pefile.DIRECTORY_ENTRY["IMAGE_DIRECTORY_ENTRY_RESOURCE"]]
+    )
+    return pe.FileInfo[0][0].StringTable[0].entries[b"FileVersion"].decode('utf-8')
 
 
-def extract_version_folder() -> str:
+def extract_version_nt_registry() -> str:
+    stream = os.popen(
+        'reg query "HKLM\\SOFTWARE\\Wow6432Node\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\Google Chrome"')
+    output = stream.read()
+    google_version = ''
+    for letter in output[output.rindex('DisplayVersion    REG_SZ') + 24:]:
+        if letter != '\n':
+            google_version += letter
+        else:
+            break
+    return google_version.strip()
+
+
+def extract_version_nt_folder() -> str:
     # Check if the Chrome folder exists in the x32 or x64 Program Files folders.
     for i in range(2):
         path = 'C:\\Program Files' + (' (x86)' if i else '') + '\\Google\\Chrome\\Application'