Fix too many open files error. resolves #983 (#1033)

Co-authored-by: ilike2burnthing <59480337+ilike2burnthing@users.noreply.github.com>

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -8,6 +8,13 @@ body:
     options:
       - label: I have checked the README
         required: true
+- type: checkboxes
+  attributes:
+    label: Have you followed our Troubleshooting?
+    description: Please follow our <a href="https://github.com/FlareSolverr/FlareSolverr/wiki/Troubleshooting">Troubleshooting</a>.
+    options:
+      - label: I have followed your Troubleshooting
+        required: true
 - type: checkboxes
   attributes:
     label: Is there already an issue for your problem?

.github/workflows/autotag.yml

@@ -6,7 +6,7 @@ on:
       - "master"
 
 jobs:
-  build:
+  tag-release:
     runs-on: ubuntu-latest
     steps:
       -

.github/workflows/release-docker.yml

@@ -6,7 +6,7 @@ on:
       - 'v*.*.*'
 
 jobs:
-  build:
+  build-docker-images:
     runs-on: ubuntu-22.04
     steps:
       -

.github/workflows/release.yml

@@ -36,7 +36,7 @@ jobs:
           draft: false
           prerelease: false
 
-  build-linux:
+  build-linux-package:
     name: Build Linux binary
     needs: create-release
     runs-on: ubuntu-22.04
@@ -54,7 +54,7 @@ jobs:
       - name: Build artifacts
         run: |
           python -m pip install -r requirements.txt
-          python -m pip install pyinstaller==5.9.0
+          python -m pip install pyinstaller==5.13.0
           cd src
           python build_package.py
 
@@ -65,7 +65,7 @@ jobs:
         with:
           asset_paths: '["./dist/flaresolverr_*"]'
 
-  build-windows:
+  build-windows-package:
     name: Build Windows binary
     needs: create-release
     runs-on: windows-2022
@@ -83,7 +83,7 @@ jobs:
       - name: Build artifacts
         run: |
           python -m pip install -r requirements.txt
-          python -m pip install pyinstaller==5.9.0
+          python -m pip install pyinstaller==5.13.0
           cd src
           python build_package.py
 

.gitignore

@@ -124,3 +124,6 @@ venv.bak/
 .mypy_cache/
 .dmypy.json
 dmypy.json
+
+# node
+node_modules/

CHANGELOG.md

@@ -1,5 +1,106 @@
 # Changelog
 
+## v3.3.13 (2024/01/07)
+
+* Fix too many open files error
+
+## v3.3.12 (2023/12/15)
+
+* Fix looping challenges and invalid cookies
+
+## v3.3.11 (2023/12/11)
+
+* Update UC 3.5.4 & Selenium 4.15.2. Thanks @txtsd
+
+## v3.3.10 (2023/11/14)
+
+* Add LANG ENV - resolves issues with YGGtorrent
+
+## v3.3.9 (2023/11/13)
+
+* Fix for Docker build, capture TypeError
+
+## v3.3.8 (2023/11/13)
+
+* Fix headless=true for Chrome 117+. Thanks @NabiKAZ
+* Support running Chrome 119 from source. Thanks @koleg and @Chris7X
+* Fix "OSError: [WinError 6] The handle is invalid" on exit. Thanks @enesgorkemgenc
+
+## v3.3.7 (2023/11/05)
+
+* Bump to rebuild. Thanks @JoachimDorchies
+
+## v3.3.6 (2023/09/15)
+
+* Update checkbox selector, again
+
+## v3.3.5 (2023/09/13)
+
+* Change checkbox selector, support languages other than English
+
+## v3.3.4 (2023/09/02)
+
+* Update checkbox selector
+
+## v3.3.3 (2023/08/31)
+
+* Update undetected_chromedriver to v3.5.3
+
+## v3.3.2 (2023/08/03)
+
+* Fix URL domain in Prometheus exporter
+
+## v3.3.1 (2023/08/03)
+
+* Fix for Cloudflare verify checkbox
+* Fix HEADLESS=false in Windows binary
+* Fix Prometheus exporter for management and health endpoints
+* Remove misleading stack trace when the verify checkbox is not found
+* Revert "Update base Docker image to Debian Bookworm" #849
+* Revert "Install Chromium 115 from Debian testing" #849
+
+## v3.3.0 (2023/08/02)
+
+* Fix for new Cloudflare detection. Thanks @cedric-bour for #845
+* Add support for proxy authentication username/password. Thanks @jacobprice808	for #807
+* Implement Prometheus metrics
+* Fix Chromium Driver for Chrome / Chromium version > 114
+* Use Chromium 115 in binary packages (Windows and Linux)
+* Install Chromium 115 from Debian testing (Docker)
+* Update base Docker image to Debian Bookworm
+* Update Selenium 4.11.2
+* Update pyinstaller 5.13.0
+* Add more traces in build_package.py
+
+## v3.2.2 (2023/07/16)
+
+* Workaround for updated 'verify you are human' check
+
+## v3.2.1 (2023/06/10)
+
+* Kill dead Chrome processes in Windows
+* Fix Chrome GL erros in ASUSTOR NAS
+
+## v3.2.0 (2023/05/23)
+
+* Support "proxy" param in requests and sessions
+* Support "cookies" param in requests
+* Fix Chromium exec permissions in Linux package
+* Update Python dependencies
+
+## v3.1.2 (2023/04/02)
+
+* Fix headless mode in macOS
+* Remove redundant artifact from Windows binary package
+* Bump Selenium dependency
+
+## v3.1.1 (2023/03/25)
+
+* Distribute binary executables in compressed package
+* Add icon for binary executable
+* Include information about supported architectures in the readme
+* Check Python version on start
+
 ## v3.1.0 (2023/03/20)
 
 * Build binaries for Linux x64 and Windows x64

Dockerfile

@@ -54,6 +54,7 @@ COPY src .
 COPY package.json ../
 
 EXPOSE 8191
+EXPOSE 8192
 
 # dumb-init avoids zombie chromium processes
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]
@@ -61,17 +62,17 @@ ENTRYPOINT ["/usr/bin/dumb-init", "--"]
 CMD ["/usr/local/bin/python", "-u", "/app/flaresolverr.py"]
 
 # Local build
-# docker build -t ngosang/flaresolverr:3.0.0 .
+# docker build -t ngosang/flaresolverr:3.3.13 .
-# docker run -p 8191:8191 ngosang/flaresolverr:3.0.0
+# docker run -p 8191:8191 ngosang/flaresolverr:3.3.13
 
 # Multi-arch build
 # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
 # docker buildx create --use
-# docker buildx build -t ngosang/flaresolverr:3.0.0 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 .
+# docker buildx build -t ngosang/flaresolverr:3.3.13 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 .
 #   add --push to publish in DockerHub
 
 # Test multi-arch build
 # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
 # docker buildx create --use
-# docker buildx build -t ngosang/flaresolverr:3.0.0 --platform linux/arm/v7 --load .
+# docker buildx build -t ngosang/flaresolverr:3.3.13 --platform linux/arm/v7 --load .
-# docker run -p 8191:8191 --platform linux/arm/v7 ngosang/flaresolverr:3.0.0
+# docker run -p 8191:8191 --platform linux/arm/v7 ngosang/flaresolverr:3.3.13

README.md

@@ -64,12 +64,18 @@ Remember to restart the Docker daemon and the container after the update.
 
 ### Precompiled binaries
 
+> **Warning**
+> Precompiled binaries are only available for x64 architecture. For other architectures see Docker images.
+
 This is the recommended way for Windows users.
 * Download the [FlareSolverr executable](https://github.com/FlareSolverr/FlareSolverr/releases) from the release's page. It is available for Windows x64 and Linux x64.
 * Execute FlareSolverr binary. In the environment variables section you can find how to change the configuration.
 
 ### From source code
 
+> **Warning**
+> Installing from source code only works for x64 architecture. For other architectures see Docker images.
+
 * Install [Python 3.11](https://www.python.org/downloads/).
 * Install [Chrome](https://www.google.com/intl/en_us/chrome/) (all OS) or [Chromium](https://www.chromium.org/getting-involved/download-chromium/) (just Linux, it doesn't work in Windows) web browser.
 * (Only in Linux / macOS) Install [Xvfb](https://en.wikipedia.org/wiki/Xvfb) package.
@@ -104,10 +110,10 @@ cookies for the browser to use.
 
 This also speeds up the requests since it won't have to launch a new browser instance for every request.
 
-| Parameter | Notes                                                                                                                                                                                                                 |
+| Parameter | Notes                                                                                                                                                                                                                                                                                                            |
-|-----------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+|-----------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| session   | Optional. The session ID that you want to be assigned to the instance. If isn't set a random UUID will be assigned.                                                                                                   |
+| session   | Optional. The session ID that you want to be assigned to the instance. If isn't set a random UUID will be assigned.                                                                                                                                                                                              |
-| proxy     | Optional, default disabled. Eg: `"proxy": {"url": "http://127.0.0.1:8888"}`. You must include the proxy schema in the URL: `http://`, `socks4://` or `socks5://`. Authorization (username/password) is not supported. |
+| proxy     | Optional, default disabled. Eg: `"proxy": {"url": "http://127.0.0.1:8888"}`. You must include the proxy schema in the URL: `http://`, `socks4://` or `socks5://`. Authorization (username/password) is supported. Eg: `"proxy": {"url": "http://127.0.0.1:8888", "username": "testuser", "password": "testpass"}` |
 
 #### + `sessions.list`
 
@@ -144,11 +150,12 @@ session. When you no longer need to use a session you should make sure to close
 | session             | Optional. Will send the request from and existing browser instance. If one is not sent it will create a temporary instance that will be destroyed immediately after the request is completed.                                                                                                                                                |
 | session_ttl_minutes | Optional. FlareSolverr will automatically rotate expired sessions based on the TTL provided in minutes.                                                                                                                                                                                                                                      |
 | maxTimeout          | Optional, default value 60000. Max timeout to solve the challenge in milliseconds.                                                                                                                                                                                                                                                           |
-| cookies             | Optional. Will be used by the headless browser. Follow [this](https://github.com/puppeteer/puppeteer/blob/v3.3.0/docs/api.md#pagesetcookiecookies) format.                                                                                                                                                                                   |
+| cookies             | Optional. Will be used by the headless browser. Eg: `"cookies": [{"name": "cookie1", "value": "value1"}, {"name": "cookie2", "value": "value2"}]`.                                                                                                                                                                                           |
 | returnOnlyCookies   | Optional, default false. Only returns the cookies. Response data, headers and other parts of the response are removed.                                                                                                                                                                                                                       |
 | proxy               | Optional, default disabled. Eg: `"proxy": {"url": "http://127.0.0.1:8888"}`. You must include the proxy schema in the URL: `http://`, `socks4://` or `socks5://`. Authorization (username/password) is not supported. (When the `session` parameter is set, the proxy is ignored; a session specific proxy can be set in `sessions.create`.) |
 
-:warning: If you want to use Cloudflare clearance cookie in your scripts, make sure you use the FlareSolverr User-Agent too. If they don't match you will see the challenge.
+> **Warning**
+> If you want to use Cloudflare clearance cookie in your scripts, make sure you use the FlareSolverr User-Agent too. If they don't match you will see the challenge.
 
 Example response from running the `curl` above:
 
@@ -219,32 +226,62 @@ This is the same as `request.get` but it takes one more param:
 
 ## Environment variables
 
-| Name            | Default                | Notes                                                                                                                                                         |
+| Name               | Default                | Notes                                                                                                                                                         |
-|-----------------|------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------|
+|--------------------|------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| LOG_LEVEL       | info                   | Verbosity of the logging. Use `LOG_LEVEL=debug` for more information.                                                                                         |
+| LOG_LEVEL          | info                   | Verbosity of the logging. Use `LOG_LEVEL=debug` for more information.                                                                                         |
-| LOG_HTML        | false                  | Only for debugging. If `true` all HTML that passes through the proxy will be logged to the console in `debug` level.                                          |
+| LOG_HTML           | false                  | Only for debugging. If `true` all HTML that passes through the proxy will be logged to the console in `debug` level.                                          |
-| CAPTCHA_SOLVER  | none                   | Captcha solving method. It is used when a captcha is encountered. See the Captcha Solvers section.                                                            |
+| CAPTCHA_SOLVER     | none                   | Captcha solving method. It is used when a captcha is encountered. See the Captcha Solvers section.                                                            |
-| TZ              | UTC                    | Timezone used in the logs and the web browser. Example: `TZ=Europe/London`.                                                                                   |
+| TZ                 | UTC                    | Timezone used in the logs and the web browser. Example: `TZ=Europe/London`.                                                                                   |
-| HEADLESS        | true                   | Only for debugging. To run the web browser in headless mode or visible.                                                                                       |
+| LANG               | none                   | Language used in the web browser. Example: `LANG=en_GB`.                                                                                   |
-| BROWSER_TIMEOUT | 40000                  | If you are experiencing errors/timeouts because your system is slow, you can try to increase this value. Remember to increase the `maxTimeout` parameter too. |
+| HEADLESS           | true                   | Only for debugging. To run the web browser in headless mode or visible.                                                                                       |
-| TEST_URL        | https://www.google.com | FlareSolverr makes a request on start to make sure the web browser is working. You can change that URL if it is blocked in your country.                      |
+| BROWSER_TIMEOUT    | 40000                  | If you are experiencing errors/timeouts because your system is slow, you can try to increase this value. Remember to increase the `maxTimeout` parameter too. |
-| PORT            | 8191                   | Listening port. You don't need to change this if you are running on Docker.                                                                                   |
+| TEST_URL           | https://www.google.com | FlareSolverr makes a request on start to make sure the web browser is working. You can change that URL if it is blocked in your country.                      |
-| HOST            | 0.0.0.0                | Listening interface. You don't need to change this if you are running on Docker.                                                                              |
+| PORT               | 8191                   | Listening port. You don't need to change this if you are running on Docker.                                                                                   |
+| HOST               | 0.0.0.0                | Listening interface. You don't need to change this if you are running on Docker.                                                                              |
+| PROMETHEUS_ENABLED | false                  | Enable Prometheus exporter. See the Prometheus section below.                                                                                                 |
+| PROMETHEUS_PORT    | 8192                   | Listening port for Prometheus exporter. See the Prometheus section below.                                                                                     |
 
 Environment variables are set differently depending on the operating system. Some examples:
 * Docker: Take a look at the Docker section in this document. Environment variables can be set in the `docker-compose.yml` file or in the Docker CLI command.
 * Linux: Run `export LOG_LEVEL=debug` and then start FlareSolverr in the same shell.
 * Windows: Open `cmd.exe`, run `set LOG_LEVEL=debug` and then start FlareSolverr in the same shell.
 
+## Prometheus exporter
+
+The Prometheus exporter for FlareSolverr is disabled by default. It can be enabled with the environment variable `PROMETHEUS_ENABLED`. If you are using Docker make sure you expose the `PROMETHEUS_PORT`.
+
+Example metrics:
+```shell
+# HELP flaresolverr_request_total Total requests with result
+# TYPE flaresolverr_request_total counter
+flaresolverr_request_total{domain="nowsecure.nl",result="solved"} 1.0
+# HELP flaresolverr_request_created Total requests with result
+# TYPE flaresolverr_request_created gauge
+flaresolverr_request_created{domain="nowsecure.nl",result="solved"} 1.690141657157109e+09
+# HELP flaresolverr_request_duration Request duration in seconds
+# TYPE flaresolverr_request_duration histogram
+flaresolverr_request_duration_bucket{domain="nowsecure.nl",le="0.0"} 0.0
+flaresolverr_request_duration_bucket{domain="nowsecure.nl",le="10.0"} 1.0
+flaresolverr_request_duration_bucket{domain="nowsecure.nl",le="25.0"} 1.0
+flaresolverr_request_duration_bucket{domain="nowsecure.nl",le="50.0"} 1.0
+flaresolverr_request_duration_bucket{domain="nowsecure.nl",le="+Inf"} 1.0
+flaresolverr_request_duration_count{domain="nowsecure.nl"} 1.0
+flaresolverr_request_duration_sum{domain="nowsecure.nl"} 5.858
+# HELP flaresolverr_request_duration_created Request duration in seconds
+# TYPE flaresolverr_request_duration_created gauge
+flaresolverr_request_duration_created{domain="nowsecure.nl"} 1.6901416571570296e+09
+```
+
 ## Captcha Solvers
 
-:warning: At this time none of the captcha solvers work. You can check the status in the open issues. Any help is welcome.
+> **Warning**
+> At this time none of the captcha solvers work. You can check the status in the open issues. Any help is welcome.
 
 Sometimes CloudFlare not only gives mathematical computations and browser tests, sometimes they also require the user to
 solve a captcha.
 If this is the case, FlareSolverr will return the error `Captcha detected but no automatic solver is configured.`
 
-FlareSolverr can be customized to solve the captchas automatically by setting the environment variable `CAPTCHA_SOLVER`
+FlareSolverr can be customized to solve the CAPTCHA automatically by setting the environment variable `CAPTCHA_SOLVER`
 to the file name of one of the adapters inside the [/captcha](src/captcha) directory.
 
 ## Related projects

package.json

@@ -1,6 +1,6 @@
 {
   "name": "flaresolverr",
-  "version": "3.1.0",
+  "version": "3.3.13",
   "description": "Proxy server to bypass Cloudflare protection",
   "author": "Diego Heras (ngosang / ngosang@hotmail.es)",
   "license": "MIT"

requirements.txt

@@ -1,11 +1,12 @@
 bottle==0.12.25
 waitress==2.1.2
-selenium==4.8.2
+selenium==4.15.2
 func-timeout==4.3.5
+prometheus-client==0.17.1
 # required by undetected_chromedriver
-requests==2.28.2
+requests==2.31.0
-certifi==2022.12.7
+certifi==2023.7.22
-websockets==10.4
+websockets==11.0.3
 # only required for linux
 xvfbwrapper==0.2.9
 # only required for windows

src/bottle_plugins/prometheus_plugin.py

@@ -0,0 +1,66 @@
+import logging
+import os
+import urllib.parse
+
+from bottle import request
+from dtos import V1RequestBase, V1ResponseBase
+from metrics import start_metrics_http_server, REQUEST_COUNTER, REQUEST_DURATION
+
+PROMETHEUS_ENABLED = os.environ.get('PROMETHEUS_ENABLED', 'false').lower() == 'true'
+PROMETHEUS_PORT = int(os.environ.get('PROMETHEUS_PORT', 8192))
+
+
+def setup():
+    if PROMETHEUS_ENABLED:
+        start_metrics_http_server(PROMETHEUS_PORT)
+
+
+def prometheus_plugin(callback):
+    """
+    Bottle plugin to expose Prometheus metrics
+    http://bottlepy.org/docs/dev/plugindev.html
+    """
+    def wrapper(*args, **kwargs):
+        actual_response = callback(*args, **kwargs)
+
+        if PROMETHEUS_ENABLED:
+            try:
+                export_metrics(actual_response)
+            except Exception as e:
+                logging.warning("Error exporting metrics: " + str(e))
+
+        return actual_response
+
+    def export_metrics(actual_response):
+        res = V1ResponseBase(actual_response)
+
+        if res.startTimestamp is None or res.endTimestamp is None:
+            # skip management and healthcheck endpoints
+            return
+
+        domain = "unknown"
+        if res.solution and res.solution.url:
+            domain = parse_domain_url(res.solution.url)
+        else:
+            # timeout error
+            req = V1RequestBase(request.json)
+            if req.url:
+                domain = parse_domain_url(req.url)
+
+        run_time = (res.endTimestamp - res.startTimestamp) / 1000
+        REQUEST_DURATION.labels(domain=domain).observe(run_time)
+
+        result = "unknown"
+        if res.message == "Challenge solved!":
+            result = "solved"
+        elif res.message == "Challenge not detected!":
+            result = "not_detected"
+        elif res.message.startswith("Error"):
+            result = "error"
+        REQUEST_COUNTER.labels(domain=domain, result=result).inc()
+
+    def parse_domain_url(url):
+        parsed_url = urllib.parse.urlparse(url)
+        return parsed_url.hostname
+
+    return wrapper

src/build_package.py

@@ -25,8 +25,8 @@ def clean_files():
 
 def download_chromium():
     # https://commondatastorage.googleapis.com/chromium-browser-snapshots/index.html?prefix=Linux_x64/
-    revision = "1090006" if os.name == 'nt' else '1090007'
+    revision = "1140001" if os.name == 'nt' else '1140000'
-    arch = 'Win' if os.name == 'nt' else 'Linux_x64'
+    arch = 'Win_x64' if os.name == 'nt' else 'Linux_x64'
     dl_file = 'chrome-win' if os.name == 'nt' else 'chrome-linux'
     dl_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'dist_chrome')
     dl_path_folder = os.path.join(dl_path, dl_file)
@@ -50,24 +50,46 @@ def download_chromium():
     with zipfile.ZipFile(dl_path_zip, 'r') as zip_ref:
         zip_ref.extractall(dl_path)
     os.remove(dl_path_zip)
-    shutil.move(dl_path_folder, os.path.join(dl_path, "chrome"))
+
+    chrome_path = os.path.join(dl_path, "chrome")
+    shutil.move(dl_path_folder, chrome_path)
+    print("Extracted in: " + chrome_path)
+
+    if os.name != 'nt':
+        # Give executable permissions for *nix
+        # file * | grep executable | cut -d: -f1
+        print("Giving executable permissions...")
+        execs = ['chrome', 'chrome_crashpad_handler', 'chrome_sandbox', 'chrome-wrapper', 'nacl_helper',
+                 'nacl_helper_bootstrap', 'nacl_irt_x86_64.nexe', 'xdg-mime', 'xdg-settings']
+        for exec_file in execs:
+            exec_path = os.path.join(chrome_path, exec_file)
+            os.chmod(exec_path, 0o755)
 
 
 def run_pyinstaller():
     sep = ';' if os.name == 'nt' else ':'
-    subprocess.check_call([sys.executable, "-m", "PyInstaller",
+    result = subprocess.run([sys.executable, "-m", "PyInstaller",
-                           "--onefile",
+                             "--icon", "resources/flaresolverr_logo.ico",
-                           "--add-data", f"package.json{sep}.",
+                             "--add-data", f"package.json{sep}.",
-                           "--add-data", f"{os.path.join('dist_chrome', 'chrome')}{sep}chrome",
+                             "--add-data", f"{os.path.join('dist_chrome', 'chrome')}{sep}chrome",
-                           os.path.join("src", "flaresolverr.py")],
+                             os.path.join("src", "flaresolverr.py")],
-                          cwd=os.pardir)
+                            cwd=os.pardir, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    exe_folder = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'dist')
+    if result.returncode != 0:
-    exe_name = 'flaresolverr.exe' if os.name == 'nt' else 'flaresolverr'
+        print(result.stderr.decode('utf-8'))
-    exe_new_name = 'flaresolverr_windows_x64.exe' if os.name == 'nt' else 'flaresolverr_linux_x64'
+        raise Exception("Error running pyInstaller")
-    exe_path = os.path.join(exe_folder, exe_name)
+
-    exe_new_path = os.path.join(exe_folder, exe_new_name)
+
-    shutil.move(exe_path, exe_new_path)
+def compress_package():
-    print("Executable path: " + exe_new_path)
+    dist_folder = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'dist')
+    package_folder = os.path.join(dist_folder, 'package')
+    shutil.move(os.path.join(dist_folder, 'flaresolverr'), os.path.join(package_folder, 'flaresolverr'))
+    print("Package folder: " + package_folder)
+
+    compr_format = 'zip' if os.name == 'nt' else 'gztar'
+    compr_file_name = 'flaresolverr_windows_x64' if os.name == 'nt' else 'flaresolverr_linux_x64'
+    compr_file_path = os.path.join(dist_folder, compr_file_name)
+    shutil.make_archive(compr_file_path, compr_format, package_folder)
+    print("Compressed file path: " + compr_file_path)
 
 
 if __name__ == "__main__":
@@ -83,4 +105,7 @@ if __name__ == "__main__":
     print("Building pyinstaller executable... ")
     run_pyinstaller()
 
+    print("Compressing package... ")
+    compress_package()
+
 # NOTE: python -m pip install pyinstaller

src/flaresolverr.py

@@ -8,6 +8,7 @@ from bottle import run, response, Bottle, request, ServerAdapter
 
 from bottle_plugins.error_plugin import error_plugin
 from bottle_plugins.logger_plugin import logger_plugin
+from bottle_plugins import prometheus_plugin
 from dtos import V1RequestBase
 import flaresolverr_service
 import utils
@@ -24,10 +25,6 @@ class JSONErrorBottle(Bottle):
 
 app = JSONErrorBottle()
 
-# plugin order is important
-app.install(logger_plugin)
-app.install(error_plugin)
-
 
 @app.route('/')
 def index():
@@ -61,6 +58,16 @@ def controller_v1():
 
 
 if __name__ == "__main__":
+    # check python version
+    if sys.version_info < (3, 9):
+        raise Exception("The Python version is less than 3.9, a version equal to or higher is required.")
+
+    # fix for HEADLESS=false in Windows binary
+    # https://stackoverflow.com/a/27694505
+    if os.name == 'nt':
+        import multiprocessing
+        multiprocessing.freeze_support()
+
     # fix ssl certificates for compiled binaries
     # https://github.com/pyinstaller/pyinstaller/issues/7229
     # https://stackoverflow.com/questions/55736855/how-to-change-the-cafile-argument-in-the-ssl-module-in-python3
@@ -97,6 +104,13 @@ if __name__ == "__main__":
     # test browser installation
     flaresolverr_service.test_browser_installation()
 
+    # start bootle plugins
+    # plugin order is important
+    app.install(logger_plugin)
+    app.install(error_plugin)
+    prometheus_plugin.setup()
+    app.install(prometheus_plugin.prometheus_plugin)
+
     # start webserver
     # default server 'wsgiref' does not support concurrent requests
     # https://github.com/FlareSolverr/FlareSolverr/issues/680

src/flaresolverr_service.py

@@ -46,7 +46,7 @@ CHALLENGE_SELECTORS = [
     # Fairlane / pararius.com
     'div.vc div.text-box h2'
 ]
-SHORT_TIMEOUT = 10
+SHORT_TIMEOUT = 1
 SESSIONS_STORAGE = SessionsStorage()
 
 
@@ -179,7 +179,7 @@ def _cmd_request_post(req: V1RequestBase) -> V1ResponseBase:
 def _cmd_sessions_create(req: V1RequestBase) -> V1ResponseBase:
     logging.debug("Creating new session...")
 
-    session, fresh = SESSIONS_STORAGE.create(session_id=req.session)
+    session, fresh = SESSIONS_STORAGE.create(session_id=req.session, proxy=req.proxy)
     session_id = session.session_id
 
     if not fresh:
@@ -236,13 +236,13 @@ def _resolve_challenge(req: V1RequestBase, method: str) -> ChallengeResolutionT:
 
             driver = session.driver
         else:
-            driver = utils.get_webdriver()
+            driver = utils.get_webdriver(req.proxy)
             logging.debug('New instance of webdriver has been created to perform the request')
         return func_timeout(timeout, _evil_logic, (req, driver, method))
     except FunctionTimedOut:
         raise Exception(f'Error solving the challenge. Timeout after {timeout} seconds.')
     except Exception as e:
-        raise Exception('Error solving the challenge. ' + str(e))
+        raise Exception('Error solving the challenge. ' + str(e).replace('\n', '\\n'))
     finally:
         if not req.session and driver is not None:
             driver.quit()
@@ -251,26 +251,26 @@ def _resolve_challenge(req: V1RequestBase, method: str) -> ChallengeResolutionT:
 
 def click_verify(driver: WebDriver):
     try:
-        logging.debug("Try to find the Cloudflare verify checkbox")
+        logging.debug("Try to find the Cloudflare verify checkbox...")
-        iframe = driver.find_element(By.XPATH, "//iframe[@title='Widget containing a Cloudflare security challenge']")
+        iframe = driver.find_element(By.XPATH, "//iframe[starts-with(@id, 'cf-chl-widget-')]")
         driver.switch_to.frame(iframe)
         checkbox = driver.find_element(
             by=By.XPATH,
-            value='//*[@id="cf-stage"]//label[@class="ctp-checkbox-label"]/input',
+            value='//*[@id="challenge-stage"]/div/label/input',
         )
         if checkbox:
             actions = ActionChains(driver)
             actions.move_to_element_with_offset(checkbox, 5, 7)
             actions.click(checkbox)
             actions.perform()
-            logging.debug("Cloudflare verify checkbox found and clicked")
+            logging.debug("Cloudflare verify checkbox found and clicked!")
     except Exception:
-        logging.debug("Cloudflare verify checkbox not found on the page")
+        logging.debug("Cloudflare verify checkbox not found on the page.")
     finally:
         driver.switch_to.default_content()
 
     try:
-        logging.debug("Try to find the Cloudflare 'Verify you are human' button")
+        logging.debug("Try to find the Cloudflare 'Verify you are human' button...")
         button = driver.find_element(
             by=By.XPATH,
             value="//input[@type='button' and @value='Verify you are human']",
@@ -280,10 +280,9 @@ def click_verify(driver: WebDriver):
             actions.move_to_element_with_offset(button, 5, 7)
             actions.click(button)
             actions.perform()
-            logging.debug("The Cloudflare 'Verify you are human' button found and clicked")
+            logging.debug("The Cloudflare 'Verify you are human' button found and clicked!")
-    except Exception as e:
+    except Exception:
-        logging.debug("The Cloudflare 'Verify you are human' button not found on the page")
+        logging.debug("The Cloudflare 'Verify you are human' button not found on the page.")
-        # print(e)
 
     time.sleep(2)
 
@@ -295,14 +294,30 @@ def _evil_logic(req: V1RequestBase, driver: WebDriver, method: str) -> Challenge
 
     # navigate to the page
     logging.debug(f'Navigating to... {req.url}')
+    driver.get(req.url)
+    driver.start_session() # required to bypass Cloudflare
     if method == 'POST':
         _post_request(req, driver)
     else:
         driver.get(req.url)
-    if utils.get_config_log_html():
+        driver.start_session()  # required to bypass Cloudflare
-        logging.debug(f"Response HTML:\n{driver.page_source}")
+
+    # set cookies if required
+    if req.cookies is not None and len(req.cookies) > 0:
+        logging.debug(f'Setting cookies...')
+        for cookie in req.cookies:
+            driver.delete_cookie(cookie['name'])
+            driver.add_cookie(cookie)
+        # reload the page
+        if method == 'POST':
+            _post_request(req, driver)
+        else:
+            driver.get(req.url)
+            driver.start_session()  # required to bypass Cloudflare
 
     # wait for the page
+    if utils.get_config_log_html():
+        logging.debug(f"Response HTML:\n{driver.page_source}")
     html_element = driver.find_element(By.TAG_NAME, "html")
     page_title = driver.title
 
@@ -418,3 +433,4 @@ def _post_request(req: V1RequestBase, driver: WebDriver):
         </body>
         </html>"""
     driver.get("data:text/html;charset=utf-8," + html_content)
+    driver.start_session()  # required to bypass Cloudflare

src/metrics.py

@@ -0,0 +1,32 @@
+import logging
+
+from prometheus_client import Counter, Histogram, start_http_server
+import time
+
+REQUEST_COUNTER = Counter(
+    name='flaresolverr_request',
+    documentation='Total requests with result',
+    labelnames=['domain', 'result']
+)
+REQUEST_DURATION = Histogram(
+    name='flaresolverr_request_duration',
+    documentation='Request duration in seconds',
+    labelnames=['domain'],
+    buckets=[0, 10, 25, 50]
+)
+
+
+def serve(port):
+    start_http_server(port=port)
+    while True:
+        time.sleep(600)
+
+
+def start_metrics_http_server(prometheus_port: int):
+    logging.info(f"Serving Prometheus exporter on http://0.0.0.0:{prometheus_port}/metrics")
+    from threading import Thread
+    Thread(
+        target=serve,
+        kwargs=dict(port=prometheus_port),
+        daemon=True,
+    ).start()

src/sessions.py

@@ -25,7 +25,8 @@ class SessionsStorage:
     def __init__(self):
         self.sessions = {}
 
-    def create(self, session_id: Optional[str] = None, force_new: Optional[bool] = False) -> Tuple[Session, bool]:
+    def create(self, session_id: Optional[str] = None, proxy: Optional[dict] = None,
+               force_new: Optional[bool] = False) -> Tuple[Session, bool]:
         """create creates new instance of WebDriver if necessary,
         assign defined (or newly generated) session_id to the instance
         and returns the session object. If a new session has been created
@@ -44,7 +45,7 @@ class SessionsStorage:
         if self.exists(session_id):
             return self.sessions[session_id], False
 
-        driver = utils.get_webdriver()
+        driver = utils.get_webdriver(proxy)
         created_at = datetime.now()
         session = Session(session_id, driver, created_at)
 

src/tests.py

@@ -1,4 +1,5 @@
 import unittest
+from typing import Optional
 
 from webtest import TestApp
 
@@ -7,7 +8,7 @@ import flaresolverr
 import utils
 
 
-def _find_obj_by_key(key: str, value: str, _list: list) -> dict | None:
+def _find_obj_by_key(key: str, value: str, _list: list) -> Optional[dict]:
     for obj in _list:
         if obj[key] == value:
             return obj
@@ -28,6 +29,8 @@ class TestFlareSolverr(unittest.TestCase):
     cloudflare_blocked_url = "https://cpasbiens3.fr/index.php?do=search&subaction=search"
 
     app = TestApp(flaresolverr.app)
+    # wait until the server is ready
+    app.get('/')
 
     def test_wrong_endpoint(self):
         res = self.app.get('/wrong', status=404)
@@ -236,7 +239,45 @@ class TestFlareSolverr(unittest.TestCase):
         self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
         self.assertEqual(utils.get_flaresolverr_version(), body.version)
 
-    # todo: test Cmd 'request.get' should return OK with 'cookies' param
+    def test_v1_endpoint_request_get_cookies_param(self):
+        res = self.app.post_json('/v1', {
+            "cmd": "request.get",
+            "url": self.google_url,
+            "cookies": [
+                {
+                    "name": "testcookie1",
+                    "value": "testvalue1"
+                },
+                {
+                    "name": "testcookie2",
+                    "value": "testvalue2"
+                }
+            ]
+        })
+        self.assertEqual(res.status_code, 200)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_OK, body.status)
+        self.assertEqual("Challenge not detected!", body.message)
+        self.assertGreater(body.startTimestamp, 10000)
+        self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
+        self.assertEqual(utils.get_flaresolverr_version(), body.version)
+
+        solution = body.solution
+        self.assertIn(self.google_url, solution.url)
+        self.assertEqual(solution.status, 200)
+        self.assertIs(len(solution.headers), 0)
+        self.assertIn("<title>Google</title>", solution.response)
+        self.assertGreater(len(solution.cookies), 1)
+        self.assertIn("Chrome/", solution.userAgent)
+
+        user_cookie1 = _find_obj_by_key("name", "testcookie1", solution.cookies)
+        self.assertIsNotNone(user_cookie1, "User cookie 1 not found")
+        self.assertEqual("testvalue1", user_cookie1["value"])
+
+        user_cookie2 = _find_obj_by_key("name", "testcookie2", solution.cookies)
+        self.assertIsNotNone(user_cookie2, "User cookie 2 not found")
+        self.assertEqual("testvalue2", user_cookie2["value"])
 
     def test_v1_endpoint_request_get_returnOnlyCookies_param(self):
         res = self.app.post_json('/v1', {
@@ -261,10 +302,124 @@ class TestFlareSolverr(unittest.TestCase):
         self.assertGreater(len(solution.cookies), 0)
         self.assertIn("Chrome/", solution.userAgent)
 
-    # todo: test Cmd 'request.get' should return OK with HTTP 'proxy' param
+    def test_v1_endpoint_request_get_proxy_http_param(self):
-    # todo: test Cmd 'request.get' should return OK with HTTP 'proxy' param with credentials
+        """
-    # todo: test Cmd 'request.get' should return OK with SOCKSv5 'proxy' param
+        To configure TinyProxy in local:
-    # todo: test Cmd 'request.get' should fail with wrong 'proxy' param
+           * sudo vim /etc/tinyproxy/tinyproxy.conf
+              * edit => LogFile "/tmp/tinyproxy.log"
+              * edit => Syslog Off
+           * sudo tinyproxy -d
+           * sudo tail -f /tmp/tinyproxy.log
+        """
+        res = self.app.post_json('/v1', {
+            "cmd": "request.get",
+            "url": self.google_url,
+            "proxy": {
+                "url": self.proxy_url
+            }
+        })
+        self.assertEqual(res.status_code, 200)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_OK, body.status)
+        self.assertEqual("Challenge not detected!", body.message)
+        self.assertGreater(body.startTimestamp, 10000)
+        self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
+        self.assertEqual(utils.get_flaresolverr_version(), body.version)
+
+        solution = body.solution
+        self.assertIn(self.google_url, solution.url)
+        self.assertEqual(solution.status, 200)
+        self.assertIs(len(solution.headers), 0)
+        self.assertIn("<title>Google</title>", solution.response)
+        self.assertGreater(len(solution.cookies), 0)
+        self.assertIn("Chrome/", solution.userAgent)
+
+    def test_v1_endpoint_request_get_proxy_http_param_with_credentials(self):
+        """
+        To configure TinyProxy in local:
+           * sudo vim /etc/tinyproxy/tinyproxy.conf
+              * edit => LogFile "/tmp/tinyproxy.log"
+              * edit => Syslog Off
+              * add => BasicAuth testuser testpass
+           * sudo tinyproxy -d
+           * sudo tail -f /tmp/tinyproxy.log
+        """
+        res = self.app.post_json('/v1', {
+            "cmd": "request.get",
+            "url": self.google_url,
+            "proxy": {
+                "url": self.proxy_url,
+                "username": "testuser",
+                "password": "testpass"
+            }
+        })
+        self.assertEqual(res.status_code, 200)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_OK, body.status)
+        self.assertEqual("Challenge not detected!", body.message)
+        self.assertGreater(body.startTimestamp, 10000)
+        self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
+        self.assertEqual(utils.get_flaresolverr_version(), body.version)
+
+        solution = body.solution
+        self.assertIn(self.google_url, solution.url)
+        self.assertEqual(solution.status, 200)
+        self.assertIs(len(solution.headers), 0)
+        self.assertIn("<title>Google</title>", solution.response)
+        self.assertGreater(len(solution.cookies), 0)
+        self.assertIn("Chrome/", solution.userAgent)
+
+    def test_v1_endpoint_request_get_proxy_socks_param(self):
+        """
+        To configure Dante in local:
+           * https://linuxhint.com/set-up-a-socks5-proxy-on-ubuntu-with-dante/
+           * sudo vim /etc/sockd.conf
+           * sudo systemctl restart sockd.service
+           * curl --socks5 socks5://127.0.0.1:1080 https://www.google.com
+        """
+        res = self.app.post_json('/v1', {
+            "cmd": "request.get",
+            "url": self.google_url,
+            "proxy": {
+                "url": self.proxy_socks_url
+            }
+        })
+        self.assertEqual(res.status_code, 200)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_OK, body.status)
+        self.assertEqual("Challenge not detected!", body.message)
+        self.assertGreater(body.startTimestamp, 10000)
+        self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
+        self.assertEqual(utils.get_flaresolverr_version(), body.version)
+
+        solution = body.solution
+        self.assertIn(self.google_url, solution.url)
+        self.assertEqual(solution.status, 200)
+        self.assertIs(len(solution.headers), 0)
+        self.assertIn("<title>Google</title>", solution.response)
+        self.assertGreater(len(solution.cookies), 0)
+        self.assertIn("Chrome/", solution.userAgent)
+
+    def test_v1_endpoint_request_get_proxy_wrong_param(self):
+        res = self.app.post_json('/v1', {
+            "cmd": "request.get",
+            "url": self.google_url,
+            "proxy": {
+                "url": "http://127.0.0.1:43210"
+            }
+        }, status=500)
+        self.assertEqual(res.status_code, 500)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_ERROR, body.status)
+        self.assertIn("Error: Error solving the challenge. Message: unknown error: net::ERR_PROXY_CONNECTION_FAILED",
+                      body.message)
+        self.assertGreater(body.startTimestamp, 10000)
+        self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
+        self.assertEqual(utils.get_flaresolverr_version(), body.version)
 
     def test_v1_endpoint_request_get_fail_timeout(self):
         res = self.app.post_json('/v1', {
@@ -401,6 +556,20 @@ class TestFlareSolverr(unittest.TestCase):
         self.assertEqual("Session created successfully.", body.message)
         self.assertEqual(body.session, "test_create_session")
 
+    def test_v1_endpoint_sessions_create_with_proxy(self):
+        res = self.app.post_json('/v1', {
+            "cmd": "sessions.create",
+            "proxy": {
+                "url": self.proxy_url
+            }
+        })
+        self.assertEqual(res.status_code, 200)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_OK, body.status)
+        self.assertEqual("Session created successfully.", body.message)
+        self.assertIsNotNone(body.session)
+
     def test_v1_endpoint_sessions_list(self):
         self.app.post_json('/v1', {
             "cmd": "sessions.create",

src/tests_sites.py

@@ -39,6 +39,8 @@ def asset_cloudflare_solution(self, res, site_url, site_text):
 
 class TestFlareSolverr(unittest.TestCase):
     app = TestApp(flaresolverr.app)
+    # wait until the server is ready
+    app.get('/')
 
     def test_v1_endpoint_request_get_cloudflare(self):
         sites_get = [

src/undetected_chromedriver/__init__.py

@@ -17,11 +17,12 @@ by UltrafunkAmsterdam (https://github.com/ultrafunkamsterdam)
 from __future__ import annotations
 
 
-__version__ = "3.4.6"
+__version__ = "3.5.4"
 
 import json
 import logging
 import os
+import pathlib
 import re
 import shutil
 import subprocess
@@ -33,7 +34,7 @@ from weakref import finalize
 import selenium.webdriver.chrome.service
 import selenium.webdriver.chrome.webdriver
 from selenium.webdriver.common.by import By
-import selenium.webdriver.common.service
+import selenium.webdriver.chromium.service
 import selenium.webdriver.remote.command
 import selenium.webdriver.remote.webdriver
 
@@ -109,11 +110,11 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
         browser_executable_path=None,
         port=0,
         enable_cdp_events=False,
-        service_args=None,
+        # service_args=None,
-        service_creationflags=None,
+        # service_creationflags=None,
         desired_capabilities=None,
         advanced_elements=False,
-        service_log_path=None,
+        # service_log_path=None,
         keep_alive=True,
         log_level=0,
         headless=False,
@@ -122,8 +123,9 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
         suppress_welcome=True,
         use_subprocess=False,
         debug=False,
-        no_sandbox=True,	
+        no_sandbox=True,
         windows_headless=False,
+        user_multi_procs: bool = False,
         **kw,
     ):
         """
@@ -235,6 +237,14 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
              uses the --no-sandbox option, and additionally does suppress the "unsecure option" status bar
              this option has a default of True since many people seem to run this as root (....) , and chrome does not start
              when running as root without using --no-sandbox flag.
+
+        user_multi_procs:
+            set to true when you are using multithreads/multiprocessing
+            ensures not all processes are trying to modify a binary which is in use by another.
+            for this to work. YOU MUST HAVE AT LEAST 1 UNDETECTED_CHROMEDRIVER BINARY IN YOUR ROAMING DATA FOLDER.
+            this requirement can be easily satisfied, by just running this program "normal" and close/kill it.
+
+
         """
 
         finalize(self, self._ensure_close, self)
@@ -243,8 +253,11 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
             executable_path=driver_executable_path,
             force=patcher_force_close,
             version_main=version_main,
+            user_multi_procs=user_multi_procs,
         )
+        # self.patcher.auto(user_multiprocess = user_multi_num_procs)
         self.patcher.auto()
+
         # self.patcher = patcher
         if not options:
             options = ChromeOptions()
@@ -361,6 +374,18 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
                 browser_executable_path or find_chrome_executable()
             )
 
+        if not options.binary_location or not \
+                pathlib.Path(options.binary_location).exists():
+                raise FileNotFoundError(
+                    "\n---------------------\n"
+                    "Could not determine browser executable."
+                    "\n---------------------\n"
+                    "Make sure your browser is installed in the default location (path).\n"
+                    "If you are sure about the browser executable, you can specify it using\n"
+                    "the `browser_executable_path='{}` parameter.\n\n"
+                    .format("/path/to/browser/executable" if IS_POSIX else "c:/path/to/your/browser.exe")
+                )
+
         self._delay = 3
 
         self.user_data_dir = user_data_dir
@@ -371,10 +396,17 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
         if no_sandbox:
             options.arguments.extend(["--no-sandbox", "--test-type"])
 
-        if headless or options.headless:
+        if headless or getattr(options, 'headless', None):
-            if self.patcher.version_main < 108:
+            #workaround until a better checking is found
-                options.add_argument("--headless=chrome")
+            try:
-            elif self.patcher.version_main >= 108:
+                v_main = int(self.patcher.version_main) if self.patcher.version_main else 108
+                if v_main < 108:
+                    options.add_argument("--headless=chrome")
+                elif v_main >= 108:
+                    options.add_argument("--headless=new")
+            except:
+                logger.warning("could not detect version_main."
+                               "therefore, we are assuming it is chrome 108 or higher")
                 options.add_argument("--headless=new")
 
         options.add_argument("--window-size=1920,1080")
@@ -418,40 +450,29 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
             self.browser_pid = start_detached(
                 options.binary_location, *options.arguments
             )
-        else:	
+        else:
-            startupinfo = subprocess.STARTUPINFO()	
+            startupinfo = subprocess.STARTUPINFO()
-            if os.name == 'nt' and windows_headless:	
+            if os.name == 'nt' and windows_headless:
                 startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
             browser = subprocess.Popen(
                 [options.binary_location, *options.arguments],
                 stdin=subprocess.PIPE,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.PIPE,
-                close_fds=IS_POSIX,	
+                close_fds=IS_POSIX,
                 startupinfo=startupinfo
             )
             self.browser_pid = browser.pid
 
-        if service_creationflags:
+
-            service = selenium.webdriver.common.service.Service(
+        service = selenium.webdriver.chromium.service.ChromiumService(
-                self.patcher.executable_path, port, service_args, service_log_path
+            self.patcher.executable_path
-            )
+        )
-            for attr_name in ("creationflags", "creation_flags"):
-                if hasattr(service, attr_name):
-                    setattr(service, attr_name, service_creationflags)
-                    break
-        else:
-            service = None
 
         super(Chrome, self).__init__(
-            executable_path=self.patcher.executable_path,
+            service=service,
-            port=port,
             options=options,
-            service_args=service_args,
-            desired_capabilities=desired_capabilities,
-            service_log_path=service_log_path,
             keep_alive=keep_alive,
-            service=service,  # needed or the service will be re-created
         )
 
         self.reactor = None
@@ -470,7 +491,7 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
         else:
             self._web_element_cls = WebElement
 
-        if options.headless:
+        if headless or getattr(options, 'headless', None):
             self._configure_headless()
 
     def _configure_headless(self):
@@ -707,13 +728,48 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
         if not capabilities:
             capabilities = self.options.to_capabilities()
         super(selenium.webdriver.chrome.webdriver.WebDriver, self).start_session(
-            capabilities, browser_profile
+            capabilities
         )
         # super(Chrome, self).start_session(capabilities, browser_profile)
 
+    def find_elements_recursive(self, by, value):
+        """
+        find elements in all frames
+        this is a generator function, which is needed
+            since if it would return a list of elements, they
+            will be stale on arrival.
+        using generator, when the element is returned we are in the correct frame
+        to use it directly
+        Args:
+            by: By
+            value: str
+        Returns: Generator[webelement.WebElement]
+        """
+        def search_frame(f=None):
+            if not f:
+                # ensure we are on main content frame
+                self.switch_to.default_content()
+            else:
+                self.switch_to.frame(f)
+            for elem in self.find_elements(by, value):
+                yield elem
+            # switch back to main content, otherwise we will get StaleElementReferenceException
+            self.switch_to.default_content()
+
+        # search root frame
+        for elem in search_frame():
+            yield elem
+        # get iframes
+        frames = self.find_elements('css selector', 'iframe')
+
+        # search per frame
+        for f in frames:
+            for elem in search_frame(f):
+                yield elem
+
     def quit(self):
         try:
-            self.service.process.kill()	
+            self.service.process.kill()
             self.service.process.wait(5)
             logger.debug("webdriver process ended")
         except (AttributeError, RuntimeError, OSError):
@@ -727,7 +783,16 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
             os.kill(self.browser_pid, 15)
             logger.debug("gracefully closed browser")
         except Exception as e:  # noqa
-            logger.debug(e, exc_info=True)
+            pass
+        # Force kill Chrome process in Windows
+        # https://github.com/FlareSolverr/FlareSolverr/issues/772
+        if os.name == 'nt':
+            try:
+                subprocess.call(['taskkill', '/f', '/pid', str(self.browser_pid)],
+                                stdout=subprocess.DEVNULL,
+                                stderr=subprocess.DEVNULL)
+            except Exception:
+                pass
         if (
             hasattr(self, "keep_user_data_dir")
             and hasattr(self, "user_data_dir")
@@ -746,7 +811,11 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
                 else:
                     logger.debug("successfully removed %s" % self.user_data_dir)
                     break
-                time.sleep(0.1)
+
+                try:
+                    time.sleep(0.1)
+                except OSError:
+                    pass
 
         # dereference patcher, so patcher can start cleaning up as well.
         # this must come last, otherwise it will throw 'in use' errors
@@ -841,10 +910,10 @@ def find_chrome_executable():
             if item is not None:
                 for subitem in (
                     "Google/Chrome/Application",
-                    "Google/Chrome Beta/Application",
-                    "Google/Chrome Canary/Application",
                 ):
                     candidates.add(os.sep.join((item, subitem, "chrome.exe")))
     for candidate in candidates:
+        logger.debug('checking if %s exists and is executable' % candidate)
         if os.path.exists(candidate) and os.access(candidate, os.X_OK):
+            logger.debug('found! using %s' % candidate)
             return os.path.normpath(candidate)

src/undetected_chromedriver/dprocess.py

@@ -41,6 +41,7 @@ def start_detached(executable, *args):
     # close pipes
     writer.close()
     reader.close()
+    process.close()
 
     return pid
 

src/undetected_chromedriver/patcher.py

@@ -3,17 +3,21 @@
 
 from distutils.version import LooseVersion
 import io
+import json
 import logging
 import os
+import pathlib
+import platform
 import random
 import re
+import shutil
 import string
 import sys
 import time
 from urllib.request import urlopen
 from urllib.request import urlretrieve
 import zipfile
-
+from multiprocessing import Lock
 
 logger = logging.getLogger(__name__)
 
@@ -21,21 +25,10 @@ IS_POSIX = sys.platform.startswith(("darwin", "cygwin", "linux", "linux2"))
 
 
 class Patcher(object):
-    url_repo = "https://chromedriver.storage.googleapis.com"
+    lock = Lock()
-    zip_name = "chromedriver_%s.zip"
     exe_name = "chromedriver%s"
 
     platform = sys.platform
-    if platform.endswith("win32"):
-        zip_name %= "win32"
-        exe_name %= ".exe"
-    if platform.endswith(("linux", "linux2")):
-        zip_name %= "linux64"
-        exe_name %= ""
-    if platform.endswith("darwin"):
-        zip_name %= "mac64"
-        exe_name %= ""
-
     if platform.endswith("win32"):
         d = "~/appdata/roaming/undetected_chromedriver"
     elif "LAMBDA_TASK_ROOT" in os.environ:
@@ -48,7 +41,13 @@ class Patcher(object):
         d = "~/.undetected_chromedriver"
     data_path = os.path.abspath(os.path.expanduser(d))
 
-    def __init__(self, executable_path=None, force=False, version_main: int = 0):
+    def __init__(
+        self,
+        executable_path=None,
+        force=False,
+        version_main: int = 0,
+        user_multi_procs=False,
+    ):
         """
         Args:
             executable_path: None = automatic
@@ -61,6 +60,21 @@ class Patcher(object):
         self.force = force
         self._custom_exe_path = False
         prefix = "undetected"
+        self.user_multi_procs = user_multi_procs
+
+        try:
+            # Try to convert version_main into an integer
+            version_main_int = int(version_main)
+            # check if version_main_int is less than or equal to e.g 114
+            self.is_old_chromedriver = version_main and version_main_int <= 114
+        except (ValueError,TypeError):
+            # If the conversion fails, print an error message
+            print("version_main cannot be converted to an integer")
+            # Set self.is_old_chromedriver to False if the conversion fails
+            self.is_old_chromedriver = False
+
+        # Needs to be called before self.exe_name is accessed
+        self._set_platform_name()
 
         if not os.path.exists(self.data_path):
             os.makedirs(self.data_path, exist_ok=True)
@@ -78,17 +92,64 @@ class Patcher(object):
         self.zip_path = os.path.join(self.data_path, prefix)
 
         if not executable_path:
-            self.executable_path = os.path.abspath(
+            if not self.user_multi_procs:
-                os.path.join(".", self.executable_path)
+                self.executable_path = os.path.abspath(
-            )
+                    os.path.join(".", self.executable_path)
+                )
 
         if executable_path:
             self._custom_exe_path = True
             self.executable_path = executable_path
+
+        # Set the correct repository to download the Chromedriver from
+        if self.is_old_chromedriver:
+            self.url_repo = "https://chromedriver.storage.googleapis.com"
+        else:
+            self.url_repo = "https://googlechromelabs.github.io/chrome-for-testing"
+
         self.version_main = version_main
         self.version_full = None
 
-    def auto(self, executable_path=None, force=False, version_main=None):
+    def _set_platform_name(self):
+        """
+        Set the platform and exe name based on the platform undetected_chromedriver is running on
+        in order to download the correct chromedriver.
+        """
+        if self.platform.endswith("win32"):
+            self.platform_name = "win32"
+            self.exe_name %= ".exe"
+        if self.platform.endswith(("linux", "linux2")):
+            self.platform_name = "linux64"
+            self.exe_name %= ""
+        if self.platform.endswith("darwin"):
+            if self.is_old_chromedriver:
+                self.platform_name = "mac64"
+            else:
+                self.platform_name = "mac-x64"
+            self.exe_name %= ""
+
+    def auto(self, executable_path=None, force=False, version_main=None, _=None):
+        """
+
+        Args:
+            executable_path:
+            force:
+            version_main:
+
+        Returns:
+
+        """
+        p = pathlib.Path(self.data_path)
+        if self.user_multi_procs:
+            with Lock():
+                files = list(p.rglob("*chromedriver*"))
+                most_recent = max(files, key=lambda f: f.stat().st_mtime)
+                files.remove(most_recent)
+                list(map(lambda f: f.unlink(), files))
+                if self.is_binary_patched(most_recent):
+                    self.executable_path = str(most_recent)
+                    return True
+
         if executable_path:
             self.executable_path = executable_path
             self._custom_exe_path = True
@@ -127,6 +188,53 @@ class Patcher(object):
         self.unzip_package(self.fetch_package())
         return self.patch()
 
+    def driver_binary_in_use(self, path: str = None) -> bool:
+        """
+        naive test to check if a found chromedriver binary is
+        currently in use
+
+        Args:
+            path: a string or PathLike object to the binary to check.
+                  if not specified, we check use this object's executable_path
+        """
+        if not path:
+            path = self.executable_path
+        p = pathlib.Path(path)
+
+        if not p.exists():
+            raise OSError("file does not exist: %s" % p)
+        try:
+            with open(p, mode="a+b") as fs:
+                exc = []
+                try:
+
+                    fs.seek(0, 0)
+                except PermissionError as e:
+                    exc.append(e)  # since some systems apprently allow seeking
+                    # we conduct another test
+                try:
+                    fs.readline()
+                except PermissionError as e:
+                    exc.append(e)
+
+                if exc:
+
+                    return True
+                return False
+            # ok safe to assume this is in use
+        except Exception as e:
+            # logger.exception("whoops ", e)
+            pass
+
+    def cleanup_unused_files(self):
+        p = pathlib.Path(self.data_path)
+        items = list(p.glob("*undetected*"))
+        for item in items:
+            try:
+                item.unlink()
+            except:
+                pass
+
     def patch(self):
         self.patch_exe()
         return self.is_binary_patched()
@@ -137,12 +245,32 @@ class Patcher(object):
         :return: version string
         :rtype: LooseVersion
         """
-        path = "/latest_release"
+        # Endpoint for old versions of Chromedriver (114 and below)
-        if self.version_main:
+        if self.is_old_chromedriver:
-            path += f"_{self.version_main}"
+            path = f"/latest_release_{self.version_main}"
-        path = path.upper()
+            path = path.upper()
+            logger.debug("getting release number from %s" % path)
+            return LooseVersion(urlopen(self.url_repo + path).read().decode())
+
+        # Endpoint for new versions of Chromedriver (115+)
+        if not self.version_main:
+            # Fetch the latest version
+            path = "/last-known-good-versions-with-downloads.json"
+            logger.debug("getting release number from %s" % path)
+            with urlopen(self.url_repo + path) as conn:
+                response = conn.read().decode()
+
+            last_versions = json.loads(response)
+            return LooseVersion(last_versions["channels"]["Stable"]["version"])
+
+        # Fetch the latest minor version of the major version provided
+        path = "/latest-versions-per-milestone-with-downloads.json"
         logger.debug("getting release number from %s" % path)
-        return LooseVersion(urlopen(self.url_repo + path).read().decode())
+        with urlopen(self.url_repo + path) as conn:
+            response = conn.read().decode()
+
+        major_versions = json.loads(response)
+        return LooseVersion(major_versions["milestones"][str(self.version_main)]["version"])
 
     def parse_exe_version(self):
         with io.open(self.executable_path, "rb") as f:
@@ -157,10 +285,16 @@ class Patcher(object):
 
         :return: path to downloaded file
         """
-        u = "%s/%s/%s" % (self.url_repo, self.version_full.vstring, self.zip_name)
+        zip_name = f"chromedriver_{self.platform_name}.zip"
-        logger.debug("downloading from %s" % u)
+        if self.is_old_chromedriver:
-        # return urlretrieve(u, filename=self.data_path)[0]
+            download_url = "%s/%s/%s" % (self.url_repo, self.version_full.vstring, zip_name)
-        return urlretrieve(u)[0]
+        else:
+            zip_name = zip_name.replace("_", "-", 1)
+            download_url = "https://edgedl.me.gvt1.com/edgedl/chrome/chrome-for-testing/%s/%s/%s"
+            download_url %= (self.version_full.vstring, self.platform_name, zip_name)
+
+        logger.debug("downloading from %s" % download_url)
+        return urlretrieve(download_url)[0]
 
     def unzip_package(self, fp):
         """
@@ -168,6 +302,12 @@ class Patcher(object):
 
         :return: path to unpacked executable
         """
+        exe_path = self.exe_name
+        if not self.is_old_chromedriver:
+            # The new chromedriver unzips into its own folder
+            zip_name = f"chromedriver-{self.platform_name}"
+            exe_path = os.path.join(zip_name, self.exe_name)
+
         logger.debug("unzipping %s" % fp)
         try:
             os.unlink(self.zip_path)
@@ -176,10 +316,10 @@ class Patcher(object):
 
         os.makedirs(self.zip_path, mode=0o755, exist_ok=True)
         with zipfile.ZipFile(fp, mode="r") as zf:
-            zf.extract(self.exe_name, self.zip_path)
+            zf.extractall(self.zip_path)
-        os.rename(os.path.join(self.zip_path, self.exe_name), self.executable_path)
+        os.rename(os.path.join(self.zip_path, exe_path), self.executable_path)
         os.remove(fp)
-        os.rmdir(self.zip_path)
+        shutil.rmtree
         os.chmod(self.executable_path, 0o755)
         return self.executable_path
 
@@ -255,21 +395,17 @@ class Patcher(object):
         else:
             timeout = 3  # stop trying after this many seconds
             t = time.monotonic()
-            while True:
+            now = lambda: time.monotonic()
-                now = time.monotonic()
+            while now() - t > timeout:
-                if now - t > timeout:
+                # we don't want to wait until the end of time
-                    # we don't want to wait until the end of time
-                    logger.debug(
-                        "could not unlink %s in time (%d seconds)"
-                        % (self.executable_path, timeout)
-                    )
-                    break
                 try:
+                    if self.user_multi_procs:
+                        break
                     os.unlink(self.executable_path)
                     logger.debug("successfully unlinked %s" % self.executable_path)
                     break
                 except (OSError, RuntimeError, PermissionError):
-                    time.sleep(0.1)
+                    time.sleep(0.01)
                     continue
                 except FileNotFoundError:
                     break

src/utils.py

@@ -3,6 +3,8 @@ import logging
 import os
 import re
 import shutil
+import urllib.parse
+import tempfile
 
 from selenium.webdriver.chrome.webdriver import WebDriver
 import undetected_chromedriver as uc
@@ -36,8 +38,82 @@ def get_flaresolverr_version() -> str:
         return FLARESOLVERR_VERSION
 
 
-def get_webdriver() -> WebDriver:
+def create_proxy_extension(proxy: dict) -> str:
-    global PATCHED_DRIVER_PATH
+    parsed_url = urllib.parse.urlparse(proxy['url'])
+    scheme = parsed_url.scheme
+    host = parsed_url.hostname
+    port = parsed_url.port
+    username = proxy['username']
+    password = proxy['password']
+    manifest_json = """
+    {
+        "version": "1.0.0",
+        "manifest_version": 2,
+        "name": "Chrome Proxy",
+        "permissions": [
+            "proxy",
+            "tabs",
+            "unlimitedStorage",
+            "storage",
+            "<all_urls>",
+            "webRequest",
+            "webRequestBlocking"
+        ],
+        "background": {"scripts": ["background.js"]},
+        "minimum_chrome_version": "76.0.0"
+    }
+    """
+
+    background_js = """
+    var config = {
+        mode: "fixed_servers",
+        rules: {
+            singleProxy: {
+                scheme: "%s",
+                host: "%s",
+                port: %d
+            },
+            bypassList: ["localhost"]
+        }
+    };
+
+    chrome.proxy.settings.set({value: config, scope: "regular"}, function() {});
+
+    function callbackFn(details) {
+        return {
+            authCredentials: {
+                username: "%s",
+                password: "%s"
+            }
+        };
+    }
+
+    chrome.webRequest.onAuthRequired.addListener(
+        callbackFn,
+        { urls: ["<all_urls>"] },
+        ['blocking']
+    );
+    """ % (
+        scheme,
+        host,
+        port,
+        username,
+        password
+    )
+
+    proxy_extension_dir = tempfile.mkdtemp()
+
+    with open(os.path.join(proxy_extension_dir, "manifest.json"), "w") as f:
+        f.write(manifest_json)
+
+    with open(os.path.join(proxy_extension_dir, "background.js"), "w") as f:
+        f.write(background_js)
+
+    return proxy_extension_dir
+
+
+def get_webdriver(proxy: dict = None) -> WebDriver:
+    global PATCHED_DRIVER_PATH, USER_AGENT
     logging.debug('Launching web browser...')
 
     # undetected_chromedriver
@@ -54,8 +130,30 @@ def get_webdriver() -> WebDriver:
     options.add_argument('--disable-software-rasterizer')
     options.add_argument('--ignore-certificate-errors')
     options.add_argument('--ignore-ssl-errors')
+    # fix GL errors in ASUSTOR NAS
+    # https://github.com/FlareSolverr/FlareSolverr/issues/782
+    # https://github.com/microsoft/vscode/issues/127800#issuecomment-873342069
+    # https://peter.sh/experiments/chromium-command-line-switches/#use-gl
+    options.add_argument('--use-gl=swiftshader')
 
-    # note: headless mode is detected (options.headless = True)
+    language = os.environ.get('LANG', None)
+    if language is not None:
+        options.add_argument('--lang=%s' % language)
+
+    # Fix for Chrome 117 | https://github.com/FlareSolverr/FlareSolverr/issues/910
+    if USER_AGENT is not None:
+        options.add_argument('--user-agent=%s' % USER_AGENT)
+
+    proxy_extension_dir = None
+    if proxy and all(key in proxy for key in ['url', 'username', 'password']):
+        proxy_extension_dir = create_proxy_extension(proxy)
+        options.add_argument("--load-extension=%s" % os.path.abspath(proxy_extension_dir))
+    elif proxy and 'url' in proxy:
+        proxy_url = proxy['url']
+        logging.debug("Using webdriver proxy: %s", proxy_url)
+        options.add_argument('--proxy-server=%s' % proxy_url)
+
+    # note: headless mode is detected (headless = True)
     # we launch the browser in head-full mode with the window hidden
     windows_headless = False
     if get_config_headless():
@@ -63,6 +161,8 @@ def get_webdriver() -> WebDriver:
             windows_headless = True
         else:
             start_xvfb_display()
+    # For normal headless mode:
+    # options.add_argument('--headless')
 
     # if we are inside the Docker container, we avoid downloading the driver
     driver_exe_path = None
@@ -82,12 +182,17 @@ def get_webdriver() -> WebDriver:
     # if we don't set driver_executable_path it downloads, patches, and deletes the driver each time
     driver = uc.Chrome(options=options, browser_executable_path=browser_executable_path,
                        driver_executable_path=driver_exe_path, version_main=version_main,
-                       windows_headless=windows_headless)
+                       windows_headless=windows_headless, headless=windows_headless)
 
     # save the patched driver to avoid re-downloads
     if driver_exe_path is None:
         PATCHED_DRIVER_PATH = os.path.join(driver.patcher.data_path, driver.patcher.exe_name)
-        shutil.copy(driver.patcher.executable_path, PATCHED_DRIVER_PATH)
+        if PATCHED_DRIVER_PATH != driver.patcher.executable_path:
+            shutil.copy(driver.patcher.executable_path, PATCHED_DRIVER_PATH)
+
+    # clean up proxy extension directory
+    if proxy_extension_dir is not None:
+        shutil.rmtree(proxy_extension_dir)
 
     # selenium vanilla
     # options = webdriver.ChromeOptions()
@@ -106,12 +211,15 @@ def get_chrome_exe_path() -> str:
         return CHROME_EXE_PATH
     # linux pyinstaller bundle
     chrome_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'chrome', "chrome")
-    if os.path.exists(chrome_path) and os.access(chrome_path, os.X_OK):
+    if os.path.exists(chrome_path):
+        if not os.access(chrome_path, os.X_OK):
+            raise Exception(f'Chrome binary "{chrome_path}" is not executable. '
+                            f'Please, extract the archive with "tar xzf <file.tar.gz>".')
         CHROME_EXE_PATH = chrome_path
         return CHROME_EXE_PATH
     # windows pyinstaller bundle
     chrome_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'chrome', "chrome.exe")
-    if os.path.exists(chrome_path) and os.access(chrome_path, os.X_OK):
+    if os.path.exists(chrome_path):
         CHROME_EXE_PATH = chrome_path
         return CHROME_EXE_PATH
     # system
@@ -193,6 +301,8 @@ def get_user_agent(driver=None) -> str:
         if driver is None:
             driver = get_webdriver()
         USER_AGENT = driver.execute_script("return navigator.userAgent")
+        # Fix for Chrome 117 | https://github.com/FlareSolverr/FlareSolverr/issues/910
+        USER_AGENT = re.sub('HEADLESS', '', USER_AGENT, flags=re.IGNORECASE)
         return USER_AGENT
     except Exception as e:
         raise Exception("Error getting browser User-Agent. " + str(e))