Update css selector to get shadowed iframe

*.0 was yanked

.github/workflows/release-docker.yml

@@ -4,50 +4,64 @@ on:
   push:
     tags:
       - 'v*.*.*'
+  pull_request:
+    branches:
+      - master
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 
 jobs:
   build-docker-images:
+    if: ${{ !github.event.pull_request.head.repo.fork }}
     runs-on: ubuntu-22.04
     steps:
-      -
+      - name: Checkout
-        name: Checkout
+        uses: actions/checkout@v4
-        uses: actions/checkout@v3
+
-      -
+      - name: Downcase repo
-        name: Downcase repo
         run: echo REPOSITORY=$(echo ${{ github.repository }} | tr '[:upper:]' '[:lower:]') >> $GITHUB_ENV
-      -
+
-        name: Docker meta
+      - name: Docker meta
         id: docker_meta
-        uses: crazy-max/ghaction-docker-meta@v3
+        uses: docker/metadata-action@v5
         with:
-          images: ${{ env.REPOSITORY }},ghcr.io/${{ env.REPOSITORY }}
+          images: |
-          tag-sha: false
+            ${{ env.REPOSITORY }},enable=${{ github.event_name != 'pull_request' }}
-      -
+            ghcr.io/${{ env.REPOSITORY }}
-        name: Set up QEMU
+          tags: |
-        uses: docker/setup-qemu-action@v2
+            type=semver,pattern={{version}},prefix=v
-      -
+            type=ref,event=pr
-        name: Set up Docker Buildx
+          flavor: |
-        uses: docker/setup-buildx-action@v2
+            latest=auto
-      -
+
-        name: Login to DockerHub
+      - name: Set up QEMU
-        uses: docker/login-action@v2
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        if: github.event_name != 'pull_request'
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      -
+
-        name: Login to GitHub Container Registry
+      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GH_PAT }}
-      -
+
-        name: Build and push
+      - name: Build and push
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: ./Dockerfile
           platforms: linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8
-          push: ${{ github.event_name != 'pull_request' }}
+          push: true
           tags: ${{ steps.docker_meta.outputs.tags }}
           labels: ${{ steps.docker_meta.outputs.labels }}

CHANGELOG.md

@@ -1,5 +1,26 @@
 # Changelog
 
+## v3.3.21 (2024/06/26)
+* Add challenge selector to catch reloading page on non-English systems
+* Escape values for generated form used in request.post. Thanks @mynameisbogdan
+
+## v3.3.20 (2024/06/21)
+* maxTimeout should always be int
+* Check not running in Docker before logging version_main error
+* Update Cloudflare challenge and checkbox selectors. Thanks @tenettow & @21hsmw
+
+## v3.3.19 (2024/05/23)
+* Fix occasional headless issue on Linux when set to "false". Thanks @21hsmw
+
+## v3.3.18 (2024/05/20)
+
+* Fix LANG ENV for Linux
+* Fix Chrome v124+ not closing on Windows. Thanks @RileyXX
+
+## v3.3.17 (2024/04/09)
+
+* Fix file descriptor leak in service on quit(). Thanks @zkulis
+
 ## v3.3.16 (2024/02/28)
 
 * Fix of the subprocess.STARTUPINFO() call. Thanks @ceconelo

Dockerfile

@@ -62,17 +62,17 @@ ENTRYPOINT ["/usr/bin/dumb-init", "--"]
 CMD ["/usr/local/bin/python", "-u", "/app/flaresolverr.py"]
 
 # Local build
-# docker build -t ngosang/flaresolverr:3.3.16 .
+# docker build -t ngosang/flaresolverr:3.3.21 .
-# docker run -p 8191:8191 ngosang/flaresolverr:3.3.16
+# docker run -p 8191:8191 ngosang/flaresolverr:3.3.21
 
 # Multi-arch build
 # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
 # docker buildx create --use
-# docker buildx build -t ngosang/flaresolverr:3.3.16 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 .
+# docker buildx build -t ngosang/flaresolverr:3.3.21 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 .
 #   add --push to publish in DockerHub
 
 # Test multi-arch build
 # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
 # docker buildx create --use
-# docker buildx build -t ngosang/flaresolverr:3.3.16 --platform linux/arm/v7 --load .
+# docker buildx build -t ngosang/flaresolverr:3.3.21 --platform linux/arm/v7 --load .
-# docker run -p 8191:8191 --platform linux/arm/v7 ngosang/flaresolverr:3.3.16
+# docker run -p 8191:8191 --platform linux/arm/v7 ngosang/flaresolverr:3.3.21

README.md

@@ -45,7 +45,8 @@ Supported architectures are:
 | ARM32        | linux/arm/v7 |
 | ARM64        | linux/arm64  |
 
-We provide a `docker-compose.yml` configuration file. Clone this repository and execute `docker-compose up -d` to start
+We provide a `docker-compose.yml` configuration file. Clone this repository and execute
+`docker-compose up -d` _(Compose V1)_ or `docker compose up -d` _(Compose V2)_ to start
 the container.
 
 If you prefer the `docker cli` execute the following command.
@@ -78,7 +79,8 @@ This is the recommended way for Windows users.
 
 * Install [Python 3.11](https://www.python.org/downloads/).
 * Install [Chrome](https://www.google.com/intl/en_us/chrome/) (all OS) or [Chromium](https://www.chromium.org/getting-involved/download-chromium/) (just Linux, it doesn't work in Windows) web browser.
-* (Only in Linux / macOS) Install [Xvfb](https://en.wikipedia.org/wiki/Xvfb) package.
+* (Only in Linux) Install [Xvfb](https://en.wikipedia.org/wiki/Xvfb) package.
+* (Only in macOS) Install [XQuartz](https://www.xquartz.org/) package.
 * Clone this repository and open a shell in that path.
 * Run `pip install -r requirements.txt` command to install FlareSolverr dependencies.
 * Run `python src/flaresolverr.py` command to start FlareSolverr.
@@ -276,8 +278,8 @@ This is the same as `request.get` but it takes one more param:
 
 Environment variables are set differently depending on the operating system. Some examples:
 * Docker: Take a look at the Docker section in this document. Environment variables can be set in the `docker-compose.yml` file or in the Docker CLI command.
-* Linux: Run `export LOG_LEVEL=debug` and then start FlareSolverr in the same shell.
+* Linux: Run `export LOG_LEVEL=debug` and then run `flaresolverr` in the same shell.
-* Windows: Open `cmd.exe`, run `set LOG_LEVEL=debug` and then start FlareSolverr in the same shell.
+* Windows: Open `cmd.exe`, run `set LOG_LEVEL=debug` and then run `flaresolverr.exe` in the same shell.
 
 ## Prometheus exporter
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "flaresolverr",
-  "version": "3.3.16",
+  "version": "3.3.21",
   "description": "Proxy server to bypass Cloudflare protection",
   "author": "Diego Heras (ngosang / ngosang@hotmail.es)",
   "license": "MIT"

requirements.txt

@@ -4,8 +4,8 @@ selenium==4.15.2
 func-timeout==4.3.5
 prometheus-client==0.17.1
 # required by undetected_chromedriver
-requests==2.31.0
+requests==2.32.3
-certifi==2023.7.22
+certifi==2024.07.04
 websockets==11.0.3
 # only required for linux and macos
 xvfbwrapper==0.2.9; platform_system != "Windows"

src/flaresolverr.py

@@ -101,6 +101,9 @@ if __name__ == "__main__":
     logging.info(f'FlareSolverr {utils.get_flaresolverr_version()}')
     logging.debug('Debug log enabled')
 
+    # Get current OS for global variable
+    utils.get_current_platform()
+
     # test browser installation
     flaresolverr_service.test_browser_installation()
 

src/flaresolverr_service.py

@@ -3,7 +3,8 @@ import platform
 import sys
 import time
 from datetime import timedelta
-from urllib.parse import unquote
+from html import escape
+from urllib.parse import unquote, quote
 
 from func_timeout import FunctionTimedOut, func_timeout
 from selenium.common import TimeoutException
@@ -40,7 +41,7 @@ CHALLENGE_TITLES = [
 ]
 CHALLENGE_SELECTORS = [
     # Cloudflare
-    '#cf-challenge-running', '.ray_id', '.attack-box', '#cf-please-wait', '#challenge-spinner', '#trk_jschal_js',
+    '#cf-challenge-running', '.ray_id', '.attack-box', '#cf-please-wait', '#challenge-spinner', '#trk_jschal_js', '#turnstile-wrapper', '.lds-ring',
     # Custom CloudFlare for EbookParadijs, Film-Paleis, MuziekFabriek and Puur-Hollands
     'td.info #js_info',
     # Fairlane / pararius.com
@@ -119,7 +120,7 @@ def _controller_v1_handler(req: V1RequestBase) -> V1ResponseBase:
         logging.warning("Request parameter 'userAgent' was removed in FlareSolverr v2.")
 
     # set default values
-    if req.maxTimeout is None or req.maxTimeout < 1:
+    if req.maxTimeout is None or int(req.maxTimeout) < 1:
         req.maxTimeout = 60000
 
     # execute the command
@@ -218,9 +219,23 @@ def _cmd_sessions_destroy(req: V1RequestBase) -> V1ResponseBase:
         "message": "The session has been removed."
     })
 
+def _init_driver(driver):
+    try:
+        driver.execute_cdp_cmd('Page.enable', {})
+        driver.execute_cdp_cmd('Page.addScriptToEvaluateOnNewDocument', {
+            'source': """
+                Element.prototype._as = Element.prototype.attachShadow;
+                Element.prototype.attachShadow = function (params) {
+                    return this._as({mode: "open"})
+                };
+            """
+        })
+    except Exception as e:
+        logging.debug("Driver init exception: %s", repr(e))
+
 
 def _resolve_challenge(req: V1RequestBase, method: str) -> ChallengeResolutionT:
-    timeout = req.maxTimeout / 1000
+    timeout = int(req.maxTimeout) / 1000
     driver = None
     try:
         if req.session:
@@ -238,6 +253,7 @@ def _resolve_challenge(req: V1RequestBase, method: str) -> ChallengeResolutionT:
         else:
             driver = utils.get_webdriver(req.proxy)
             logging.debug('New instance of webdriver has been created to perform the request')
+        _init_driver(driver)
         return func_timeout(timeout, _evil_logic, (req, driver, method))
     except FunctionTimedOut:
         raise Exception(f'Error solving the challenge. Timeout after {timeout} seconds.')
@@ -245,27 +261,39 @@ def _resolve_challenge(req: V1RequestBase, method: str) -> ChallengeResolutionT:
         raise Exception('Error solving the challenge. ' + str(e).replace('\n', '\\n'))
     finally:
         if not req.session and driver is not None:
+            if utils.PLATFORM_VERSION == "nt":
+                driver.close()
             driver.quit()
             logging.debug('A used instance of webdriver has been destroyed')
 
 
+def get_shadowed_iframe(driver: WebDriver, css_selector: str):
+    logging.debug("Getting ShadowRoot by selector: %s", css_selector)
+    shadow_element = driver.execute_script("""
+        return document.querySelector(arguments[0]).shadowRoot.firstChild;
+    """, css_selector)
+    if shadow_element:
+        logging.debug("iframe found")
+    else:
+        logging.debug("iframe not found")
+    return shadow_element
+
+
 def click_verify(driver: WebDriver):
     try:
         logging.debug("Try to find the Cloudflare verify checkbox...")
-        iframe = driver.find_element(By.XPATH, "//iframe[starts-with(@id, 'cf-chl-widget-')]")
+        iframe = get_shadowed_iframe(driver, "div:not(:has(div))")
         driver.switch_to.frame(iframe)
-        checkbox = driver.find_element(
+        iframe_body = driver.find_element(By.CSS_SELECTOR, "body")
-            by=By.XPATH,
+        if iframe_body:
-            value='//*[@id="challenge-stage"]/div/label/input',
+            iframe_body.click()
-        )
-        if checkbox:
             actions = ActionChains(driver)
-            actions.move_to_element_with_offset(checkbox, 5, 7)
+            actions.move_to_element_with_offset(iframe_body, 10, 10)
-            actions.click(checkbox)
+            actions.click(iframe_body)
             actions.perform()
-            logging.debug("Cloudflare verify checkbox found and clicked!")
+            logging.debug("Attempted to click on iframe body")
-    except Exception:
+    except Exception as e:
-        logging.debug("Cloudflare verify checkbox not found on the page.")
+        logging.debug("Cloudflare verify checkbox not found on the page. %s", repr(e))
     finally:
         driver.switch_to.default_content()
 
@@ -437,7 +465,7 @@ def _post_request(req: V1RequestBase, driver: WebDriver):
             value = unquote(parts[1])
         except Exception:
             value = parts[1]
-        post_form += f'<input type="text" name="{name}" value="{value}"><br>'
+        post_form += f'<input type="text" name="{escape(quote(name))}" value="{escape(quote(value))}"><br>'
     post_form += '</form>'
     html_content = f"""
         <!DOCTYPE html>
@@ -447,6 +475,6 @@ def _post_request(req: V1RequestBase, driver: WebDriver):
             <script>document.getElementById('hackForm').submit();</script>
         </body>
         </html>"""
-    driver.get("data:text/html;charset=utf-8," + html_content)
+    driver.get("data:text/html;charset=utf-8,{html_content}".format(html_content=html_content))
     driver.start_session()
     driver.start_session()  # required to bypass Cloudflare

src/sessions.py

@@ -66,6 +66,8 @@ class SessionsStorage:
             return False
 
         session = self.sessions.pop(session_id)
+        if utils.PLATFORM_VERSION == "nt":
+            session.driver.close()
         session.driver.quit()
         return True
 

src/undetected_chromedriver/__init__.py

@@ -771,7 +771,9 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
 
     def quit(self):
         try:
+            self.service.stop()
             self.service.process.kill()
+            self.command_executor.close()
             self.service.process.wait(5)
             logger.debug("webdriver process ended")
         except (AttributeError, RuntimeError, OSError):
@@ -786,15 +788,6 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
             logger.debug("gracefully closed browser")
         except Exception as e:  # noqa
             pass
-        # Force kill Chrome process in Windows
-        # https://github.com/FlareSolverr/FlareSolverr/issues/772
-        if os.name == 'nt':
-            try:
-                subprocess.call(['taskkill', '/f', '/pid', str(self.browser_pid)],
-                                stdout=subprocess.DEVNULL,
-                                stderr=subprocess.DEVNULL)
-            except Exception:
-                pass
         if (
             hasattr(self, "keep_user_data_dir")
             and hasattr(self, "user_data_dir")

src/undetected_chromedriver/devtool.py

@@ -2,6 +2,7 @@ import asyncio
 from collections.abc import Mapping
 from collections.abc import Sequence
 from functools import wraps
+import os
 import logging
 import threading
 import time
@@ -187,4 +188,6 @@ def test():
 
     time.sleep(10)
 
+    if os.name == "nt":
+        driver.close()
     driver.quit()

src/undetected_chromedriver/patcher.py

@@ -68,8 +68,10 @@ class Patcher(object):
             # check if version_main_int is less than or equal to e.g 114
             self.is_old_chromedriver = version_main and version_main_int <= 114
         except (ValueError,TypeError):
-            # If the conversion fails, print an error message
+            # Check not running inside Docker
-            print("version_main cannot be converted to an integer")
+            if not os.path.exists("/app/chromedriver"):
+                # If the conversion fails, log an error message
+                logging.info("version_main cannot be converted to an integer")
             # Set self.is_old_chromedriver to False if the conversion fails
             self.is_old_chromedriver = False
 

src/utils.py

@@ -11,6 +11,7 @@ from selenium.webdriver.chrome.webdriver import WebDriver
 import undetected_chromedriver as uc
 
 FLARESOLVERR_VERSION = None
+PLATFORM_VERSION = None
 CHROME_EXE_PATH = None
 CHROME_MAJOR_VERSION = None
 USER_AGENT = None
@@ -38,6 +39,13 @@ def get_flaresolverr_version() -> str:
         FLARESOLVERR_VERSION = json.loads(f.read())['version']
         return FLARESOLVERR_VERSION
 
+def get_current_platform() -> str:
+    global PLATFORM_VERSION
+    if PLATFORM_VERSION is not None:
+        return PLATFORM_VERSION
+    PLATFORM_VERSION = os.name
+    return PLATFORM_VERSION
+
 
 def create_proxy_extension(proxy: dict) -> str:
     parsed_url = urllib.parse.urlparse(proxy['url'])
@@ -139,7 +147,7 @@ def get_webdriver(proxy: dict = None) -> WebDriver:
 
     language = os.environ.get('LANG', None)
     if language is not None:
-        options.add_argument('--lang=%s' % language)
+        options.add_argument('--accept-lang=%s' % language)
 
     # Fix for Chrome 117 | https://github.com/FlareSolverr/FlareSolverr/issues/910
     if USER_AGENT is not None:
@@ -314,6 +322,8 @@ def get_user_agent(driver=None) -> str:
         raise Exception("Error getting browser User-Agent. " + str(e))
     finally:
         if driver is not None:
+            if PLATFORM_VERSION == "nt":
+                driver.close()
             driver.quit()