Resolve turnstile captcha (#1634)

Fixes #804

src/dtos.py

@@ -11,6 +11,7 @@ class ChallengeResolutionResultT:
     cookies: list = None
     userAgent: str = None
     screenshot: str | None = None
+    turnstile_token: str = None
 
     def __init__(self, _dict):
         self.__dict__.update(_dict)
@@ -48,6 +49,8 @@ class V1RequestBase(object):
     waitInSeconds: int = None
     # Optional resource blocking flag (blocks images, CSS, and fonts)
     disableMedia: bool = None
+    # Optional when you've got a turnstile captcha that needs to be clicked after X number of Tab presses
+    tabs_till_verify : int = None
 
     def __init__(self, _dict):
         self.__dict__.update(_dict)

src/flaresolverr_service.py

@@ -48,6 +48,11 @@ CHALLENGE_SELECTORS = [
     # Fairlane / pararius.com
     'div.vc div.text-box h2'
 ]
+
+TURNSTILE_SELECTORS = [
+    "input[name='cf-turnstile-response']"
+]
+
 SHORT_TIMEOUT = 1
 SESSIONS_STORAGE = SessionsStorage()
 
@@ -253,12 +258,17 @@ def _resolve_challenge(req: V1RequestBase, method: str) -> ChallengeResolutionT:
             logging.debug('A used instance of webdriver has been destroyed')
 
 
-def click_verify(driver: WebDriver):
+def click_verify(driver: WebDriver, num_tabs: int = 1):
     try:
         logging.debug("Try to find the Cloudflare verify checkbox...")
         actions = ActionChains(driver)
-        actions.pause(5).send_keys(Keys.TAB).pause(1).send_keys(Keys.SPACE).perform()
-        logging.debug("Cloudflare verify checkbox found and clicked!")
+        actions.pause(5)
+        for _ in range(num_tabs):
+            actions.send_keys(Keys.TAB).pause(0.1)
+        actions.pause(1)
+        actions.send_keys(Keys.SPACE).perform()
+        
+        logging.debug(f"Cloudflare verify checkbox clicked after {num_tabs} tabs!")
     except Exception:
         logging.debug("Cloudflare verify checkbox not found on the page.")
     finally:
@@ -281,6 +291,47 @@ def click_verify(driver: WebDriver):
 
     time.sleep(2)
 
+def _get_turnstile_token(driver: WebDriver, tabs: int):
+    token_input = driver.find_element(By.CSS_SELECTOR, "input[name='cf-turnstile-response']")
+    current_value = token_input.get_attribute("value")
+    while True:
+        click_verify(driver, num_tabs=tabs)
+        turnstile_token = token_input.get_attribute("value")
+        if turnstile_token:
+            if turnstile_token != current_value:
+                logging.info(f"Turnstile token: {turnstile_token}")
+                return turnstile_token
+        logging.debug(f"Failed to extract token possibly click failed")        
+
+        # reset focus
+        driver.execute_script("""
+            let el = document.createElement('button');
+            el.style.position='fixed';
+            el.style.top='0';
+            el.style.left='0';
+            document.body.prepend(el);
+            el.focus();
+        """)
+        time.sleep(1)
+
+def _resolve_turnstile_captcha(req: V1RequestBase, driver: WebDriver):
+    turnstile_token = None
+    if req.tabs_till_verify is not None:
+        logging.debug(f'Navigating to... {req.url} in order to pass the turnstile challenge')
+        driver.get(req.url)
+
+        turnstile_challenge_found = False
+        for selector in TURNSTILE_SELECTORS:
+            found_elements = driver.find_elements(By.CSS_SELECTOR, selector)   
+            if len(found_elements) > 0:
+                turnstile_challenge_found = True
+                logging.info("Turnstile challenge detected. Selector found: " + selector)
+                break
+        if turnstile_challenge_found:
+            turnstile_token = _get_turnstile_token(driver=driver, tabs=req.tabs_till_verify)
+        else:
+            logging.debug(f'Turnstile challenge not found')
+    return turnstile_token
 
 def _evil_logic(req: V1RequestBase, driver: WebDriver, method: str) -> ChallengeResolutionT:
     res = ChallengeResolutionT({})
@@ -315,11 +366,15 @@ def _evil_logic(req: V1RequestBase, driver: WebDriver, method: str) -> Challenge
 
     # navigate to the page
     logging.debug(f"Navigating to... {req.url}")
+    turnstile_token = None
 
     if method == "POST":
         _post_request(req, driver)
     else:
-        driver.get(req.url)
+        if req.tabs_till_verify is None:
+            driver.get(req.url)
+        else:
+            turnstile_token = _resolve_turnstile_captcha(req, driver)
 
     # set cookies if required
     if req.cookies is not None and len(req.cookies) > 0:
@@ -413,6 +468,7 @@ def _evil_logic(req: V1RequestBase, driver: WebDriver, method: str) -> Challenge
     challenge_res.status = 200  # todo: fix, selenium not provides this info
     challenge_res.cookies = driver.get_cookies()
     challenge_res.userAgent = utils.get_user_agent(driver)
+    challenge_res.turnstile_token = turnstile_token
 
     if not req.returnOnlyCookies:
         challenge_res.headers = {}  # todo: fix, selenium not provides this info