Bump version 3.2.0

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## v3.2.0 (2023/05/23)
+
+* Support "proxy" param in requests and sessions
+* Support "cookies" param in requests
+* Fix Chromium exec permissions in Linux package
+* Update Python dependencies
+
 ## v3.1.2 (2023/04/02)
 
 * Fix headless mode in macOS

Dockerfile

@@ -61,17 +61,17 @@ ENTRYPOINT ["/usr/bin/dumb-init", "--"]
 CMD ["/usr/local/bin/python", "-u", "/app/flaresolverr.py"]
 
 # Local build
-# docker build -t ngosang/flaresolverr:3.0.0 .
+# docker build -t ngosang/flaresolverr:3.2.0 .
-# docker run -p 8191:8191 ngosang/flaresolverr:3.0.0
+# docker run -p 8191:8191 ngosang/flaresolverr:3.2.0
 
 # Multi-arch build
 # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
 # docker buildx create --use
-# docker buildx build -t ngosang/flaresolverr:3.0.0 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 .
+# docker buildx build -t ngosang/flaresolverr:3.2.0 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 .
 #   add --push to publish in DockerHub
 
 # Test multi-arch build
 # docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
 # docker buildx create --use
-# docker buildx build -t ngosang/flaresolverr:3.0.0 --platform linux/arm/v7 --load .
+# docker buildx build -t ngosang/flaresolverr:3.2.0 --platform linux/arm/v7 --load .
-# docker run -p 8191:8191 --platform linux/arm/v7 ngosang/flaresolverr:3.0.0
+# docker run -p 8191:8191 --platform linux/arm/v7 ngosang/flaresolverr:3.2.0

README.md

@@ -150,7 +150,7 @@ session. When you no longer need to use a session you should make sure to close
 | session             | Optional. Will send the request from and existing browser instance. If one is not sent it will create a temporary instance that will be destroyed immediately after the request is completed.                                                                                                                                                |
 | session_ttl_minutes | Optional. FlareSolverr will automatically rotate expired sessions based on the TTL provided in minutes.                                                                                                                                                                                                                                      |
 | maxTimeout          | Optional, default value 60000. Max timeout to solve the challenge in milliseconds.                                                                                                                                                                                                                                                           |
-| cookies             | Optional. Will be used by the headless browser. Follow [this](https://github.com/puppeteer/puppeteer/blob/v3.3.0/docs/api.md#pagesetcookiecookies) format.                                                                                                                                                                                   |
+| cookies             | Optional. Will be used by the headless browser. Eg: `"cookies": [{"name": "cookie1", "value": "value1"}, {"name": "cookie2", "value": "value2"}]`.                                                                                                                                                                                           |
 | returnOnlyCookies   | Optional, default false. Only returns the cookies. Response data, headers and other parts of the response are removed.                                                                                                                                                                                                                       |
 | proxy               | Optional, default disabled. Eg: `"proxy": {"url": "http://127.0.0.1:8888"}`. You must include the proxy schema in the URL: `http://`, `socks4://` or `socks5://`. Authorization (username/password) is not supported. (When the `session` parameter is set, the proxy is ignored; a session specific proxy can be set in `sessions.create`.) |
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "flaresolverr",
-  "version": "3.1.2",
+  "version": "3.2.0",
   "description": "Proxy server to bypass Cloudflare protection",
   "author": "Diego Heras (ngosang / ngosang@hotmail.es)",
   "license": "MIT"

requirements.txt

@@ -1,11 +1,11 @@
 bottle==0.12.25
 waitress==2.1.2
-selenium==4.8.3
+selenium==4.9.1
 func-timeout==4.3.5
 # required by undetected_chromedriver
-requests==2.28.2
+requests==2.31.0
-certifi==2022.12.7
+certifi==2023.5.7
-websockets==10.4
+websockets==11.0.3
 # only required for linux
 xvfbwrapper==0.2.9
 # only required for windows

src/build_package.py

@@ -50,7 +50,20 @@ def download_chromium():
     with zipfile.ZipFile(dl_path_zip, 'r') as zip_ref:
         zip_ref.extractall(dl_path)
     os.remove(dl_path_zip)
-    shutil.move(dl_path_folder, os.path.join(dl_path, "chrome"))
+
+    chrome_path = os.path.join(dl_path, "chrome")
+    shutil.move(dl_path_folder, chrome_path)
+    print("Extracted in: " + chrome_path)
+
+    if os.name != 'nt':
+        # Give executable permissions for *nix
+        # file * | grep executable | cut -d: -f1
+        print("Giving executable permissions...")
+        execs = ['chrome', 'chrome_crashpad_handler', 'chrome_sandbox', 'chrome-wrapper', 'nacl_helper',
+                 'nacl_helper_bootstrap', 'nacl_irt_x86_64.nexe', 'xdg-mime', 'xdg-settings']
+        for exec_file in execs:
+            exec_path = os.path.join(chrome_path, exec_file)
+            os.chmod(exec_path, 0o755)
 
 
 def run_pyinstaller():

src/flaresolverr_service.py

@@ -179,7 +179,7 @@ def _cmd_request_post(req: V1RequestBase) -> V1ResponseBase:
 def _cmd_sessions_create(req: V1RequestBase) -> V1ResponseBase:
     logging.debug("Creating new session...")
 
-    session, fresh = SESSIONS_STORAGE.create(session_id=req.session)
+    session, fresh = SESSIONS_STORAGE.create(session_id=req.session, proxy=req.proxy)
     session_id = session.session_id
 
     if not fresh:
@@ -236,13 +236,13 @@ def _resolve_challenge(req: V1RequestBase, method: str) -> ChallengeResolutionT:
 
             driver = session.driver
         else:
-            driver = utils.get_webdriver()
+            driver = utils.get_webdriver(req.proxy)
             logging.debug('New instance of webdriver has been created to perform the request')
         return func_timeout(timeout, _evil_logic, (req, driver, method))
     except FunctionTimedOut:
         raise Exception(f'Error solving the challenge. Timeout after {timeout} seconds.')
     except Exception as e:
-        raise Exception('Error solving the challenge. ' + str(e))
+        raise Exception('Error solving the challenge. ' + str(e).replace('\n', '\\n'))
     finally:
         if not req.session and driver is not None:
             driver.quit()
@@ -299,10 +299,22 @@ def _evil_logic(req: V1RequestBase, driver: WebDriver, method: str) -> Challenge
         _post_request(req, driver)
     else:
         driver.get(req.url)
-    if utils.get_config_log_html():
+
-        logging.debug(f"Response HTML:\n{driver.page_source}")
+    # set cookies if required
+    if req.cookies is not None and len(req.cookies) > 0:
+        logging.debug(f'Setting cookies...')
+        for cookie in req.cookies:
+            driver.delete_cookie(cookie['name'])
+            driver.add_cookie(cookie)
+        # reload the page
+        if method == 'POST':
+            _post_request(req, driver)
+        else:
+            driver.get(req.url)
 
     # wait for the page
+    if utils.get_config_log_html():
+        logging.debug(f"Response HTML:\n{driver.page_source}")
     html_element = driver.find_element(By.TAG_NAME, "html")
     page_title = driver.title
 

src/sessions.py

@@ -25,7 +25,8 @@ class SessionsStorage:
     def __init__(self):
         self.sessions = {}
 
-    def create(self, session_id: Optional[str] = None, force_new: Optional[bool] = False) -> Tuple[Session, bool]:
+    def create(self, session_id: Optional[str] = None, proxy: Optional[dict] = None,
+               force_new: Optional[bool] = False) -> Tuple[Session, bool]:
         """create creates new instance of WebDriver if necessary,
         assign defined (or newly generated) session_id to the instance
         and returns the session object. If a new session has been created
@@ -44,7 +45,7 @@ class SessionsStorage:
         if self.exists(session_id):
             return self.sessions[session_id], False
 
-        driver = utils.get_webdriver()
+        driver = utils.get_webdriver(proxy)
         created_at = datetime.now()
         session = Session(session_id, driver, created_at)
 

src/tests.py

@@ -1,4 +1,5 @@
 import unittest
+from typing import Optional
 
 from webtest import TestApp
 
@@ -7,7 +8,7 @@ import flaresolverr
 import utils
 
 
-def _find_obj_by_key(key: str, value: str, _list: list) -> dict | None:
+def _find_obj_by_key(key: str, value: str, _list: list) -> Optional[dict]:
     for obj in _list:
         if obj[key] == value:
             return obj
@@ -28,6 +29,8 @@ class TestFlareSolverr(unittest.TestCase):
     cloudflare_blocked_url = "https://cpasbiens3.fr/index.php?do=search&subaction=search"
 
     app = TestApp(flaresolverr.app)
+    # wait until the server is ready
+    app.get('/')
 
     def test_wrong_endpoint(self):
         res = self.app.get('/wrong', status=404)
@@ -236,7 +239,45 @@ class TestFlareSolverr(unittest.TestCase):
         self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
         self.assertEqual(utils.get_flaresolverr_version(), body.version)
 
-    # todo: test Cmd 'request.get' should return OK with 'cookies' param
+    def test_v1_endpoint_request_get_cookies_param(self):
+        res = self.app.post_json('/v1', {
+            "cmd": "request.get",
+            "url": self.google_url,
+            "cookies": [
+                {
+                    "name": "testcookie1",
+                    "value": "testvalue1"
+                },
+                {
+                    "name": "testcookie2",
+                    "value": "testvalue2"
+                }
+            ]
+        })
+        self.assertEqual(res.status_code, 200)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_OK, body.status)
+        self.assertEqual("Challenge not detected!", body.message)
+        self.assertGreater(body.startTimestamp, 10000)
+        self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
+        self.assertEqual(utils.get_flaresolverr_version(), body.version)
+
+        solution = body.solution
+        self.assertIn(self.google_url, solution.url)
+        self.assertEqual(solution.status, 200)
+        self.assertIs(len(solution.headers), 0)
+        self.assertIn("<title>Google</title>", solution.response)
+        self.assertGreater(len(solution.cookies), 1)
+        self.assertIn("Chrome/", solution.userAgent)
+
+        user_cookie1 = _find_obj_by_key("name", "testcookie1", solution.cookies)
+        self.assertIsNotNone(user_cookie1, "User cookie 1 not found")
+        self.assertEqual("testvalue1", user_cookie1["value"])
+
+        user_cookie2 = _find_obj_by_key("name", "testcookie2", solution.cookies)
+        self.assertIsNotNone(user_cookie2, "User cookie 2 not found")
+        self.assertEqual("testvalue2", user_cookie2["value"])
 
     def test_v1_endpoint_request_get_returnOnlyCookies_param(self):
         res = self.app.post_json('/v1', {
@@ -261,10 +302,88 @@ class TestFlareSolverr(unittest.TestCase):
         self.assertGreater(len(solution.cookies), 0)
         self.assertIn("Chrome/", solution.userAgent)
 
-    # todo: test Cmd 'request.get' should return OK with HTTP 'proxy' param
+    def test_v1_endpoint_request_get_proxy_http_param(self):
-    # todo: test Cmd 'request.get' should return OK with HTTP 'proxy' param with credentials
+        """
-    # todo: test Cmd 'request.get' should return OK with SOCKSv5 'proxy' param
+        To configure TinyProxy in local:
-    # todo: test Cmd 'request.get' should fail with wrong 'proxy' param
+           * sudo vim /etc/tinyproxy/tinyproxy.conf
+              * edit => LogFile "/tmp/tinyproxy.log"
+              * edit => Syslog Off
+           * sudo tinyproxy -d
+           * sudo tail -f /tmp/tinyproxy.log
+        """
+        res = self.app.post_json('/v1', {
+            "cmd": "request.get",
+            "url": self.google_url,
+            "proxy": {
+                "url": self.proxy_url
+            }
+        })
+        self.assertEqual(res.status_code, 200)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_OK, body.status)
+        self.assertEqual("Challenge not detected!", body.message)
+        self.assertGreater(body.startTimestamp, 10000)
+        self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
+        self.assertEqual(utils.get_flaresolverr_version(), body.version)
+
+        solution = body.solution
+        self.assertIn(self.google_url, solution.url)
+        self.assertEqual(solution.status, 200)
+        self.assertIs(len(solution.headers), 0)
+        self.assertIn("<title>Google</title>", solution.response)
+        self.assertGreater(len(solution.cookies), 0)
+        self.assertIn("Chrome/", solution.userAgent)
+
+    def test_v1_endpoint_request_get_proxy_socks_param(self):
+        """
+        To configure Dante in local:
+           * https://linuxhint.com/set-up-a-socks5-proxy-on-ubuntu-with-dante/
+           * sudo vim /etc/sockd.conf
+           * sudo systemctl restart sockd.service
+           * curl --socks5 socks5://127.0.0.1:1080 https://www.google.com
+        """
+        res = self.app.post_json('/v1', {
+            "cmd": "request.get",
+            "url": self.google_url,
+            "proxy": {
+                "url": self.proxy_socks_url
+            }
+        })
+        self.assertEqual(res.status_code, 200)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_OK, body.status)
+        self.assertEqual("Challenge not detected!", body.message)
+        self.assertGreater(body.startTimestamp, 10000)
+        self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
+        self.assertEqual(utils.get_flaresolverr_version(), body.version)
+
+        solution = body.solution
+        self.assertIn(self.google_url, solution.url)
+        self.assertEqual(solution.status, 200)
+        self.assertIs(len(solution.headers), 0)
+        self.assertIn("<title>Google</title>", solution.response)
+        self.assertGreater(len(solution.cookies), 0)
+        self.assertIn("Chrome/", solution.userAgent)
+
+    def test_v1_endpoint_request_get_proxy_wrong_param(self):
+        res = self.app.post_json('/v1', {
+            "cmd": "request.get",
+            "url": self.google_url,
+            "proxy": {
+                "url": "http://127.0.0.1:43210"
+            }
+        }, status=500)
+        self.assertEqual(res.status_code, 500)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_ERROR, body.status)
+        self.assertIn("Error: Error solving the challenge. Message: unknown error: net::ERR_PROXY_CONNECTION_FAILED",
+                      body.message)
+        self.assertGreater(body.startTimestamp, 10000)
+        self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
+        self.assertEqual(utils.get_flaresolverr_version(), body.version)
 
     def test_v1_endpoint_request_get_fail_timeout(self):
         res = self.app.post_json('/v1', {
@@ -401,6 +520,20 @@ class TestFlareSolverr(unittest.TestCase):
         self.assertEqual("Session created successfully.", body.message)
         self.assertEqual(body.session, "test_create_session")
 
+    def test_v1_endpoint_sessions_create_with_proxy(self):
+        res = self.app.post_json('/v1', {
+            "cmd": "sessions.create",
+            "proxy": {
+                "url": self.proxy_url
+            }
+        })
+        self.assertEqual(res.status_code, 200)
+
+        body = V1ResponseBase(res.json)
+        self.assertEqual(STATUS_OK, body.status)
+        self.assertEqual("Session created successfully.", body.message)
+        self.assertIsNotNone(body.session)
+
     def test_v1_endpoint_sessions_list(self):
         self.app.post_json('/v1', {
             "cmd": "sessions.create",

src/tests_sites.py

@@ -39,6 +39,8 @@ def asset_cloudflare_solution(self, res, site_url, site_text):
 
 class TestFlareSolverr(unittest.TestCase):
     app = TestApp(flaresolverr.app)
+    # wait until the server is ready
+    app.get('/')
 
     def test_v1_endpoint_request_get_cloudflare(self):
         sites_get = [

src/utils.py

@@ -36,7 +36,7 @@ def get_flaresolverr_version() -> str:
         return FLARESOLVERR_VERSION
 
 
-def get_webdriver() -> WebDriver:
+def get_webdriver(proxy: dict = None) -> WebDriver:
     global PATCHED_DRIVER_PATH
     logging.debug('Launching web browser...')
 
@@ -55,6 +55,11 @@ def get_webdriver() -> WebDriver:
     options.add_argument('--ignore-certificate-errors')
     options.add_argument('--ignore-ssl-errors')
 
+    if proxy and 'url' in proxy:
+        proxy_url = proxy['url']
+        logging.debug("Using webdriver proxy: %s", proxy_url)
+        options.add_argument('--proxy-server=%s' % proxy_url)
+
     # note: headless mode is detected (options.headless = True)
     # we launch the browser in head-full mode with the window hidden
     windows_headless = False
@@ -106,12 +111,15 @@ def get_chrome_exe_path() -> str:
         return CHROME_EXE_PATH
     # linux pyinstaller bundle
     chrome_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'chrome', "chrome")
-    if os.path.exists(chrome_path) and os.access(chrome_path, os.X_OK):
+    if os.path.exists(chrome_path):
+        if not os.access(chrome_path, os.X_OK):
+            raise Exception(f'Chrome binary "{chrome_path}" is not executable. '
+                            f'Please, extract the archive with "tar xzf <file.tar.gz>".')
         CHROME_EXE_PATH = chrome_path
         return CHROME_EXE_PATH
     # windows pyinstaller bundle
     chrome_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'chrome', "chrome.exe")
-    if os.path.exists(chrome_path) and os.access(chrome_path, os.X_OK):
+    if os.path.exists(chrome_path):
         CHROME_EXE_PATH = chrome_path
         return CHROME_EXE_PATH
     # system