mirror of
https://github.com/FlareSolverr/FlareSolverr.git
synced 2025-12-06 09:38:49 +01:00
Compare commits
21 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b42c22f5b1 | ||
|
|
9c62410a8b | ||
|
|
b8768ae17d | ||
|
|
9b2c602a1f | ||
|
|
8316350b98 | ||
|
|
33307ce461 | ||
|
|
cedb7bc54e | ||
|
|
6ecaf2362c | ||
|
|
3c97c9603a | ||
|
|
efaa5f31b6 | ||
|
|
4db85a2d0f | ||
|
|
66b9db21e5 | ||
|
|
ab0fe58d4a | ||
|
|
f68ddb7573 | ||
|
|
ac77110578 | ||
|
|
a9d1a2de2d | ||
|
|
ab5f14d6c3 | ||
|
|
e0bf02fb8b | ||
|
|
82a1cd835a | ||
|
|
7017715e21 | ||
|
|
ae18559db1 |
7
.github/ISSUE_TEMPLATE/bug_report.yml
vendored
7
.github/ISSUE_TEMPLATE/bug_report.yml
vendored
@@ -8,6 +8,13 @@ body:
|
|||||||
options:
|
options:
|
||||||
- label: I have checked the README
|
- label: I have checked the README
|
||||||
required: true
|
required: true
|
||||||
|
- type: checkboxes
|
||||||
|
attributes:
|
||||||
|
label: Have you followed our Troubleshooting?
|
||||||
|
description: Please follow our <a href="https://github.com/FlareSolverr/FlareSolverr/wiki/Troubleshooting">Troubleshooting</a>.
|
||||||
|
options:
|
||||||
|
- label: I have followed your Troubleshooting
|
||||||
|
required: true
|
||||||
- type: checkboxes
|
- type: checkboxes
|
||||||
attributes:
|
attributes:
|
||||||
label: Is there already an issue for your problem?
|
label: Is there already an issue for your problem?
|
||||||
|
|||||||
2
.github/workflows/autotag.yml
vendored
2
.github/workflows/autotag.yml
vendored
@@ -6,7 +6,7 @@ on:
|
|||||||
- "master"
|
- "master"
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
build:
|
tag-release:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
-
|
-
|
||||||
|
|||||||
2
.github/workflows/release-docker.yml
vendored
2
.github/workflows/release-docker.yml
vendored
@@ -6,7 +6,7 @@ on:
|
|||||||
- 'v*.*.*'
|
- 'v*.*.*'
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
build:
|
build-docker-images:
|
||||||
runs-on: ubuntu-22.04
|
runs-on: ubuntu-22.04
|
||||||
steps:
|
steps:
|
||||||
-
|
-
|
||||||
|
|||||||
4
.github/workflows/release.yml
vendored
4
.github/workflows/release.yml
vendored
@@ -36,7 +36,7 @@ jobs:
|
|||||||
draft: false
|
draft: false
|
||||||
prerelease: false
|
prerelease: false
|
||||||
|
|
||||||
build-linux:
|
build-linux-package:
|
||||||
name: Build Linux binary
|
name: Build Linux binary
|
||||||
needs: create-release
|
needs: create-release
|
||||||
runs-on: ubuntu-22.04
|
runs-on: ubuntu-22.04
|
||||||
@@ -65,7 +65,7 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
asset_paths: '["./dist/flaresolverr_*"]'
|
asset_paths: '["./dist/flaresolverr_*"]'
|
||||||
|
|
||||||
build-windows:
|
build-windows-package:
|
||||||
name: Build Windows binary
|
name: Build Windows binary
|
||||||
needs: create-release
|
needs: create-release
|
||||||
runs-on: windows-2022
|
runs-on: windows-2022
|
||||||
|
|||||||
25
CHANGELOG.md
25
CHANGELOG.md
@@ -1,5 +1,30 @@
|
|||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
|
## v3.2.1 (2023/06/10)
|
||||||
|
|
||||||
|
* Kill dead Chrome processes in Windows
|
||||||
|
* Fix Chrome GL erros in ASUSTOR NAS
|
||||||
|
|
||||||
|
## v3.2.0 (2023/05/23)
|
||||||
|
|
||||||
|
* Support "proxy" param in requests and sessions
|
||||||
|
* Support "cookies" param in requests
|
||||||
|
* Fix Chromium exec permissions in Linux package
|
||||||
|
* Update Python dependencies
|
||||||
|
|
||||||
|
## v3.1.2 (2023/04/02)
|
||||||
|
|
||||||
|
* Fix headless mode in macOS
|
||||||
|
* Remove redundant artifact from Windows binary package
|
||||||
|
* Bump Selenium dependency
|
||||||
|
|
||||||
|
## v3.1.1 (2023/03/25)
|
||||||
|
|
||||||
|
* Distribute binary executables in compressed package
|
||||||
|
* Add icon for binary executable
|
||||||
|
* Include information about supported architectures in the readme
|
||||||
|
* Check Python version on start
|
||||||
|
|
||||||
## v3.1.0 (2023/03/20)
|
## v3.1.0 (2023/03/20)
|
||||||
|
|
||||||
* Build binaries for Linux x64 and Windows x64
|
* Build binaries for Linux x64 and Windows x64
|
||||||
|
|||||||
10
Dockerfile
10
Dockerfile
@@ -61,17 +61,17 @@ ENTRYPOINT ["/usr/bin/dumb-init", "--"]
|
|||||||
CMD ["/usr/local/bin/python", "-u", "/app/flaresolverr.py"]
|
CMD ["/usr/local/bin/python", "-u", "/app/flaresolverr.py"]
|
||||||
|
|
||||||
# Local build
|
# Local build
|
||||||
# docker build -t ngosang/flaresolverr:3.0.0 .
|
# docker build -t ngosang/flaresolverr:3.2.0 .
|
||||||
# docker run -p 8191:8191 ngosang/flaresolverr:3.0.0
|
# docker run -p 8191:8191 ngosang/flaresolverr:3.2.0
|
||||||
|
|
||||||
# Multi-arch build
|
# Multi-arch build
|
||||||
# docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
|
# docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
|
||||||
# docker buildx create --use
|
# docker buildx create --use
|
||||||
# docker buildx build -t ngosang/flaresolverr:3.0.0 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 .
|
# docker buildx build -t ngosang/flaresolverr:3.2.0 --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64/v8 .
|
||||||
# add --push to publish in DockerHub
|
# add --push to publish in DockerHub
|
||||||
|
|
||||||
# Test multi-arch build
|
# Test multi-arch build
|
||||||
# docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
|
# docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
|
||||||
# docker buildx create --use
|
# docker buildx create --use
|
||||||
# docker buildx build -t ngosang/flaresolverr:3.0.0 --platform linux/arm/v7 --load .
|
# docker buildx build -t ngosang/flaresolverr:3.2.0 --platform linux/arm/v7 --load .
|
||||||
# docker run -p 8191:8191 --platform linux/arm/v7 ngosang/flaresolverr:3.0.0
|
# docker run -p 8191:8191 --platform linux/arm/v7 ngosang/flaresolverr:3.2.0
|
||||||
|
|||||||
16
README.md
16
README.md
@@ -64,12 +64,18 @@ Remember to restart the Docker daemon and the container after the update.
|
|||||||
|
|
||||||
### Precompiled binaries
|
### Precompiled binaries
|
||||||
|
|
||||||
|
> **Warning**
|
||||||
|
> Precompiled binaries are only available for x64 architecture. For other architectures see Docker images.
|
||||||
|
|
||||||
This is the recommended way for Windows users.
|
This is the recommended way for Windows users.
|
||||||
* Download the [FlareSolverr executable](https://github.com/FlareSolverr/FlareSolverr/releases) from the release's page. It is available for Windows x64 and Linux x64.
|
* Download the [FlareSolverr executable](https://github.com/FlareSolverr/FlareSolverr/releases) from the release's page. It is available for Windows x64 and Linux x64.
|
||||||
* Execute FlareSolverr binary. In the environment variables section you can find how to change the configuration.
|
* Execute FlareSolverr binary. In the environment variables section you can find how to change the configuration.
|
||||||
|
|
||||||
### From source code
|
### From source code
|
||||||
|
|
||||||
|
> **Warning**
|
||||||
|
> Installing from source code only works for x64 architecture. For other architectures see Docker images.
|
||||||
|
|
||||||
* Install [Python 3.11](https://www.python.org/downloads/).
|
* Install [Python 3.11](https://www.python.org/downloads/).
|
||||||
* Install [Chrome](https://www.google.com/intl/en_us/chrome/) (all OS) or [Chromium](https://www.chromium.org/getting-involved/download-chromium/) (just Linux, it doesn't work in Windows) web browser.
|
* Install [Chrome](https://www.google.com/intl/en_us/chrome/) (all OS) or [Chromium](https://www.chromium.org/getting-involved/download-chromium/) (just Linux, it doesn't work in Windows) web browser.
|
||||||
* (Only in Linux / macOS) Install [Xvfb](https://en.wikipedia.org/wiki/Xvfb) package.
|
* (Only in Linux / macOS) Install [Xvfb](https://en.wikipedia.org/wiki/Xvfb) package.
|
||||||
@@ -144,11 +150,12 @@ session. When you no longer need to use a session you should make sure to close
|
|||||||
| session | Optional. Will send the request from and existing browser instance. If one is not sent it will create a temporary instance that will be destroyed immediately after the request is completed. |
|
| session | Optional. Will send the request from and existing browser instance. If one is not sent it will create a temporary instance that will be destroyed immediately after the request is completed. |
|
||||||
| session_ttl_minutes | Optional. FlareSolverr will automatically rotate expired sessions based on the TTL provided in minutes. |
|
| session_ttl_minutes | Optional. FlareSolverr will automatically rotate expired sessions based on the TTL provided in minutes. |
|
||||||
| maxTimeout | Optional, default value 60000. Max timeout to solve the challenge in milliseconds. |
|
| maxTimeout | Optional, default value 60000. Max timeout to solve the challenge in milliseconds. |
|
||||||
| cookies | Optional. Will be used by the headless browser. Follow [this](https://github.com/puppeteer/puppeteer/blob/v3.3.0/docs/api.md#pagesetcookiecookies) format. |
|
| cookies | Optional. Will be used by the headless browser. Eg: `"cookies": [{"name": "cookie1", "value": "value1"}, {"name": "cookie2", "value": "value2"}]`. |
|
||||||
| returnOnlyCookies | Optional, default false. Only returns the cookies. Response data, headers and other parts of the response are removed. |
|
| returnOnlyCookies | Optional, default false. Only returns the cookies. Response data, headers and other parts of the response are removed. |
|
||||||
| proxy | Optional, default disabled. Eg: `"proxy": {"url": "http://127.0.0.1:8888"}`. You must include the proxy schema in the URL: `http://`, `socks4://` or `socks5://`. Authorization (username/password) is not supported. (When the `session` parameter is set, the proxy is ignored; a session specific proxy can be set in `sessions.create`.) |
|
| proxy | Optional, default disabled. Eg: `"proxy": {"url": "http://127.0.0.1:8888"}`. You must include the proxy schema in the URL: `http://`, `socks4://` or `socks5://`. Authorization (username/password) is not supported. (When the `session` parameter is set, the proxy is ignored; a session specific proxy can be set in `sessions.create`.) |
|
||||||
|
|
||||||
:warning: If you want to use Cloudflare clearance cookie in your scripts, make sure you use the FlareSolverr User-Agent too. If they don't match you will see the challenge.
|
> **Warning**
|
||||||
|
> If you want to use Cloudflare clearance cookie in your scripts, make sure you use the FlareSolverr User-Agent too. If they don't match you will see the challenge.
|
||||||
|
|
||||||
Example response from running the `curl` above:
|
Example response from running the `curl` above:
|
||||||
|
|
||||||
@@ -238,13 +245,14 @@ Environment variables are set differently depending on the operating system. Som
|
|||||||
|
|
||||||
## Captcha Solvers
|
## Captcha Solvers
|
||||||
|
|
||||||
:warning: At this time none of the captcha solvers work. You can check the status in the open issues. Any help is welcome.
|
> **Warning**
|
||||||
|
> At this time none of the captcha solvers work. You can check the status in the open issues. Any help is welcome.
|
||||||
|
|
||||||
Sometimes CloudFlare not only gives mathematical computations and browser tests, sometimes they also require the user to
|
Sometimes CloudFlare not only gives mathematical computations and browser tests, sometimes they also require the user to
|
||||||
solve a captcha.
|
solve a captcha.
|
||||||
If this is the case, FlareSolverr will return the error `Captcha detected but no automatic solver is configured.`
|
If this is the case, FlareSolverr will return the error `Captcha detected but no automatic solver is configured.`
|
||||||
|
|
||||||
FlareSolverr can be customized to solve the captchas automatically by setting the environment variable `CAPTCHA_SOLVER`
|
FlareSolverr can be customized to solve the CAPTCHA automatically by setting the environment variable `CAPTCHA_SOLVER`
|
||||||
to the file name of one of the adapters inside the [/captcha](src/captcha) directory.
|
to the file name of one of the adapters inside the [/captcha](src/captcha) directory.
|
||||||
|
|
||||||
## Related projects
|
## Related projects
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "flaresolverr",
|
"name": "flaresolverr",
|
||||||
"version": "3.1.0",
|
"version": "3.2.2",
|
||||||
"description": "Proxy server to bypass Cloudflare protection",
|
"description": "Proxy server to bypass Cloudflare protection",
|
||||||
"author": "Diego Heras (ngosang / ngosang@hotmail.es)",
|
"author": "Diego Heras (ngosang / ngosang@hotmail.es)",
|
||||||
"license": "MIT"
|
"license": "MIT"
|
||||||
|
|||||||
@@ -1,11 +1,11 @@
|
|||||||
bottle==0.12.25
|
bottle==0.12.25
|
||||||
waitress==2.1.2
|
waitress==2.1.2
|
||||||
selenium==4.8.2
|
selenium==4.9.1
|
||||||
func-timeout==4.3.5
|
func-timeout==4.3.5
|
||||||
# required by undetected_chromedriver
|
# required by undetected_chromedriver
|
||||||
requests==2.28.2
|
requests==2.31.0
|
||||||
certifi==2022.12.7
|
certifi==2023.5.7
|
||||||
websockets==10.4
|
websockets==11.0.3
|
||||||
# only required for linux
|
# only required for linux
|
||||||
xvfbwrapper==0.2.9
|
xvfbwrapper==0.2.9
|
||||||
# only required for windows
|
# only required for windows
|
||||||
|
|||||||
BIN
resources/flaresolverr_logo.ico
Normal file
BIN
resources/flaresolverr_logo.ico
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 8.8 KiB |
@@ -50,24 +50,43 @@ def download_chromium():
|
|||||||
with zipfile.ZipFile(dl_path_zip, 'r') as zip_ref:
|
with zipfile.ZipFile(dl_path_zip, 'r') as zip_ref:
|
||||||
zip_ref.extractall(dl_path)
|
zip_ref.extractall(dl_path)
|
||||||
os.remove(dl_path_zip)
|
os.remove(dl_path_zip)
|
||||||
shutil.move(dl_path_folder, os.path.join(dl_path, "chrome"))
|
|
||||||
|
chrome_path = os.path.join(dl_path, "chrome")
|
||||||
|
shutil.move(dl_path_folder, chrome_path)
|
||||||
|
print("Extracted in: " + chrome_path)
|
||||||
|
|
||||||
|
if os.name != 'nt':
|
||||||
|
# Give executable permissions for *nix
|
||||||
|
# file * | grep executable | cut -d: -f1
|
||||||
|
print("Giving executable permissions...")
|
||||||
|
execs = ['chrome', 'chrome_crashpad_handler', 'chrome_sandbox', 'chrome-wrapper', 'nacl_helper',
|
||||||
|
'nacl_helper_bootstrap', 'nacl_irt_x86_64.nexe', 'xdg-mime', 'xdg-settings']
|
||||||
|
for exec_file in execs:
|
||||||
|
exec_path = os.path.join(chrome_path, exec_file)
|
||||||
|
os.chmod(exec_path, 0o755)
|
||||||
|
|
||||||
|
|
||||||
def run_pyinstaller():
|
def run_pyinstaller():
|
||||||
sep = ';' if os.name == 'nt' else ':'
|
sep = ';' if os.name == 'nt' else ':'
|
||||||
subprocess.check_call([sys.executable, "-m", "PyInstaller",
|
subprocess.check_call([sys.executable, "-m", "PyInstaller",
|
||||||
"--onefile",
|
"--icon", "resources/flaresolverr_logo.ico",
|
||||||
"--add-data", f"package.json{sep}.",
|
"--add-data", f"package.json{sep}.",
|
||||||
"--add-data", f"{os.path.join('dist_chrome', 'chrome')}{sep}chrome",
|
"--add-data", f"{os.path.join('dist_chrome', 'chrome')}{sep}chrome",
|
||||||
os.path.join("src", "flaresolverr.py")],
|
os.path.join("src", "flaresolverr.py")],
|
||||||
cwd=os.pardir)
|
cwd=os.pardir)
|
||||||
exe_folder = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'dist')
|
|
||||||
exe_name = 'flaresolverr.exe' if os.name == 'nt' else 'flaresolverr'
|
|
||||||
exe_new_name = 'flaresolverr_windows_x64.exe' if os.name == 'nt' else 'flaresolverr_linux_x64'
|
def compress_package():
|
||||||
exe_path = os.path.join(exe_folder, exe_name)
|
dist_folder = os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, 'dist')
|
||||||
exe_new_path = os.path.join(exe_folder, exe_new_name)
|
package_folder = os.path.join(dist_folder, 'package')
|
||||||
shutil.move(exe_path, exe_new_path)
|
shutil.move(os.path.join(dist_folder, 'flaresolverr'), os.path.join(package_folder, 'flaresolverr'))
|
||||||
print("Executable path: " + exe_new_path)
|
print("Package folder: " + package_folder)
|
||||||
|
|
||||||
|
compr_format = 'zip' if os.name == 'nt' else 'gztar'
|
||||||
|
compr_file_name = 'flaresolverr_windows_x64' if os.name == 'nt' else 'flaresolverr_linux_x64'
|
||||||
|
compr_file_path = os.path.join(dist_folder, compr_file_name)
|
||||||
|
shutil.make_archive(compr_file_path, compr_format, package_folder)
|
||||||
|
print("Compressed file path: " + compr_file_path)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
@@ -83,4 +102,7 @@ if __name__ == "__main__":
|
|||||||
print("Building pyinstaller executable... ")
|
print("Building pyinstaller executable... ")
|
||||||
run_pyinstaller()
|
run_pyinstaller()
|
||||||
|
|
||||||
|
print("Compressing package... ")
|
||||||
|
compress_package()
|
||||||
|
|
||||||
# NOTE: python -m pip install pyinstaller
|
# NOTE: python -m pip install pyinstaller
|
||||||
|
|||||||
@@ -61,6 +61,10 @@ def controller_v1():
|
|||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
# check python version
|
||||||
|
if sys.version_info < (3, 9):
|
||||||
|
raise Exception("The Python version is less than 3.9, a version equal to or higher is required.")
|
||||||
|
|
||||||
# fix ssl certificates for compiled binaries
|
# fix ssl certificates for compiled binaries
|
||||||
# https://github.com/pyinstaller/pyinstaller/issues/7229
|
# https://github.com/pyinstaller/pyinstaller/issues/7229
|
||||||
# https://stackoverflow.com/questions/55736855/how-to-change-the-cafile-argument-in-the-ssl-module-in-python3
|
# https://stackoverflow.com/questions/55736855/how-to-change-the-cafile-argument-in-the-ssl-module-in-python3
|
||||||
|
|||||||
@@ -179,7 +179,7 @@ def _cmd_request_post(req: V1RequestBase) -> V1ResponseBase:
|
|||||||
def _cmd_sessions_create(req: V1RequestBase) -> V1ResponseBase:
|
def _cmd_sessions_create(req: V1RequestBase) -> V1ResponseBase:
|
||||||
logging.debug("Creating new session...")
|
logging.debug("Creating new session...")
|
||||||
|
|
||||||
session, fresh = SESSIONS_STORAGE.create(session_id=req.session)
|
session, fresh = SESSIONS_STORAGE.create(session_id=req.session, proxy=req.proxy)
|
||||||
session_id = session.session_id
|
session_id = session.session_id
|
||||||
|
|
||||||
if not fresh:
|
if not fresh:
|
||||||
@@ -236,13 +236,13 @@ def _resolve_challenge(req: V1RequestBase, method: str) -> ChallengeResolutionT:
|
|||||||
|
|
||||||
driver = session.driver
|
driver = session.driver
|
||||||
else:
|
else:
|
||||||
driver = utils.get_webdriver()
|
driver = utils.get_webdriver(req.proxy)
|
||||||
logging.debug('New instance of webdriver has been created to perform the request')
|
logging.debug('New instance of webdriver has been created to perform the request')
|
||||||
return func_timeout(timeout, _evil_logic, (req, driver, method))
|
return func_timeout(timeout, _evil_logic, (req, driver, method))
|
||||||
except FunctionTimedOut:
|
except FunctionTimedOut:
|
||||||
raise Exception(f'Error solving the challenge. Timeout after {timeout} seconds.')
|
raise Exception(f'Error solving the challenge. Timeout after {timeout} seconds.')
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
raise Exception('Error solving the challenge. ' + str(e))
|
raise Exception('Error solving the challenge. ' + str(e).replace('\n', '\\n'))
|
||||||
finally:
|
finally:
|
||||||
if not req.session and driver is not None:
|
if not req.session and driver is not None:
|
||||||
driver.quit()
|
driver.quit()
|
||||||
@@ -299,10 +299,22 @@ def _evil_logic(req: V1RequestBase, driver: WebDriver, method: str) -> Challenge
|
|||||||
_post_request(req, driver)
|
_post_request(req, driver)
|
||||||
else:
|
else:
|
||||||
driver.get(req.url)
|
driver.get(req.url)
|
||||||
if utils.get_config_log_html():
|
|
||||||
logging.debug(f"Response HTML:\n{driver.page_source}")
|
# set cookies if required
|
||||||
|
if req.cookies is not None and len(req.cookies) > 0:
|
||||||
|
logging.debug(f'Setting cookies...')
|
||||||
|
for cookie in req.cookies:
|
||||||
|
driver.delete_cookie(cookie['name'])
|
||||||
|
driver.add_cookie(cookie)
|
||||||
|
# reload the page
|
||||||
|
if method == 'POST':
|
||||||
|
_post_request(req, driver)
|
||||||
|
else:
|
||||||
|
driver.get(req.url)
|
||||||
|
|
||||||
# wait for the page
|
# wait for the page
|
||||||
|
if utils.get_config_log_html():
|
||||||
|
logging.debug(f"Response HTML:\n{driver.page_source}")
|
||||||
html_element = driver.find_element(By.TAG_NAME, "html")
|
html_element = driver.find_element(By.TAG_NAME, "html")
|
||||||
page_title = driver.title
|
page_title = driver.title
|
||||||
|
|
||||||
|
|||||||
@@ -25,7 +25,8 @@ class SessionsStorage:
|
|||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.sessions = {}
|
self.sessions = {}
|
||||||
|
|
||||||
def create(self, session_id: Optional[str] = None, force_new: Optional[bool] = False) -> Tuple[Session, bool]:
|
def create(self, session_id: Optional[str] = None, proxy: Optional[dict] = None,
|
||||||
|
force_new: Optional[bool] = False) -> Tuple[Session, bool]:
|
||||||
"""create creates new instance of WebDriver if necessary,
|
"""create creates new instance of WebDriver if necessary,
|
||||||
assign defined (or newly generated) session_id to the instance
|
assign defined (or newly generated) session_id to the instance
|
||||||
and returns the session object. If a new session has been created
|
and returns the session object. If a new session has been created
|
||||||
@@ -44,7 +45,7 @@ class SessionsStorage:
|
|||||||
if self.exists(session_id):
|
if self.exists(session_id):
|
||||||
return self.sessions[session_id], False
|
return self.sessions[session_id], False
|
||||||
|
|
||||||
driver = utils.get_webdriver()
|
driver = utils.get_webdriver(proxy)
|
||||||
created_at = datetime.now()
|
created_at = datetime.now()
|
||||||
session = Session(session_id, driver, created_at)
|
session = Session(session_id, driver, created_at)
|
||||||
|
|
||||||
|
|||||||
145
src/tests.py
145
src/tests.py
@@ -1,4 +1,5 @@
|
|||||||
import unittest
|
import unittest
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
from webtest import TestApp
|
from webtest import TestApp
|
||||||
|
|
||||||
@@ -7,7 +8,7 @@ import flaresolverr
|
|||||||
import utils
|
import utils
|
||||||
|
|
||||||
|
|
||||||
def _find_obj_by_key(key: str, value: str, _list: list) -> dict | None:
|
def _find_obj_by_key(key: str, value: str, _list: list) -> Optional[dict]:
|
||||||
for obj in _list:
|
for obj in _list:
|
||||||
if obj[key] == value:
|
if obj[key] == value:
|
||||||
return obj
|
return obj
|
||||||
@@ -28,6 +29,8 @@ class TestFlareSolverr(unittest.TestCase):
|
|||||||
cloudflare_blocked_url = "https://cpasbiens3.fr/index.php?do=search&subaction=search"
|
cloudflare_blocked_url = "https://cpasbiens3.fr/index.php?do=search&subaction=search"
|
||||||
|
|
||||||
app = TestApp(flaresolverr.app)
|
app = TestApp(flaresolverr.app)
|
||||||
|
# wait until the server is ready
|
||||||
|
app.get('/')
|
||||||
|
|
||||||
def test_wrong_endpoint(self):
|
def test_wrong_endpoint(self):
|
||||||
res = self.app.get('/wrong', status=404)
|
res = self.app.get('/wrong', status=404)
|
||||||
@@ -236,7 +239,45 @@ class TestFlareSolverr(unittest.TestCase):
|
|||||||
self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
|
self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
|
||||||
self.assertEqual(utils.get_flaresolverr_version(), body.version)
|
self.assertEqual(utils.get_flaresolverr_version(), body.version)
|
||||||
|
|
||||||
# todo: test Cmd 'request.get' should return OK with 'cookies' param
|
def test_v1_endpoint_request_get_cookies_param(self):
|
||||||
|
res = self.app.post_json('/v1', {
|
||||||
|
"cmd": "request.get",
|
||||||
|
"url": self.google_url,
|
||||||
|
"cookies": [
|
||||||
|
{
|
||||||
|
"name": "testcookie1",
|
||||||
|
"value": "testvalue1"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "testcookie2",
|
||||||
|
"value": "testvalue2"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
})
|
||||||
|
self.assertEqual(res.status_code, 200)
|
||||||
|
|
||||||
|
body = V1ResponseBase(res.json)
|
||||||
|
self.assertEqual(STATUS_OK, body.status)
|
||||||
|
self.assertEqual("Challenge not detected!", body.message)
|
||||||
|
self.assertGreater(body.startTimestamp, 10000)
|
||||||
|
self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
|
||||||
|
self.assertEqual(utils.get_flaresolverr_version(), body.version)
|
||||||
|
|
||||||
|
solution = body.solution
|
||||||
|
self.assertIn(self.google_url, solution.url)
|
||||||
|
self.assertEqual(solution.status, 200)
|
||||||
|
self.assertIs(len(solution.headers), 0)
|
||||||
|
self.assertIn("<title>Google</title>", solution.response)
|
||||||
|
self.assertGreater(len(solution.cookies), 1)
|
||||||
|
self.assertIn("Chrome/", solution.userAgent)
|
||||||
|
|
||||||
|
user_cookie1 = _find_obj_by_key("name", "testcookie1", solution.cookies)
|
||||||
|
self.assertIsNotNone(user_cookie1, "User cookie 1 not found")
|
||||||
|
self.assertEqual("testvalue1", user_cookie1["value"])
|
||||||
|
|
||||||
|
user_cookie2 = _find_obj_by_key("name", "testcookie2", solution.cookies)
|
||||||
|
self.assertIsNotNone(user_cookie2, "User cookie 2 not found")
|
||||||
|
self.assertEqual("testvalue2", user_cookie2["value"])
|
||||||
|
|
||||||
def test_v1_endpoint_request_get_returnOnlyCookies_param(self):
|
def test_v1_endpoint_request_get_returnOnlyCookies_param(self):
|
||||||
res = self.app.post_json('/v1', {
|
res = self.app.post_json('/v1', {
|
||||||
@@ -261,10 +302,88 @@ class TestFlareSolverr(unittest.TestCase):
|
|||||||
self.assertGreater(len(solution.cookies), 0)
|
self.assertGreater(len(solution.cookies), 0)
|
||||||
self.assertIn("Chrome/", solution.userAgent)
|
self.assertIn("Chrome/", solution.userAgent)
|
||||||
|
|
||||||
# todo: test Cmd 'request.get' should return OK with HTTP 'proxy' param
|
def test_v1_endpoint_request_get_proxy_http_param(self):
|
||||||
# todo: test Cmd 'request.get' should return OK with HTTP 'proxy' param with credentials
|
"""
|
||||||
# todo: test Cmd 'request.get' should return OK with SOCKSv5 'proxy' param
|
To configure TinyProxy in local:
|
||||||
# todo: test Cmd 'request.get' should fail with wrong 'proxy' param
|
* sudo vim /etc/tinyproxy/tinyproxy.conf
|
||||||
|
* edit => LogFile "/tmp/tinyproxy.log"
|
||||||
|
* edit => Syslog Off
|
||||||
|
* sudo tinyproxy -d
|
||||||
|
* sudo tail -f /tmp/tinyproxy.log
|
||||||
|
"""
|
||||||
|
res = self.app.post_json('/v1', {
|
||||||
|
"cmd": "request.get",
|
||||||
|
"url": self.google_url,
|
||||||
|
"proxy": {
|
||||||
|
"url": self.proxy_url
|
||||||
|
}
|
||||||
|
})
|
||||||
|
self.assertEqual(res.status_code, 200)
|
||||||
|
|
||||||
|
body = V1ResponseBase(res.json)
|
||||||
|
self.assertEqual(STATUS_OK, body.status)
|
||||||
|
self.assertEqual("Challenge not detected!", body.message)
|
||||||
|
self.assertGreater(body.startTimestamp, 10000)
|
||||||
|
self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
|
||||||
|
self.assertEqual(utils.get_flaresolverr_version(), body.version)
|
||||||
|
|
||||||
|
solution = body.solution
|
||||||
|
self.assertIn(self.google_url, solution.url)
|
||||||
|
self.assertEqual(solution.status, 200)
|
||||||
|
self.assertIs(len(solution.headers), 0)
|
||||||
|
self.assertIn("<title>Google</title>", solution.response)
|
||||||
|
self.assertGreater(len(solution.cookies), 0)
|
||||||
|
self.assertIn("Chrome/", solution.userAgent)
|
||||||
|
|
||||||
|
def test_v1_endpoint_request_get_proxy_socks_param(self):
|
||||||
|
"""
|
||||||
|
To configure Dante in local:
|
||||||
|
* https://linuxhint.com/set-up-a-socks5-proxy-on-ubuntu-with-dante/
|
||||||
|
* sudo vim /etc/sockd.conf
|
||||||
|
* sudo systemctl restart sockd.service
|
||||||
|
* curl --socks5 socks5://127.0.0.1:1080 https://www.google.com
|
||||||
|
"""
|
||||||
|
res = self.app.post_json('/v1', {
|
||||||
|
"cmd": "request.get",
|
||||||
|
"url": self.google_url,
|
||||||
|
"proxy": {
|
||||||
|
"url": self.proxy_socks_url
|
||||||
|
}
|
||||||
|
})
|
||||||
|
self.assertEqual(res.status_code, 200)
|
||||||
|
|
||||||
|
body = V1ResponseBase(res.json)
|
||||||
|
self.assertEqual(STATUS_OK, body.status)
|
||||||
|
self.assertEqual("Challenge not detected!", body.message)
|
||||||
|
self.assertGreater(body.startTimestamp, 10000)
|
||||||
|
self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
|
||||||
|
self.assertEqual(utils.get_flaresolverr_version(), body.version)
|
||||||
|
|
||||||
|
solution = body.solution
|
||||||
|
self.assertIn(self.google_url, solution.url)
|
||||||
|
self.assertEqual(solution.status, 200)
|
||||||
|
self.assertIs(len(solution.headers), 0)
|
||||||
|
self.assertIn("<title>Google</title>", solution.response)
|
||||||
|
self.assertGreater(len(solution.cookies), 0)
|
||||||
|
self.assertIn("Chrome/", solution.userAgent)
|
||||||
|
|
||||||
|
def test_v1_endpoint_request_get_proxy_wrong_param(self):
|
||||||
|
res = self.app.post_json('/v1', {
|
||||||
|
"cmd": "request.get",
|
||||||
|
"url": self.google_url,
|
||||||
|
"proxy": {
|
||||||
|
"url": "http://127.0.0.1:43210"
|
||||||
|
}
|
||||||
|
}, status=500)
|
||||||
|
self.assertEqual(res.status_code, 500)
|
||||||
|
|
||||||
|
body = V1ResponseBase(res.json)
|
||||||
|
self.assertEqual(STATUS_ERROR, body.status)
|
||||||
|
self.assertIn("Error: Error solving the challenge. Message: unknown error: net::ERR_PROXY_CONNECTION_FAILED",
|
||||||
|
body.message)
|
||||||
|
self.assertGreater(body.startTimestamp, 10000)
|
||||||
|
self.assertGreaterEqual(body.endTimestamp, body.startTimestamp)
|
||||||
|
self.assertEqual(utils.get_flaresolverr_version(), body.version)
|
||||||
|
|
||||||
def test_v1_endpoint_request_get_fail_timeout(self):
|
def test_v1_endpoint_request_get_fail_timeout(self):
|
||||||
res = self.app.post_json('/v1', {
|
res = self.app.post_json('/v1', {
|
||||||
@@ -401,6 +520,20 @@ class TestFlareSolverr(unittest.TestCase):
|
|||||||
self.assertEqual("Session created successfully.", body.message)
|
self.assertEqual("Session created successfully.", body.message)
|
||||||
self.assertEqual(body.session, "test_create_session")
|
self.assertEqual(body.session, "test_create_session")
|
||||||
|
|
||||||
|
def test_v1_endpoint_sessions_create_with_proxy(self):
|
||||||
|
res = self.app.post_json('/v1', {
|
||||||
|
"cmd": "sessions.create",
|
||||||
|
"proxy": {
|
||||||
|
"url": self.proxy_url
|
||||||
|
}
|
||||||
|
})
|
||||||
|
self.assertEqual(res.status_code, 200)
|
||||||
|
|
||||||
|
body = V1ResponseBase(res.json)
|
||||||
|
self.assertEqual(STATUS_OK, body.status)
|
||||||
|
self.assertEqual("Session created successfully.", body.message)
|
||||||
|
self.assertIsNotNone(body.session)
|
||||||
|
|
||||||
def test_v1_endpoint_sessions_list(self):
|
def test_v1_endpoint_sessions_list(self):
|
||||||
self.app.post_json('/v1', {
|
self.app.post_json('/v1', {
|
||||||
"cmd": "sessions.create",
|
"cmd": "sessions.create",
|
||||||
|
|||||||
@@ -39,6 +39,8 @@ def asset_cloudflare_solution(self, res, site_url, site_text):
|
|||||||
|
|
||||||
class TestFlareSolverr(unittest.TestCase):
|
class TestFlareSolverr(unittest.TestCase):
|
||||||
app = TestApp(flaresolverr.app)
|
app = TestApp(flaresolverr.app)
|
||||||
|
# wait until the server is ready
|
||||||
|
app.get('/')
|
||||||
|
|
||||||
def test_v1_endpoint_request_get_cloudflare(self):
|
def test_v1_endpoint_request_get_cloudflare(self):
|
||||||
sites_get = [
|
sites_get = [
|
||||||
|
|||||||
@@ -372,9 +372,10 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
|
|||||||
options.arguments.extend(["--no-sandbox", "--test-type"])
|
options.arguments.extend(["--no-sandbox", "--test-type"])
|
||||||
|
|
||||||
if headless or options.headless:
|
if headless or options.headless:
|
||||||
if self.patcher.version_main < 108:
|
v_main = int(self.patcher.version_main) if self.patcher.version_main else 108
|
||||||
|
if v_main < 108:
|
||||||
options.add_argument("--headless=chrome")
|
options.add_argument("--headless=chrome")
|
||||||
elif self.patcher.version_main >= 108:
|
elif v_main >= 108:
|
||||||
options.add_argument("--headless=new")
|
options.add_argument("--headless=new")
|
||||||
|
|
||||||
options.add_argument("--window-size=1920,1080")
|
options.add_argument("--window-size=1920,1080")
|
||||||
@@ -728,6 +729,15 @@ class Chrome(selenium.webdriver.chrome.webdriver.WebDriver):
|
|||||||
logger.debug("gracefully closed browser")
|
logger.debug("gracefully closed browser")
|
||||||
except Exception as e: # noqa
|
except Exception as e: # noqa
|
||||||
logger.debug(e, exc_info=True)
|
logger.debug(e, exc_info=True)
|
||||||
|
# Force kill Chrome process in Windows
|
||||||
|
# https://github.com/FlareSolverr/FlareSolverr/issues/772
|
||||||
|
if os.name == 'nt':
|
||||||
|
try:
|
||||||
|
subprocess.call(['taskkill', '/f', '/pid', str(self.browser_pid)],
|
||||||
|
stdout=subprocess.DEVNULL,
|
||||||
|
stderr=subprocess.DEVNULL)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
if (
|
if (
|
||||||
hasattr(self, "keep_user_data_dir")
|
hasattr(self, "keep_user_data_dir")
|
||||||
and hasattr(self, "user_data_dir")
|
and hasattr(self, "user_data_dir")
|
||||||
|
|||||||
30
src/utils.py
30
src/utils.py
@@ -36,7 +36,7 @@ def get_flaresolverr_version() -> str:
|
|||||||
return FLARESOLVERR_VERSION
|
return FLARESOLVERR_VERSION
|
||||||
|
|
||||||
|
|
||||||
def get_webdriver() -> WebDriver:
|
def get_webdriver(proxy: dict = None) -> WebDriver:
|
||||||
global PATCHED_DRIVER_PATH
|
global PATCHED_DRIVER_PATH
|
||||||
logging.debug('Launching web browser...')
|
logging.debug('Launching web browser...')
|
||||||
|
|
||||||
@@ -54,6 +54,20 @@ def get_webdriver() -> WebDriver:
|
|||||||
options.add_argument('--disable-software-rasterizer')
|
options.add_argument('--disable-software-rasterizer')
|
||||||
options.add_argument('--ignore-certificate-errors')
|
options.add_argument('--ignore-certificate-errors')
|
||||||
options.add_argument('--ignore-ssl-errors')
|
options.add_argument('--ignore-ssl-errors')
|
||||||
|
# fix GL errors in ASUSTOR NAS
|
||||||
|
# https://github.com/FlareSolverr/FlareSolverr/issues/782
|
||||||
|
# https://github.com/microsoft/vscode/issues/127800#issuecomment-873342069
|
||||||
|
# https://peter.sh/experiments/chromium-command-line-switches/#use-gl
|
||||||
|
options.add_argument('--use-gl=swiftshader')
|
||||||
|
# workaround for updated 'verify your are human' check
|
||||||
|
# https://github.com/FlareSolverr/FlareSolverr/issues/811
|
||||||
|
options.add_argument('--auto-open-devtools-for-tabs')
|
||||||
|
options.add_argument('--headless=true')
|
||||||
|
|
||||||
|
if proxy and 'url' in proxy:
|
||||||
|
proxy_url = proxy['url']
|
||||||
|
logging.debug("Using webdriver proxy: %s", proxy_url)
|
||||||
|
options.add_argument('--proxy-server=%s' % proxy_url)
|
||||||
|
|
||||||
# note: headless mode is detected (options.headless = True)
|
# note: headless mode is detected (options.headless = True)
|
||||||
# we launch the browser in head-full mode with the window hidden
|
# we launch the browser in head-full mode with the window hidden
|
||||||
@@ -89,6 +103,13 @@ def get_webdriver() -> WebDriver:
|
|||||||
PATCHED_DRIVER_PATH = os.path.join(driver.patcher.data_path, driver.patcher.exe_name)
|
PATCHED_DRIVER_PATH = os.path.join(driver.patcher.data_path, driver.patcher.exe_name)
|
||||||
shutil.copy(driver.patcher.executable_path, PATCHED_DRIVER_PATH)
|
shutil.copy(driver.patcher.executable_path, PATCHED_DRIVER_PATH)
|
||||||
|
|
||||||
|
# workaround for updated 'verify your are human' check
|
||||||
|
# https://github.com/FlareSolverr/FlareSolverr/issues/811
|
||||||
|
driver.execute_script('''window.open("","_blank");''')
|
||||||
|
driver.switch_to.window(window_name=driver.window_handles[0])
|
||||||
|
driver.close()
|
||||||
|
driver.switch_to.window(window_name=driver.window_handles[0])
|
||||||
|
|
||||||
# selenium vanilla
|
# selenium vanilla
|
||||||
# options = webdriver.ChromeOptions()
|
# options = webdriver.ChromeOptions()
|
||||||
# options.add_argument('--no-sandbox')
|
# options.add_argument('--no-sandbox')
|
||||||
@@ -106,12 +127,15 @@ def get_chrome_exe_path() -> str:
|
|||||||
return CHROME_EXE_PATH
|
return CHROME_EXE_PATH
|
||||||
# linux pyinstaller bundle
|
# linux pyinstaller bundle
|
||||||
chrome_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'chrome', "chrome")
|
chrome_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'chrome', "chrome")
|
||||||
if os.path.exists(chrome_path) and os.access(chrome_path, os.X_OK):
|
if os.path.exists(chrome_path):
|
||||||
|
if not os.access(chrome_path, os.X_OK):
|
||||||
|
raise Exception(f'Chrome binary "{chrome_path}" is not executable. '
|
||||||
|
f'Please, extract the archive with "tar xzf <file.tar.gz>".')
|
||||||
CHROME_EXE_PATH = chrome_path
|
CHROME_EXE_PATH = chrome_path
|
||||||
return CHROME_EXE_PATH
|
return CHROME_EXE_PATH
|
||||||
# windows pyinstaller bundle
|
# windows pyinstaller bundle
|
||||||
chrome_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'chrome', "chrome.exe")
|
chrome_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'chrome', "chrome.exe")
|
||||||
if os.path.exists(chrome_path) and os.access(chrome_path, os.X_OK):
|
if os.path.exists(chrome_path):
|
||||||
CHROME_EXE_PATH = chrome_path
|
CHROME_EXE_PATH = chrome_path
|
||||||
return CHROME_EXE_PATH
|
return CHROME_EXE_PATH
|
||||||
# system
|
# system
|
||||||
|
|||||||
Reference in New Issue
Block a user