Compare commits

..

122 Commits

Author SHA1 Message Date
ngosang
93d8350097 Bump version 2.2.3 2022-04-16 22:29:36 +02:00
ngosang
d34b43e0a8 Fix 2000 ms navigation timeout 2022-04-16 21:39:50 +02:00
ngosang
2bf4dc62da Update README.md (libseccomp2 package in Debian) 2022-04-16 20:47:31 +02:00
termonio
bb0d757755 Update README.md (clarify proxy parameter) (#307)
Clarify that `request.get` will not use the provided proxy when a session is set.
2022-04-16 20:08:48 +02:00
ngosang
fc1fa601eb Update NPM dependencies 2022-04-16 19:19:08 +02:00
ngosang
9b1f8332c7 Disable Cloudflare ban detection 2022-04-16 18:32:58 +02:00
ilike2burnthing
6175fee75a Bump version 2.2.2 (#339) 2022-03-19 04:28:16 +00:00
Harold
bb4fa9cabc Fix ban detection. Resolves #330 (#336) 2022-03-19 04:24:49 +00:00
ngosang
c951ba2523 Bump version 2.2.1 2022-02-06 16:40:03 +01:00
ngosang
6c598d5360 Fix max timeout error in some pages 2022-02-06 16:35:52 +01:00
ngosang
2893f72237 Avoid crashing in NodeJS 17 due to Unhandled promise rejection 2022-02-06 13:31:30 +01:00
ngosang
cd221bbbf1 Improve proxy validation and debug traces 2022-02-06 13:07:11 +01:00
ngosang
68fb96f0d8 Remove @types/puppeteer dependency 2022-02-06 12:53:59 +01:00
ngosang
07724e598f Bump version 2.2.0 2022-01-31 00:20:44 +01:00
ngosang
56fc688517 Increase default BROWSER_TIMEOUT=40000 (40 seconds) 2022-01-30 23:24:15 +01:00
ngosang
0a438358d1 Fix Puppeter deprecation warnings 2022-01-30 23:23:06 +01:00
ngosang
0cbca1fb79 Update base Docker image Alpine 3.15 / NodeJS 16 2022-01-30 23:17:14 +01:00
ngosang
05dcae979c Build precompiled binaries with NodeJS 16 2022-01-30 23:09:28 +01:00
ngosang
fe6cfd75b8 Update Puppeter and other dependencies 2022-01-30 22:49:15 +01:00
ngosang
bb7e82e6c4 Add support for Custom CloudFlare challenge
EbookParadijs, Film-Paleis, MuziekFabriek and Puur-Hollands
2022-01-30 21:32:16 +01:00
ngosang
fdd1d245f4 Add support for DDoS-GUARD challenge 2022-01-30 20:36:38 +01:00
ngosang
bc6ac68e52 Bump version 2.1.0 2021-12-12 16:47:33 +01:00
simonfr
a9ab2569bc Add aarch64 to user agents to be replaced (#248)
Co-authored-by: Simon <simon@perols.dev>
2021-12-12 16:46:20 +01:00
ngosang
b1a6ad7688 Fix SOCKSv4 and SOCKSv5 proxy. resolves #214 #220 2021-12-12 14:29:38 +01:00
David Refoua
642d67b927 Remove redundant JSON key (postData) (#242) 2021-12-12 12:38:10 +01:00
ngosang
c4ef6a472e Make test URL configurable with TEST_URL env var. resolves #240 2021-12-12 12:35:05 +01:00
ngosang
a24b665bd1 Bypass new Cloudflare protection 2021-12-12 12:35:05 +01:00
Diego Heras
6576e1908d Update donation links 2021-12-04 23:43:30 +01:00
ngosang
8e518d7267 Bump version 2.0.2 2021-10-31 22:46:12 +01:00
ngosang
3005ba3629 Fix SOCKS5 proxy. Resolves #214 2021-10-31 22:39:32 +01:00
ngosang
176c69d1e8 Replace Firefox ERS with a newer version 2021-10-31 22:22:28 +01:00
ngosang
7a1cf7dd80 Catch startup exceptions and give some advices 2021-10-31 22:12:55 +01:00
ngosang
456dfc222e Add env var BROWSER_TIMEOUT for slow systems 2021-10-31 21:56:25 +01:00
ngosang
23fde49f2b Fix NPM warning in Docker images 2021-10-31 21:38:57 +01:00
ngosang
78daf24bc3 Bump version 2.0.1 2021-10-24 16:38:15 +02:00
ngosang
47c83ded58 Check user home dir before testing web browser installation 2021-10-24 15:52:03 +02:00
ngosang
35890cade4 Bump version 2.0.0 2021-10-20 18:37:39 +02:00
ngosang
753e8e1be8 Set puppeteer timeout half of maxTimeout param. Resolves #180 2021-10-20 18:28:30 +02:00
ngosang
a6628d0cda Add test for blocked IP 2021-10-20 18:06:25 +02:00
ngosang
a79a5f2b42 Avoid reloading the page in case of error 2021-10-20 18:06:15 +02:00
ngosang
1e463bb3e2 Improve Cloudflare detection 2021-10-20 18:05:59 +02:00
ngosang
02204a84d3 Fix version 2021-10-20 03:39:24 +02:00
ngosang
95d178b37a Fix browser preferences and proxy 2021-10-20 01:00:54 +02:00
ngosang
c4f890f9a1 Fix request.post method and clean error traces 2021-10-20 01:00:35 +02:00
ngosang
d16b982bb9 Use Firefox ESR for Docker images 2021-10-18 22:52:38 +02:00
ngosang
075b53ee24 Improve Firefox start time and code clean up 2021-10-18 21:45:21 +02:00
ngosang
356b893c18 Improve bad request management and tests 2021-10-18 19:27:21 +02:00
ngosang
a841d67745 Build native packages with Firefox 2021-10-18 11:13:49 +02:00
ngosang
2408a75a70 Update readme 2021-10-18 01:11:31 +02:00
ngosang
77a87c79fd Improve Docker image and clean TODOs 2021-10-18 00:23:28 +02:00
ngosang
cfd158462f Add proxy support 2021-10-18 00:02:30 +02:00
ngosang
ccfe21c15a Implement request.post method for Firefox 2021-10-17 22:05:15 +02:00
ngosang
a5b3e08e1f Code clean up, remove returnRawHtml, download, headers params 2021-10-17 20:43:36 +02:00
ngosang
a0e897067a Remove outdated chaptcha solvers 2021-10-17 18:25:44 +02:00
ngosang
744de4d158 Refactor the app to use Express server and Jest for tests 2021-10-17 18:00:19 +02:00
ngosang
0459f2642d Fix Cloudflare resolver for Linux ARM builds 2021-10-16 20:26:10 +02:00
ngosang
ca3f84f458 Fix Cloudflare resolver 2021-10-16 19:32:52 +02:00
ngosang
5dd563e003 Replace Chrome web browser with Firefox 2021-10-16 19:16:25 +02:00
ngosang
78c10d6b24 Remove userAgent parameter since any modification is detected by CF 2021-10-16 17:46:04 +02:00
ngosang
3de2e44bfd Update dependencies 2021-10-16 17:29:58 +02:00
ngosang
7738f7a360 Remove Puppeter steath plugin 2021-10-16 16:54:56 +02:00
ngosang
1b01caaa78 Bump version 1.2.9 2021-08-01 22:11:55 +02:00
ngosang
447c8f67a1 Improve "Execution context was destroyed" error handling 2021-08-01 22:10:53 +02:00
ngosang
9dae74bc28 Implement returnRawHtml parameter. resolves #172 resolves #165 2021-08-01 22:08:55 +02:00
ngosang
4199db5a41 Capture Docker stop signal. resolves #158 2021-08-01 21:37:45 +02:00
ngosang
2a4fae37c0 Reduce Docker image size 20 MB 2021-08-01 21:27:27 +02:00
ngosang
232ddca512 Fix page reload after challenge is solved. resolves #162 resolves #143 2021-08-01 20:34:38 +02:00
ngosang
8572fab781 Avoid loading images/css/fonts to speed up page load 2021-08-01 19:35:26 +02:00
ngosang
fdb3eae051 Improve Cloudflare IP ban detection 2021-08-01 19:32:09 +02:00
ngosang
6dd8206a10 Fix vulnerabilities 2021-08-01 19:15:24 +02:00
ngosang
c4e4d28c8d Bump version 1.2.8 2021-06-01 02:00:39 +02:00
ngosang
543ce89eb6 Improve old JS challenge waiting. Resolves #129 2021-06-01 01:59:57 +02:00
ngosang
0f30e17ef1 Bump version 1.2.7 2021-06-01 01:22:36 +02:00
ngosang
24f1b4ec6f Improvements in Cloudflare redirect detection. Resolves #140 2021-06-01 01:21:06 +02:00
ngosang
f3b30268c3 Fix installation instructions 2021-05-31 22:59:51 +02:00
ngosang
be4354c68d Bump version 1.2.6 2021-05-30 14:58:13 +02:00
ngosang
5242cf3359 Show an error in hcaptcha-solver. Resolves #132 2021-05-30 14:15:08 +02:00
ngosang
c6677f4d84 Handle new Cloudflare challenge. Resolves #135 Resolves #134 2021-05-30 13:40:17 +02:00
ngosang
805a34c9d6 Provide reference Systemd unit file. Resolves #72 2021-05-30 12:16:34 +02:00
ngosang
2f9fe05a76 Update issue template. Resolves #130 2021-05-30 11:44:28 +02:00
ngosang
8961d67a29 Regenerate package-lock.json lockfileVersion 2 2021-05-30 11:41:03 +02:00
ngosang
5da5156851 Fix EACCES: permission denied, open '/tmp/flaresolverr.txt'. Resolves #120 2021-05-30 11:38:20 +02:00
ngosang
05f8ef95d9 Configure timezone with TZ env var. Resolves #109 2021-05-30 11:28:43 +02:00
dependabot[bot]
10f8b83e83 Bump ws from 7.4.1 to 7.4.6 (#137)
Bumps [ws](https://github.com/websockets/ws) from 7.4.1 to 7.4.6.
- [Release notes](https://github.com/websockets/ws/releases)
- [Commits](https://github.com/websockets/ws/compare/7.4.1...7.4.6)

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2021-05-30 09:30:00 +02:00
Arias800
6cf948d0e1 Return the redirected URL in the response (#126)
It adds the possibility for the user to get the final url after a redirection.
2021-05-30 09:29:21 +02:00
dependabot[bot]
dcdc70273f Bump hosted-git-info from 2.8.8 to 2.8.9 (#124)
Bumps [hosted-git-info](https://github.com/npm/hosted-git-info) from 2.8.8 to 2.8.9.
- [Release notes](https://github.com/npm/hosted-git-info/releases)
- [Changelog](https://github.com/npm/hosted-git-info/blob/v2.8.9/CHANGELOG.md)
- [Commits](https://github.com/npm/hosted-git-info/compare/v2.8.8...v2.8.9)

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2021-05-30 09:28:09 +02:00
dependabot[bot]
e2dc39ee4e Bump lodash from 4.17.20 to 4.17.21 (#125)
Bumps [lodash](https://github.com/lodash/lodash) from 4.17.20 to 4.17.21.
- [Release notes](https://github.com/lodash/lodash/releases)
- [Commits](https://github.com/lodash/lodash/compare/4.17.20...4.17.21)

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2021-05-30 09:27:55 +02:00
ngosang
340638ca54 Bump version 1.2.5 2021-04-05 05:27:37 +02:00
ngosang
05abe69df6 Fix memory regression, close test browser 2021-04-05 05:26:45 +02:00
ngosang
e596906c19 Fix release-docker GitHub action 2021-04-04 22:46:48 +02:00
ngosang
8a1b0ea05c Bump version 1.2.4 2021-04-04 22:42:03 +02:00
ngosang
916fbf2c9d Include license in release zips. resolves #75 2021-04-04 22:39:02 +02:00
ngosang
a85e9c2c8c Validate Chrome is working at startup 2021-04-04 22:37:53 +02:00
ngosang
71814a86bc Speedup Docker image build 2021-04-04 22:36:53 +02:00
ngosang
757ec4358a Add health check endpoint 2021-04-04 20:33:07 +02:00
ngosang
f278c7cf8e Update issue template 2021-04-04 19:53:54 +02:00
ngosang
b4c99d8426 Minor improvements in debug traces 2021-04-04 18:42:04 +02:00
ngosang
8aa7723f45 Validate environment variables at startup. resolves #101 2021-04-04 18:02:17 +02:00
ngosang
c48d342b9c Add FlareSolverr logo. resolves #23 2021-01-10 16:19:20 +01:00
ngosang
7c361af204 Bump version 1.2.3 2021-01-10 15:40:09 +01:00
ngosang
6400449344 CI/CD: Generate release changelog from commits. resolves #34 2021-01-10 15:39:10 +01:00
Diego Heras
69c4d9edfa Update README.md 2021-01-10 15:25:42 +01:00
ngosang
85428a32f4 Add donation links 2021-01-10 15:13:17 +01:00
ngosang
ea5e461fb4 Simplify docker-compose.yml 2021-01-10 15:08:39 +01:00
ngosang
a57510aa0d Allow to configure "none" captcha resolver 2021-01-10 15:04:18 +01:00
JoshDi
91d1f0cb4a Override docker-compose.yml variables via .env resolves #64 (#66) 2021-01-10 15:03:30 +01:00
ngosang
7376ef9bc9 Bump version 1.2.2 2021-01-09 00:57:24 +01:00
ngosang
de9c7bcf76 Add documentation for precompiled binaries installation 2021-01-09 00:54:33 +01:00
ngosang
bef9411e1c Add instructions to set environment variables in Windows 2021-01-09 00:40:38 +01:00
ngosang
27ad58b2c6 Build Windows and Linux binaries. resolves #18 2021-01-09 00:30:51 +01:00
Diego Heras
d038944089 Add release badge in the readme 2021-01-09 00:22:42 +01:00
ngosang
a8bc6f5468 CI/CD: Generate release changelog from commits. resolves #34 2021-01-09 00:13:05 +01:00
ngosang
39fdde9a74 Add a notice about captcha solvers 2021-01-08 18:33:44 +01:00
ngosang
8234cdb516 Add Chrome flag --disable-dev-shm-usage to fix crashes. resolves #45 2021-01-08 16:20:35 +01:00
ngosang
66fe775d27 Fix Docker CLI documentation 2021-01-08 16:18:55 +01:00
ngosang
ade05bb7a8 Add traces with captcha solver service. resolves #39 2021-01-08 16:01:28 +01:00
ngosang
5710c08581 Improve logic to detect Cloudflare captcha. resolves #48 2021-01-08 15:45:40 +01:00
ngosang
f1e829fd3a Move Cloudflare provider logic to his own class 2021-01-08 13:01:52 +01:00
ngosang
dfc4383b50 Simplify and document the "return only cookies" parameter 2021-01-08 12:54:04 +01:00
ngosang
d140e9369d Show message when debug log is enabled 2021-01-07 13:49:04 +01:00
Diego Heras
6677329842 Update readme to add more clarifications. resolves #53 (#60) 2021-01-07 12:55:47 +01:00
ilike2burnthing
0f40054a73 issue_template: typo fix (#52) 2020-12-31 14:07:24 +01:00
32 changed files with 14661 additions and 2956 deletions

View File

@@ -1,6 +1,7 @@
node_modules
npm-debug.log
Dockerfile
.dockerignore
.git
.gitignore
.git/
.github/
.idea/
bin/
dist/
node_modules/
resources/

View File

@@ -1,31 +1,26 @@
**Please use the search bar** at the top of the page and make sure you are not creating an already submitted issue.
Check closed issues as well, because your issue may have already been fixed.
### Instruction on how to enable debug and html trace
### How to enable debug and html traces
[Follow the instructions from this wiki page](https://github.com/FlareSolverr/FlareSolverr/wiki/How-to-enable-debug-and-html-trace)
### Environment
**FlareSolverr Version**:
**Docker**: [yes/no]
**OS**:
**Last Working FlareSolverr Version**:
**Are you using a proxy or VPN?** [yes/no]
**Using Captcha Solver:** [yse/no]
**If using captcha solver, which one:**
* **FlareSolverr version**:
* **Last working FlareSolverr version**:
* **Operating system**:
* **Are you using Docker**: [yes/no]
* **FlareSolverr User-Agent (see log traces or / endpoint)**:
* **Are you using a proxy or VPN?** [yes/no]
* **Are you using Captcha Solver:** [yes/no]
* **If using captcha solver, which one:**
* **URL to test this issue:**
### Description
[List steps to reproduce the error and details on what happens and what you expected to happen]
### Logged Error Messages
[Place any relevant error messages you noticed from the logs here.]

View File

@@ -1,4 +1,4 @@
name: publish
name: release-docker
on:
push:
@@ -24,7 +24,7 @@ jobs:
tag-sha: false
-
name: Set up QEMU
uses: docker/setup-qemu-action@v1
uses: docker/setup-qemu-action@v1.0.1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1

View File

@@ -7,19 +7,35 @@ on:
jobs:
build:
name: Create Release
name: Create release
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v2
with:
fetch-depth: 0 # get all commits, branches and tags (required for the changelog)
- name: Build Changelog
id: github_release
uses: mikepenz/release-changelog-builder-action@main
env:
GITHUB_TOKEN: ${{ secrets.GH_PAT }}
- name: Setup Node
uses: actions/setup-node@v2
with:
node-version: '14'
- name: Create Release
- name: Build artifacts
run: |
npm install
npm run build
npm run package
- name: Build changelog
id: github_changelog
run: |
changelog=$(git log $(git tag | tail -2 | head -1)..HEAD --no-merges --oneline)
changelog="${changelog//'%'/'%25'}"
changelog="${changelog//$'\n'/'%0A'}"
changelog="${changelog//$'\r'/'%0D'}"
echo "##[set-output name=changelog;]${changelog}"
- name: Create release
id: create_release
uses: actions/create-release@v1
env:
@@ -27,6 +43,13 @@ jobs:
with:
tag_name: ${{ github.ref }}
release_name: ${{ github.ref }}
body: ${{ steps.github_release.outputs.changelog }}
body: ${{ steps.github_changelog.outputs.changelog }}
draft: false
prerelease: false
- name: Upload release artifacts
uses: alexellis/upload-assets@0.2.2
env:
GITHUB_TOKEN: ${{ secrets.GH_PAT }}
with:
asset_paths: '["./bin/*.zip"]'

3
.gitignore vendored
View File

@@ -121,3 +121,6 @@ dist
# Project Development
testing/
# Binaries
bin/

View File

@@ -1,13 +1,14 @@
FROM --platform=${TARGETPLATFORM:-linux/amd64} node:15.2.1-alpine3.11
FROM --platform=${TARGETPLATFORM:-linux/amd64} node:16-alpine3.15
# Print build information
ARG TARGETPLATFORM
ARG BUILDPLATFORM
RUN printf "I am running on ${BUILDPLATFORM:-linux/amd64}, building for ${TARGETPLATFORM:-linux/amd64}\n$(uname -a)\n"
# Install Chromium, dumb-init and remove all locales but en-US
RUN apk add --no-cache chromium dumb-init && \
find /usr/lib/chromium/locales -type f ! -name 'en-US.*' -delete
# Install the web browser (package firefox-esr is available too)
RUN apk update && \
apk add --no-cache firefox dumb-init && \
rm -Rf /var/cache
# Copy FlareSolverr code
USER node
@@ -16,15 +17,18 @@ WORKDIR /home/node/flaresolverr
COPY --chown=node:node package.json package-lock.json tsconfig.json ./
COPY --chown=node:node src ./src/
# Install package. Skip installing Chrome, we will use the installed package.
ENV PUPPETEER_PRODUCT=chrome \
# Install package. Skip installing the browser, we will use the installed package.
ENV PUPPETEER_PRODUCT=firefox \
PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
PUPPETEER_EXECUTABLE_PATH=/usr/bin/firefox
RUN npm install && \
npm run build && \
rm -rf src tsconfig.json && \
npm prune --production
npm prune --production && \
rm -rf /home/node/.npm
EXPOSE 8191
ENTRYPOINT ["/usr/bin/dumb-init", "--"]
CMD ["npm", "start"]
CMD ["node", "./dist/server.js"]
# docker build -t flaresolverr:custom .
# docker run -p 8191:8191 -e LOG_LEVEL=debug flaresolverr:custom

131
README.md
View File

@@ -1,19 +1,21 @@
# FlareSolverr
[![GitHub issues](https://img.shields.io/github/issues/FlareSolverr/FlareSolverr.svg?maxAge=60&style=flat-square)](https://github.com/FlareSolverr/FlareSolverr/issues)
[![GitHub pull requests](https://img.shields.io/github/issues-pr/FlareSolverr/FlareSolverr.svg?maxAge=60&style=flat-square)](https://github.com/FlareSolverr/FlareSolverr/pulls)
[![Docker Pulls](https://img.shields.io/docker/pulls/flaresolverr/flaresolverr.svg?maxAge=60&style=flat-square)](https://hub.docker.com/r/flaresolverr/flaresolverr/)
[![Latest release](https://img.shields.io/github/v/release/FlareSolverr/FlareSolverr)](https://github.com/FlareSolverr/FlareSolverr/releases)
[![Docker Pulls](https://img.shields.io/docker/pulls/flaresolverr/flaresolverr)](https://hub.docker.com/r/flaresolverr/flaresolverr/)
[![GitHub issues](https://img.shields.io/github/issues/FlareSolverr/FlareSolverr)](https://github.com/FlareSolverr/FlareSolverr/issues)
[![GitHub pull requests](https://img.shields.io/github/issues-pr/FlareSolverr/FlareSolverr)](https://github.com/FlareSolverr/FlareSolverr/pulls)
[![Donate PayPal](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=X5NJLLX5GLTV6&source=url)
[![Donate Bitcoin](https://en.cryptobadges.io/badge/micro/13Hcv77AdnFWEUZ9qUpoPBttQsUT7q9TTh)](https://en.cryptobadges.io/donate/13Hcv77AdnFWEUZ9qUpoPBttQsUT7q9TTh)
[![Donate Ethereum](https://en.cryptobadges.io/badge/micro/0x0D1549BbB00926BF3D92c1A8A58695e982f1BE2E)](https://en.cryptobadges.io/donate/0x0D1549BbB00926BF3D92c1A8A58695e982f1BE2E)
FlareSolverr is a proxy server to bypass Cloudflare protection
:warning: This project is in beta state. Some things may not work and the API can change at any time.
FlareSolverr is a proxy server to bypass Cloudflare protection.
## How it works
FlareSolverr starts a proxy server and it waits for user requests in an idle state using few resources.
When some request arrives, it uses [puppeteer](https://github.com/puppeteer/puppeteer) with the
[stealth plugin](https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin-stealth)
to create a headless browser (Chrome). It opens the URL with user parameters and waits until the Cloudflare challenge
to create a headless browser (Firefox). It opens the URL with user parameters and waits until the Cloudflare challenge
is solved (or timeout). The HTML code and the cookies are sent back to the user, and those cookies can be used to
bypass Cloudflare using other HTTP clients.
@@ -48,19 +50,37 @@ If you prefer the `docker cli` execute the following command.
```bash
docker run -d \
--name=flaresolverr \
-p 8191:8191 \
-e LOG_LEVEL=info \
--restart unless-stopped \
ghcr.io/flaresolverr/flaresolverr:latest
```
If your host OS is Debian, make sure `libseccomp2` version is 2.5.x. You can check the version with `sudo apt-cache policy libseccomp2`
and update the package with `sudo apt install libseccomp2=2.5.1-1~bpo10+1` or `sudo apt install libseccomp2=2.5.1-1+deb11u1`.
Remember to restart the Docker daemon and the container after the update.
### Precompiled binaries
This is the recommended way for Windows users.
* Download the [FlareSolverr zip](https://github.com/FlareSolverr/FlareSolverr/releases) from the release's assets. It is available for Windows and Linux.
* Extract the zip file. FlareSolverr executable and firefox folder must be in the same directory.
* Execute FlareSolverr binary. In the environment variables section you can find how to change the configuration.
### From source code
This is the recommended way for Windows / MacOS users and for developers.
* Install [NodeJS](https://nodejs.org/)
* Clone this repository and open a shell in that path
* Run `npm install` command to install FlareSolverr dependencies
* Run `npm run build` command to compile TypeScript code
* Run `npm start` command to start FlareSolverr
This is the recommended way for macOS users and for developers.
* Install [NodeJS](https://nodejs.org/) 16.
* Clone this repository and open a shell in that path.
* Run `export PUPPETEER_PRODUCT=firefox` (Linux/macOS) or `set PUPPETEER_PRODUCT=firefox` (Windows).
* Run `npm install` command to install FlareSolverr dependencies.
* Run `npm start` command to compile TypeScript code and start FlareSolverr.
If you get errors related to firefox not installed try running `node node_modules/puppeteer/install.js` to install Firefox.
### Systemd service
We provide an example Systemd unit file `flaresolverr.service` as reference. You have to modify the file to suit your needs: paths, user and environment variables.
## Usage
@@ -71,11 +91,7 @@ curl -L -X POST 'http://localhost:8191/v1' \
--data-raw '{
"cmd": "request.get",
"url":"http://www.google.com/",
"userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleW...",
"maxTimeout": 60000,
"headers": {
"X-Test": "Testing 123..."
}
"maxTimeout": 60000
}'
```
@@ -92,7 +108,7 @@ This also speeds up the requests since it won't have to launch a new browser ins
Parameter | Notes
|--|--|
session | Optional. The session ID that you want to be assigned to the instance. If isn't set a random UUID will be assigned.
userAgent | Optional. Will be used by the headless browser.
proxy | Optional, default disabled. Eg: `"proxy": {"url": "http://127.0.0.1:8888"}`. You must include the proxy schema in the URL: `http://`, `socks4://` or `socks5://`. Authorization (username/password) is not supported.
#### + `sessions.list`
@@ -127,9 +143,12 @@ Parameter | Notes
|--|--|
url | Mandatory
session | Optional. Will send the request from and existing browser instance. If one is not sent it will create a temporary instance that will be destroyed immediately after the request is completed.
headers | Optional. To specify user headers.
maxTimeout | Optional. Max timeout to solve the challenge
cookies | Optional. Will be used by the headless browser. Follow [this](https://github.com/puppeteer/puppeteer/blob/v3.3.0/docs/api.md#pagesetcookiecookies) format
maxTimeout | Optional, default value 60000. Max timeout to solve the challenge in milliseconds.
cookies | Optional. Will be used by the headless browser. Follow [this](https://github.com/puppeteer/puppeteer/blob/v3.3.0/docs/api.md#pagesetcookiecookies) format.
returnOnlyCookies | Optional, default false. Only returns the cookies. Response data, headers and other parts of the response are removed.
proxy | Optional, default disabled. Eg: `"proxy": {"url": "http://127.0.0.1:8888"}`. You must include the proxy schema in the URL: `http://`, `socks4://` or `socks5://`. Authorization (username/password) is not supported. (When the `session` parameter is set, the proxy is ignored; a session specific proxy can be set in `sessions.create`.)
:warning: If you want to use Cloudflare clearance cookie in your scripts, make sure you use the FlareSolverr User-Agent too. If they don't match you will see the challenge.
Example response from running the `curl` above:
@@ -196,66 +215,38 @@ This is the same as `request.get` but it takes one more param:
Parameter | Notes
|--|--|
postData | Must be a string. If you want to POST a form, don't forget to set the `Content-Type` header to `application/x-www-form-urlencoded` or the server might not understand your request.
### Download small files
If you need to access an image/pdf or small file, you should pass the `download` parameter to `request.get` setting it
to `true`. Rather than access the html and return text it will return the buffer **base64** encoded which you will be
able to decode and save the image/pdf.
This method isn't recommended for videos or anything larger. As that should be streamed back to the client and at the
moment there is nothing setup to do so. If this is something you need feel free to create an issue and/or submit a PR.
postData | Must be a string with `application/x-www-form-urlencoded`. Eg: `a=b&c=d`
## Environment variables
To set the environment vars in Linux run `export LOG_LEVEL=debug` and then start FlareSolverr in the same shell.
Name | Default | Notes
|--|--|--|
LOG_LEVEL | info | Used to change the verbosity of the logging.
LOG_HTML | false | Used for debugging. If `true` all HTML that passes through the proxy will be logged to the console in `debug` level.
PORT | 8191 | Change this if you already have a process running on port `8191`.
HOST | 0.0.0.0 | This shouldn't need to be messed with but if you insist, it's here!
CAPTCHA_SOLVER | None | This is used to select which captcha solving method it used when a captcha is encountered.
HEADLESS | true | This is used to debug the browser by not running it in headless mode.
LOG_LEVEL | info | Verbosity of the logging. Use `LOG_LEVEL=debug` for more information.
LOG_HTML | false | Only for debugging. If `true` all HTML that passes through the proxy will be logged to the console in `debug` level.
CAPTCHA_SOLVER | none | Captcha solving method. It is used when a captcha is encountered. See the Captcha Solvers section.
TZ | UTC | Timezone used in the logs and the web browser. Example: `TZ=Europe/London`.
HEADLESS | true | Only for debugging. To run the web browser in headless mode or visible.
BROWSER_TIMEOUT | 40000 | If you are experiencing errors/timeouts because your system is slow, you can try to increase this value. Remember to increase the `maxTimeout` parameter too.
TEST_URL | https://www.google.com | FlareSolverr makes a request on start to make sure the web browser is working. You can change that URL if it is blocked in your country.
PORT | 8191 | Listening port. You don't need to change this if you are running on Docker.
HOST | 0.0.0.0 | Listening interface. You don't need to change this if you are running on Docker.
Environment variables are set differently depending on the operating system. Some examples:
* Docker: Take a look at the Docker section in this document. Environment variables can be set in the `docker-compose.yml` file or in the Docker CLI command.
* Linux: Run `export LOG_LEVEL=debug` and then start FlareSolverr in the same shell.
* Windows: Open `cmd.exe`, run `set LOG_LEVEL=debug` and then start FlareSolverr in the same shell.
## Captcha Solvers
Sometimes CF not only gives mathematical computations and browser tests, sometimes they also require the user to solve
a captcha. If this is the case, FlareSolverr will return the captcha page. But that's not very helpful to you is it?
:warning: At this time none of the captcha solvers work. You can check the status in the open issues. Any help is welcome.
Sometimes CloudFlare not only gives mathematical computations and browser tests, sometimes they also require the user to
solve a captcha.
If this is the case, FlareSolverr will return the error `Captcha detected but no automatic solver is configured.`
FlareSolverr can be customized to solve the captchas automatically by setting the environment variable `CAPTCHA_SOLVER`
to the file name of one of the adapters inside the [/captcha](src/captcha) directory.
### hcaptcha-solver
This method makes use of the [hcaptcha-solver](https://github.com/JimmyLaurent/hcaptcha-solver) project which attempts
to solve hCaptcha by randomly selecting images.
To use this solver you must first install it and then set it as the `CAPTCHA_SOLVER`.
```bash
npm i hcaptcha-solver
CAPTCHA_SOLVER=hcaptcha-solver
```
### CaptchaHarvester
This method makes use of the [CaptchaHarvester](https://github.com/NoahCardoza/CaptchaHarvester) project which allows
users to collect thier own tokens from ReCaptcha V2/V3 and hCaptcha for free.
To use this method you must set these ENV variables:
```bash
CAPTCHA_SOLVER=harvester
HARVESTER_ENDPOINT=https://127.0.0.1:5000/token
```
**Note**: above I set `HARVESTER_ENDPOINT` to the default configuration of the captcha harvester's server, but that
could change if you customize the command line flags. Simply put, `HARVESTER_ENDPOINT` should be set to the URI of the
route that returns a token in plain text when called.
## Related projects
* C# implementation => https://github.com/FlareSolverr/FlareSolverrSharp

111
build-binaries.js Normal file
View File

@@ -0,0 +1,111 @@
const fs = require('fs')
const path = require('path')
const { execSync } = require('child_process')
const archiver = require('archiver')
const https = require('https')
const puppeteer = require('puppeteer')
const version = 'v' + require('./package.json').version;
function getFirefoxNightlyVersion() {
const firefoxVersions = 'https://product-details.mozilla.org/1.0/firefox_versions.json';
return new Promise((resolve, reject) => {
let data = '';
https
.get(firefoxVersions, (r) => {
if (r.statusCode >= 400)
return reject(new Error(`Got status code ${r.statusCode}`));
r.on('data', (chunk) => {
data += chunk;
});
r.on('end', () => {
try {
const versions = JSON.parse(data);
return resolve(versions.FIREFOX_NIGHTLY);
} catch {
return reject(new Error('Firefox version not found'));
}
});
})
.on('error', reject);
});
}
(async () => {
const builds = [
{
platform: 'linux',
firefoxFolder: 'firefox',
fsExec: 'flaresolverr-linux',
fsZipExec: 'flaresolverr',
fsZipName: 'linux-x64',
fsLicenseName: 'LICENSE'
},
{
platform: 'win64',
firefoxFolder: 'firefox',
fsExec: 'flaresolverr-win.exe',
fsZipExec: 'flaresolverr.exe',
fsZipName: 'windows-x64',
fsLicenseName: 'LICENSE.txt'
}
// todo: this has to be build in macOS (hdiutil is required). changes required in sessions.ts too
// {
// platform: 'mac',
// firefoxFolder: 'firefox',
// fsExec: 'flaresolverr-macos',
// fsZipExec: 'flaresolverr',
// fsZipName: 'macos',
// fsLicenseName: 'LICENSE'
// }
]
// generate executables
console.log('Generating executables...')
if (fs.existsSync('bin')) {
fs.rmSync('bin', { recursive: true })
}
execSync('./node_modules/.bin/pkg -t node16-win-x64,node16-linux-x64 --out-path bin .')
// execSync('./node_modules/.bin/pkg -t node16-win-x64,node16-mac-x64,node16-linux-x64 --out-path bin .')
// get firefox revision
const revision = await getFirefoxNightlyVersion();
// download firefox and zip together
for (const os of builds) {
console.log('Building ' + os.fsZipName + ' artifact')
// download firefox
console.log(`Downloading firefox ${revision} for ${os.platform} ...`)
const f = puppeteer.createBrowserFetcher({
product: 'firefox',
platform: os.platform,
path: path.join(__dirname, 'bin', 'puppeteer')
})
await f.download(revision)
// compress in zip
console.log('Compressing zip file...')
const zipName = 'bin/flaresolverr-' + version + '-' + os.fsZipName + '.zip'
const output = fs.createWriteStream(zipName)
const archive = archiver('zip')
output.on('close', function () {
console.log('File ' + zipName + ' created. Size: ' + archive.pointer() + ' bytes')
})
archive.on('error', function (err) {
throw err
})
archive.pipe(output)
archive.file('LICENSE', { name: 'flaresolverr/' + os.fsLicenseName })
archive.file('bin/' + os.fsExec, { name: 'flaresolverr/' + os.fsZipExec })
archive.directory('bin/puppeteer/' + os.platform + '-' + revision + '/' + os.firefoxFolder, 'flaresolverr/firefox')
if (os.platform === 'linux') {
archive.file('flaresolverr.service', { name: 'flaresolverr/flaresolverr.service' })
}
await archive.finalize()
}
})()

View File

@@ -6,13 +6,10 @@ services:
image: ghcr.io/flaresolverr/flaresolverr:latest
container_name: flaresolverr
environment:
# Used to change the verbosity of the logging
- LOG_LEVEL=info
# Enables hcaptcha-solver => https://github.com/JimmyLaurent/hcaptcha-solver
#- CAPTCHA_SOLVER=hcaptcha-solver
# Enables CaptchaHarvester => https://github.com/NoahCardoza/CaptchaHarvester
#- CAPTCHA_SOLVER=harvester
#- HARVESTER_ENDPOINT=https://127.0.0.1:5000/token
- LOG_LEVEL=${LOG_LEVEL:-info}
- LOG_HTML=${LOG_HTML:-false}
- CAPTCHA_SOLVER=${CAPTCHA_SOLVER:-none}
- TZ=Europe/London
ports:
- 8191:8191
- "${PORT:-8191}:8191"
restart: unless-stopped

19
flaresolverr.service Normal file
View File

@@ -0,0 +1,19 @@
[Unit]
Description=FlareSolverr
After=network.target
[Service]
SyslogIdentifier=flaresolverr
Restart=always
RestartSec=5
Type=simple
User=flaresolverr
Group=flaresolverr
Environment="LOG_LEVEL=info"
Environment="CAPTCHA_SOLVER=none"
WorkingDirectory=/opt/flaresolverr
ExecStart=/opt/flaresolverr/flaresolverr
TimeoutStopSec=30
[Install]
WantedBy=multi-user.target

12
jest.config.js Normal file
View File

@@ -0,0 +1,12 @@
module.exports = {
// A list of paths to directories that Jest should use to search for files in
roots: [
"./src/"
],
// Compile Typescript
transform: {
'^.+\\.(ts|tsx)$': 'ts-jest'
},
// Default value for FlareSolverr maxTimeout is 60000
testTimeout: 70000
}

15770
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,48 +1,45 @@
{
"name": "flaresolverr",
"version": "1.2.1",
"version": "2.2.3",
"description": "Proxy server to bypass Cloudflare protection.",
"scripts": {
"start": "node ./dist/index.js",
"start": "tsc && node ./dist/server.js",
"build": "tsc",
"dev": "nodemon -e ts --exec ts-node src/index.ts"
"dev": "nodemon -e ts --exec ts-node src/server.ts",
"package": "tsc && node build-binaries.js",
"test": "jest --runInBand"
},
"author": "Diego Heras (ngosang)",
"contributors": [
{
"name": "Noah Cardoza",
"url": "https://github.com/NoahCardoza/CloudProxy.git"
}
],
"license": "MIT",
"repository": {
"type": "git",
"url": "https://github.com/ngosang/FlareSolverr"
},
"bin": {
"flaresolverr": "dist/server.js"
},
"dependencies": {
"await-timeout": "^1.1.1",
"body-parser": "^1.20.0",
"console-log-level": "^1.4.1",
"got": "^11.5.1",
"hcaptcha-solver": "^1.0.2",
"puppeteer": "^3.3.0",
"puppeteer-extra": "^3.1.15",
"puppeteer-extra-plugin-stealth": "^2.6.5",
"uuid": "^8.2.0"
"express": "^4.17.3",
"puppeteer": "^13.5.2",
"uuid": "^8.3.2"
},
"devDependencies": {
"@types/await-timeout": "^0.3.1",
"@types/node": "^14.0.23",
"@types/puppeteer": "^3.0.1",
"@types/uuid": "^8.0.0",
"eslint": "^7.5.0",
"eslint-config-airbnb-base": "^14.2.0",
"eslint-config-standard": "^14.1.1",
"eslint-plugin-import": "^2.22.0",
"eslint-plugin-node": "^11.1.0",
"eslint-plugin-promise": "^4.2.1",
"eslint-plugin-standard": "^4.0.1",
"nodemon": "^2.0.4",
"ts-node": "^8.10.2",
"typescript": "^3.9.7"
"@types/body-parser": "^1.19.1",
"@types/express": "^4.17.13",
"@types/jest": "^27.4.1",
"@types/node": "^16.11.27",
"@types/supertest": "^2.0.12",
"@types/uuid": "^8.3.1",
"archiver": "^5.3.1",
"nodemon": "^2.0.13",
"pkg": "^5.6.0",
"supertest": "^6.1.6",
"ts-jest": "^27.1.4",
"ts-node": "^10.7.0",
"typescript": "^4.6.3"
}
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 9.1 KiB

View File

@@ -0,0 +1,180 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!-- Created with Inkscape (http://www.inkscape.org/) -->
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
width="256"
height="256"
viewBox="0 0 256 256"
id="svg2"
version="1.1"
inkscape:version="0.91 r13725"
sodipodi:docname="flaresolverr_logo.svg"
inkscape:export-filename="C:\Users\Diego\Desktop\flaresolverr_logo.png"
inkscape:export-xdpi="90"
inkscape:export-ydpi="90">
<defs
id="defs4" />
<sodipodi:namedview
id="base"
pagecolor="#ffffff"
bordercolor="#666666"
borderopacity="1.0"
inkscape:pageopacity="0.0"
inkscape:pageshadow="2"
inkscape:zoom="0.64"
inkscape:cx="-88.263072"
inkscape:cy="-93.571587"
inkscape:document-units="px"
inkscape:current-layer="layer1"
showgrid="false"
inkscape:window-width="2560"
inkscape:window-height="1377"
inkscape:window-x="-8"
inkscape:window-y="-8"
inkscape:window-maximized="1"
units="px"
showborder="true" />
<metadata
id="metadata7">
<rdf:RDF>
<cc:Work
rdf:about="">
<dc:format>image/svg+xml</dc:format>
<dc:type
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
<dc:title></dc:title>
</cc:Work>
</rdf:RDF>
</metadata>
<g
inkscape:label="Capa 1"
inkscape:groupmode="layer"
id="layer1"
transform="translate(0,-796.36219)">
<g
id="g4177"
transform="matrix(0.51436047,0,0,0.59495735,-334.60687,650.43877)">
<g
id="g4141" />
<g
id="g4143" />
<g
id="g4145" />
<g
id="g4147" />
<g
id="g4149" />
<g
id="g4151" />
<g
id="g4153" />
<g
id="g4155" />
<g
id="g4157" />
<g
id="g4159" />
<g
id="g4161" />
<g
id="g4163" />
<g
id="g4165" />
<g
id="g4167" />
<g
id="g4169" />
<g
id="g4263"
transform="matrix(0.94954959,0,0,0.94954959,-111.49858,393.65111)">
<g
id="g4269" />
<g
id="g4342"
transform="translate(736.24631,-345.97247)">
<path
style="fill:#9dc6fb"
inkscape:connector-curvature="0"
d="m 584.32729,454.42324 c -0.995,-51.995 -44.49,-93.257 -96.488,-91.376 -7.616,0.273 -14.792,-3.862 -18.446,-10.55 -22.605,-41.376 -66.519,-69.441 -116.989,-69.441 -51.757,0 -96.596,29.528 -118.647,72.648 -6.423,12.56 -19.224,10.9 -24.689,10.9 -40.126,0 -74.199,25.852 -86.512,61.804 -2.046,5.973 -6.938,10.463 -12.894,12.556 -22.389998,7.87 -38.250998,29.605 -37.275998,54.902 1.163,30.174 26.849,53.631 57.044998,53.631 l 359.817,0 c 52.291,0 96.08,-42.793 95.079,-95.074 z"
id="path4285" />
<path
style="fill:#80b4fb"
inkscape:connector-curvature="0"
d="m 190.59629,495.86724 c -0.975,-25.298 14.885,-47.033 37.276,-54.902 5.956,-2.094 10.848,-6.584 12.894,-12.556 12.313,-35.952 46.385,-61.804 86.512,-61.804 5.465,0 18.265,1.66 24.688,-10.9 13.005,-25.43 33.94,-46.125 59.541,-58.832 -17.812,-8.834 -37.873,-13.816 -59.103,-13.816 -51.757,0 -96.596,29.528 -118.647,72.648 -6.423,12.56 -19.224,10.9 -24.689,10.9 -40.126,0 -74.199,25.852 -86.512,61.804 -2.046,5.973 -6.938,10.463 -12.894,12.556 -22.389998,7.87 -38.250998,29.605 -37.275998,54.902 1.163,30.174 26.849,53.63 57.044998,53.63 l 118.21,0 c -30.196,0 -55.881,-23.457 -57.045,-53.63 z"
id="path4287" />
</g>
</g>
</g>
<g
id="g4241"
transform="matrix(0.1453379,0,0,0.1453379,47.012211,854.83732)"
style="fill:#4d4d4d">
<g
id="g4197"
style="fill:#4d4d4d;fill-opacity:1">
<path
id="path4201"
d="m 867.699,356.238 -31.5,-26.6 c -9.699,-8.2 -24,-7.8 -33.199,0.9 l -17.4,16.3 c -14.699,-7.1 -30.299,-12.1 -46.4,-15 l -4.898,-24 c -2.5,-12.4 -14,-21 -26.602,-20 l -41.1,3.5 c -12.6,1.1 -22.5,11.4 -22.9,24.1 l -0.799,24.4 c -15.801,5.7 -30.701,13.5 -44.301,23.3 l -20.799,-13.8 c -10.602,-7 -24.701,-5 -32.9,4.7 l -26.6,31.7 c -8.201,9.7 -7.801,24 0.898,33.2 l 18.201,19.399 c -6.301,14.2 -10.801,29.101 -13.4,44.4 l -26,5.3 c -12.4,2.5 -21,14 -20,26.601 l 3.5,41.1 c 1.1,12.6 11.4,22.5 24.1,22.9 l 28.1,0.899 c 5.102,13.4 11.801,26.101 19.9,38 l -15.699,23.7 c -7,10.6 -5,24.7 4.699,32.9 l 31.5,26.6 c 9.701,8.2 24,7.8 33.201,-0.9 l 20.6,-19.3 c 13.5,6.3 27.699,11 42.299,13.8 l 5.701,28.2 c 2.5,12.4 14,21 26.6,20 l 41.1,-3.5 c 12.6,-1.1 22.5,-11.399 22.9,-24.1 l 0.9,-27.601 c 15,-5.3 29.199,-12.5 42.299,-21.399 l 22.701,15 c 10.6,7 24.699,5 32.9,-4.7 l 26.6,-31.5 c 8.199,-9.7 7.799,-24 -0.9,-33.2 L 872.7,592.138 c 6.701,-14.2 11.602,-29.2 14.4,-44.601 l 25,-5.1 c 12.4,-2.5 21,-14 20,-26.601 l -3.5,-41.1 c -1.1,-12.6 -11.4,-22.5 -24.1,-22.9 l -25.1,-0.8 c -5.201,-14.6 -12.201,-28.399 -20.9,-41.2 l 13.699,-20.6 c 7.201,-10.598 5.201,-24.798 -4.5,-32.998 z M 712.801,593.837 c -44.4,3.801 -83.602,-29.3 -87.301,-73.699 -3.801,-44.4 29.301,-83.601 73.699,-87.301 44.4,-3.8 83.602,29.301 87.301,73.7 3.801,44.401 -29.301,83.601 -73.699,87.3 z"
inkscape:connector-curvature="0"
style="fill:#4d4d4d;fill-opacity:1" />
<path
id="path4203"
d="m 205,704.438 c -12.6,1.3 -22.3,11.899 -22.4,24.6 l -0.3,25.3 c -0.2,12.7 9.2,23.5 21.8,25.101 l 18.6,2.399 c 3.1,11.301 7.5,22.101 13.2,32.301 l -12,14.8 c -8,9.899 -7.4,24.1 1.5,33.2 l 17.7,18.1 c 8.9,9.1 23.1,10.1 33.2,2.3 l 14.899,-11.5 c 10.5,6.2 21.601,11.101 33.2,14.5 l 2,19.2 c 1.3,12.6 11.9,22.3 24.6,22.4 l 25.301,0.3 c 12.699,0.2 23.5,-9.2 25.1,-21.8 l 2.3,-18.2 c 12.601,-3.101 24.601,-7.8 36,-14 l 14,11.3 c 9.9,8 24.101,7.4 33.201,-1.5 l 18.1,-17.7 c 9.1,-8.899 10.1,-23.1 2.301,-33.2 L 496.6,818.438 c 6.6,-11 11.701,-22.7 15.201,-35 l 16.6,-1.7 c 12.6,-1.3 22.299,-11.9 22.4,-24.6 l 0.299,-25.301 c 0.201,-12.699 -9.199,-23.5 -21.799,-25.1 l -16.201,-2.1 c -3.1,-12.2 -7.699,-24 -13.699,-35 l 10.1,-12.4 c 8,-9.9 7.4,-24.1 -1.5,-33.2 l -17.699,-18.1 c -8.9,-9.101 -23.102,-10.101 -33.201,-2.3 l -12.101,9.3 c -11.399,-6.9 -23.6,-12.2 -36.399,-15.8 L 407,581.437 c -1.3,-12.601 -11.899,-22.3 -24.6,-22.4 l -25.3,-0.3 c -12.7,-0.2 -23.5,9.2 -25.101,21.8 l -2,15.601 c -13.199,3.399 -25.899,8.6 -37.699,15.399 l -12.5,-10.2 c -9.9,-8 -24.101,-7.399 -33.201,1.5 l -18.2,17.801 c -9.1,8.899 -10.1,23.1 -2.3,33.199 l 10.7,13.801 c -6.2,11 -11.1,22.699 -14.3,35 l -17.499,1.8 z m 163.3,-28.601 c 36.3,0.4 65.399,30.301 65,66.601 -0.4,36.3 -30.301,65.399 -66.601,65 -36.3,-0.4 -65.399,-30.3 -65,-66.601 0.401,-36.299 30.301,-65.399 66.601,-65 z"
inkscape:connector-curvature="0"
style="fill:#4d4d4d;fill-opacity:1" />
</g>
<g
id="g4205"
style="fill:#4d4d4d" />
<g
id="g4207"
style="fill:#4d4d4d" />
<g
id="g4209"
style="fill:#4d4d4d" />
<g
id="g4211"
style="fill:#4d4d4d" />
<g
id="g4213"
style="fill:#4d4d4d" />
<g
id="g4215"
style="fill:#4d4d4d" />
<g
id="g4217"
style="fill:#4d4d4d" />
<g
id="g4219"
style="fill:#4d4d4d" />
<g
id="g4221"
style="fill:#4d4d4d" />
<g
id="g4223"
style="fill:#4d4d4d" />
<g
id="g4225"
style="fill:#4d4d4d" />
<g
id="g4227"
style="fill:#4d4d4d" />
<g
id="g4229"
style="fill:#4d4d4d" />
<g
id="g4231"
style="fill:#4d4d4d" />
<g
id="g4233"
style="fill:#4d4d4d" />
</g>
</g>
</svg>

After

Width:  |  Height:  |  Size: 8.0 KiB

83
src/app.ts Normal file
View File

@@ -0,0 +1,83 @@
import log from './services/log'
import {NextFunction, Request, Response} from 'express';
import {getUserAgent} from "./services/sessions";
import {controllerV1} from "./controllers/v1";
const express = require('express');
const app = express();
const bodyParser = require('body-parser');
const version: string = 'v' + require('../package.json').version
// Convert request objects to JSON
app.use(bodyParser.json({
limit: '50mb',
verify(req: Request, res: Response, buf: any) {
req.body = buf;
}
}));
// Access log
app.use(function(req: Request, res: Response, next: NextFunction) {
if (req.url != '/health') {
// count the request for the log prefix
log.incRequests()
// build access message
let body = "";
if (req.method == 'POST' && req.body) {
body += " body: "
try {
body += JSON.stringify(req.body)
} catch(e) {
body += req.body
}
}
log.info(`Incoming request => ${req.method} ${req.url}${body}`);
}
next();
});
// *********************************************************************************************************************
// Routes
// Show welcome message
app.get("/", ( req: Request, res: Response ) => {
res.send({
"msg": "FlareSolverr is ready!",
"version": version,
"userAgent": getUserAgent()
});
});
// Health endpoint. this endpoint is special because it doesn't print traces
app.get("/health", ( req: Request, res: Response ) => {
res.send({
"status": "ok"
});
});
// Controller v1
app.post("/v1", async( req: Request, res: Response ) => {
await controllerV1(req, res);
});
// *********************************************************************************************************************
// Unknown paths or verbs
app.use(function (req : Request, res : Response) {
res.status(404)
.send({"error": "Unknown resource or HTTP verb"})
})
// Errors
app.use(function (err: any, req: Request, res: Response, next: NextFunction) {
if (err) {
let msg = 'Invalid request: ' + err;
msg = msg.replace("\n", "").replace("\r", "")
log.error(msg)
res.send({"error": msg})
} else {
next()
}
})
module.exports = app;

View File

@@ -1,31 +0,0 @@
import got from 'got'
import { sleep } from '../utils'
/*
This method uses the captcha-harvester project:
https://github.com/NoahCardoza/CaptchaHarvester
While the function must take url/sitekey/type args,
they aren't used because the harvester server must
be preconfigured.
ENV:
HARVESTER_ENDPOINT: This must be the full path
to the /token endpoint of the harvester.
E.G. "https://127.0.0.1:5000/token"
*/
export default async function solve(): Promise<string> {
const endpoint = process.env.HARVESTER_ENDPOINT
if (!endpoint) { throw Error('ENV variable `HARVESTER_ENDPOINT` must be set.') }
while (true) {
try {
return (await got.get(process.env.HARVESTER_ENDPOINT, {
https: { rejectUnauthorized: false }
})).body
} catch (e) {
if (e.response.statusCode !== 418) { throw e }
}
await sleep(3000)
}
}

View File

@@ -1,21 +0,0 @@
const solveCaptcha = require('hcaptcha-solver');
import { SolverOptions } from '.'
/*
This method uses the hcaptcha-solver project:
https://github.com/JimmyLaurent/hcaptcha-solver
TODO: allow user pass custom options to the solver.
ENV:
There are no other variables that must be set to get this to work
*/
export default async function solve({ url }: SolverOptions): Promise<string> {
try {
const token = await solveCaptcha(url)
return token
} catch (e) {
console.error(e)
return null
}
}

View File

@@ -1,3 +1,5 @@
import log from "../services/log";
export enum CaptchaType {
re = 'reCaptcha',
h = 'hCaptcha'
@@ -16,7 +18,9 @@ const captchaSolvers: { [key: string]: Solver } = {}
export default (): Solver => {
const method = process.env.CAPTCHA_SOLVER
if (!method) { return null }
if (!method || method.toLowerCase() == 'none') {
return null;
}
if (!(method in captchaSolvers)) {
try {
@@ -26,10 +30,12 @@ export default (): Solver => {
throw Error(`The solver '${method}' is not a valid captcha solving method.`)
} else {
console.error(e)
throw Error(`An error occured loading the solver '${method}'.`)
throw Error(`An error occurred loading the solver '${method}'.`)
}
}
}
log.info(`Using '${method}' to solve the captcha.`);
return captchaSolvers[method]
}

178
src/controllers/v1.ts Normal file
View File

@@ -0,0 +1,178 @@
import {Request, Response} from 'express';
import {Protocol} from "devtools-protocol";
import log from '../services/log'
import {browserRequest, ChallengeResolutionResultT, ChallengeResolutionT} from "../services/solver";
import {SessionCreateOptions} from "../services/sessions";
const sessions = require('../services/sessions')
const version: string = 'v' + require('../../package.json').version
interface V1Routes {
[key: string]: (params: V1RequestBase, response: V1ResponseBase) => Promise<void>
}
export interface Proxy {
url?: string
username?: string
password?: string
}
export interface V1RequestBase {
cmd: string
cookies?: Protocol.Network.CookieParam[],
maxTimeout?: number
proxy?: Proxy
session: string
headers?: Record<string, string> // deprecated v2, not used
userAgent?: string // deprecated v2, not used
}
interface V1RequestSession extends V1RequestBase {
}
export interface V1Request extends V1RequestBase {
url: string
method?: string
postData?: string
returnOnlyCookies?: boolean
download?: boolean // deprecated v2, not used
returnRawHtml?: boolean // deprecated v2, not used
}
export interface V1ResponseBase {
status: string
message: string
startTimestamp: number
endTimestamp: number
version: string
}
export interface V1ResponseSolution extends V1ResponseBase {
solution: ChallengeResolutionResultT
}
export interface V1ResponseSession extends V1ResponseBase {
session: string
}
export interface V1ResponseSessions extends V1ResponseBase {
sessions: string[]
}
export const routes: V1Routes = {
'sessions.create': async (params: V1RequestSession, response: V1ResponseSession): Promise<void> => {
const options: SessionCreateOptions = {
oneTimeSession: false,
cookies: params.cookies,
maxTimeout: params.maxTimeout,
proxy: params.proxy
}
const { sessionId, browser } = await sessions.create(params.session, options)
if (browser) {
response.status = "ok";
response.message = "Session created successfully.";
response.session = sessionId
} else {
throw Error('Error creating session.')
}
},
'sessions.list': async (params: V1RequestSession, response: V1ResponseSessions): Promise<void> => {
response.status = "ok";
response.message = "";
response.sessions = sessions.list();
},
'sessions.destroy': async (params: V1RequestSession, response: V1ResponseBase): Promise<void> => {
if (await sessions.destroy(params.session)) {
response.status = "ok";
response.message = "The session has been removed.";
} else {
throw Error('This session does not exist.')
}
},
'request.get': async (params: V1Request, response: V1ResponseSolution): Promise<void> => {
params.method = 'GET'
if (params.postData) {
throw Error('Cannot use "postBody" when sending a GET request.')
}
if (params.returnRawHtml) {
log.warn("Request parameter 'returnRawHtml' was removed in FlareSolverr v2.")
}
if (params.download) {
log.warn("Request parameter 'download' was removed in FlareSolverr v2.")
}
const result: ChallengeResolutionT = await browserRequest(params)
response.status = result.status;
response.message = result.message;
response.solution = result.result;
if (response.message) {
log.info(response.message)
}
},
'request.post': async (params: V1Request, response: V1ResponseSolution): Promise<void> => {
params.method = 'POST'
if (!params.postData) {
throw Error('Must send param "postBody" when sending a POST request.')
}
if (params.returnRawHtml) {
log.warn("Request parameter 'returnRawHtml' was removed in FlareSolverr v2.")
}
if (params.download) {
log.warn("Request parameter 'download' was removed in FlareSolverr v2.")
}
const result: ChallengeResolutionT = await browserRequest(params)
response.status = result.status;
response.message = result.message;
response.solution = result.result;
if (response.message) {
log.info(response.message)
}
},
}
export async function controllerV1(req: Request, res: Response): Promise<void> {
const response: V1ResponseBase = {
status: null,
message: null,
startTimestamp: Date.now(),
endTimestamp: 0,
version: version
}
try {
const params: V1RequestBase = req.body
// do some validations
if (!params.cmd) {
throw Error("Request parameter 'cmd' is mandatory.")
}
if (params.headers) {
log.warn("Request parameter 'headers' was removed in FlareSolverr v2.")
}
if (params.userAgent) {
log.warn("Request parameter 'userAgent' was removed in FlareSolverr v2.")
}
// set default values
if (!params.maxTimeout || params.maxTimeout < 1) {
params.maxTimeout = 60000;
}
// execute the command
const route = routes[params.cmd]
if (route) {
await route(params, response)
} else {
throw Error(`The command '${params.cmd}' is invalid.`)
}
} catch (e) {
res.status(500)
response.status = "error";
response.message = e.toString();
log.error(response.message)
}
response.endTimestamp = Date.now()
log.info(`Response in ${(response.endTimestamp - response.startTimestamp) / 1000} s`)
res.send(response)
}

View File

@@ -1,114 +0,0 @@
import log from './log'
import { createServer, IncomingMessage, ServerResponse } from 'http';
import { RequestContext } from './types'
import Router, { BaseAPICall } from './routes'
const version: string = "v" + require('../package.json').version
const serverPort: number = Number(process.env.PORT) || 8191
const serverHost: string = process.env.HOST || '0.0.0.0'
function errorResponse(errorMsg: string, res: ServerResponse, startTimestamp: number) {
log.error(errorMsg)
const response = {
status: 'error',
message: errorMsg,
startTimestamp,
endTimestamp: Date.now(),
version
}
res.writeHead(500, {
'Content-Type': 'application/json'
})
res.write(JSON.stringify(response))
res.end()
}
function successResponse(successMsg: string, extendedProperties: object, res: ServerResponse, startTimestamp: number) {
const endTimestamp = Date.now()
log.info(`Successful response in ${(endTimestamp - startTimestamp) / 1000} s`)
if (successMsg) { log.info(successMsg) }
const response = Object.assign({
status: 'ok',
message: successMsg || '',
startTimestamp,
endTimestamp,
version
}, extendedProperties || {})
res.writeHead(200, {
'Content-Type': 'application/json'
})
res.write(JSON.stringify(response))
res.end()
}
function validateIncomingRequest(ctx: RequestContext, params: BaseAPICall) {
log.info(`Params: ${JSON.stringify(params)}`)
if (ctx.req.method !== 'POST') {
ctx.errorResponse('Only the POST method is allowed')
return false
}
if (ctx.req.url !== '/v1') {
ctx.errorResponse('Only /v1 endpoint is allowed')
return false
}
if (!params.cmd) {
ctx.errorResponse("Parameter 'cmd' is mandatory")
return false
}
return true
}
createServer((req: IncomingMessage, res: ServerResponse) => {
const startTimestamp = Date.now()
// count the request for the log prefix
log.incRequests()
log.info(`Incoming request: ${req.method} ${req.url}`)
// show welcome message
if (req.url == '/') {
successResponse("FlareSolverr is ready!", null, res, startTimestamp);
return;
}
// get request body
const bodyParts: any[] = []
req.on('data', chunk => {
bodyParts.push(chunk)
}).on('end', () => {
// parse params
const body = Buffer.concat(bodyParts).toString()
let params: BaseAPICall = null
try {
params = JSON.parse(body)
} catch (err) {
errorResponse('Body must be in JSON format', res, startTimestamp)
return
}
const ctx: RequestContext = {
req,
res,
startTimestamp,
errorResponse: (msg) => errorResponse(msg, res, startTimestamp),
successResponse: (msg, extendedProperties) => successResponse(msg, extendedProperties, res, startTimestamp)
}
// validate params
if (!validateIncomingRequest(ctx, params)) { return }
// process request
Router(ctx, params).catch(e => {
console.error(e)
ctx.errorResponse(e.message)
})
})
}).listen(serverPort, serverHost, () => {
log.info(`FlareSolverr ${version} listening on http://${serverHost}:${serverPort}`)
})

View File

@@ -1,19 +0,0 @@
let requests = 0
const LOG_HTML: boolean = Boolean(process.env.LOG_HTML) || false
export default {
incRequests: () => { requests++ },
html(html: string) {
if (LOG_HTML)
this.debug(html)
},
...require('console-log-level')(
{
level: process.env.LOG_LEVEL || 'info',
prefix(level: string) {
return `${new Date().toISOString()} ${level.toUpperCase()} REQ-${requests}`
}
}
)
}

151
src/providers/cloudflare.ts Normal file
View File

@@ -0,0 +1,151 @@
import {Page, HTTPResponse} from 'puppeteer'
import log from "../services/log";
/**
* This class contains the logic to solve protections provided by CloudFlare
**/
// the selector '.text-gray-600' is not working well because it can be hidden
// <span style="display: none;" class="text-gray-600" data-translate="error">error code: 1020</span>
const BAN_SELECTORS: string[] = [];
const CHALLENGE_SELECTORS: string[] = [
'#trk_jschal_js', '.ray_id', '.attack-box', '#cf-please-wait', // CloudFlare
'#link-ddg', // DDoS-GUARD
'td.info #js_info' // Custom CloudFlare for EbookParadijs, Film-Paleis, MuziekFabriek and Puur-Hollands
];
const CAPTCHA_SELECTORS: string[] = [
'input[name="cf_captcha_kind"]'
];
export default async function resolveChallenge(url: string, page: Page, response: HTTPResponse): Promise<HTTPResponse> {
// look for challenge and return fast if not detected
let cfDetected = response.headers().server && response.headers().server.startsWith('cloudflare');
if (cfDetected) {
if (response.status() == 403 || response.status() == 503) {
cfDetected = true; // Defected CloudFlare and DDoS-GUARD
} else if (response.headers().vary && response.headers().vary.trim() == 'Accept-Encoding,User-Agent' &&
response.headers()['content-encoding'] && response.headers()['content-encoding'].trim() == 'br') {
cfDetected = true; // Detected Custom CloudFlare for EbookParadijs, Film-Paleis, MuziekFabriek and Puur-Hollands
} else {
cfDetected = false;
}
}
if (cfDetected) {
log.info('Cloudflare detected');
} else {
log.info('Cloudflare not detected');
return response;
}
if (await findAnySelector(page, BAN_SELECTORS)) {
throw new Error('Cloudflare has blocked this request. Probably your IP is banned for this site, check in your web browser.');
}
// find Cloudflare selectors
let selectorFound = false;
let selector: string = await findAnySelector(page, CHALLENGE_SELECTORS)
if (selector) {
selectorFound = true;
log.debug(`Javascript challenge element '${selector}' detected.`)
log.debug('Waiting for Cloudflare challenge...')
while (true) {
try {
selector = await findAnySelector(page, CHALLENGE_SELECTORS)
if (!selector) {
// solved!
log.debug('Challenge element not found')
break
} else {
log.debug(`Javascript challenge element '${selector}' detected.`)
// new Cloudflare Challenge #cf-please-wait
const displayStyle = await page.evaluate((selector) => {
return getComputedStyle(document.querySelector(selector)).getPropertyValue("display");
}, selector);
if (displayStyle == "none") {
// spinner is hidden, could be a captcha or not
log.debug('Challenge element is hidden')
// wait until redirecting disappears
while (true) {
try {
await page.waitForTimeout(1000)
const displayStyle2 = await page.evaluate(() => {
return getComputedStyle(document.querySelector('#cf-spinner-redirecting')).getPropertyValue("display");
});
if (displayStyle2 == "none") {
break // hCaptcha detected
}
} catch (error) {
break // redirection completed
}
}
break
} else {
log.debug('Challenge element is visible')
}
}
log.debug('Found challenge element again')
} catch (error)
{
log.debug("Unexpected error: " + error);
if (!error.toString().includes("Execution context was destroyed")) {
break
}
}
log.debug('Waiting for Cloudflare challenge...')
await page.waitForTimeout(1000)
}
log.debug('Validating HTML code...')
} else {
log.debug(`No challenge element detected.`)
}
// check for CAPTCHA challenge
if (await findAnySelector(page, CAPTCHA_SELECTORS)) {
log.info('CAPTCHA challenge detected');
throw new Error('FlareSolverr can not resolve CAPTCHA challenges. Since the captcha doesn\'t always appear, you may have better luck with the next request.');
// const captchaSolver = getCaptchaSolver()
// if (captchaSolver) {
// // to-do: get the params
// log.info('Waiting to receive captcha token to bypass challenge...')
// const token = await captchaSolver({
// url,
// sitekey,
// type: captchaType
// })
// log.debug(`Token received: ${token}`);
// // to-do: send the token
// }
// } else {
// throw new Error('Captcha detected but no automatic solver is configured.');
// }
} else {
if (!selectorFound)
{
throw new Error('No challenge selectors found, unable to proceed.')
} else {
log.info('Challenge solved');
}
}
return response;
}
async function findAnySelector(page: Page, selectors: string[]) {
for (const selector of selectors) {
const cfChallengeElem = await page.$(selector)
if (cfChallengeElem) {
return selector;
}
}
return null;
}

View File

@@ -1,491 +0,0 @@
import { v1 as UUIDv1 } from 'uuid'
import sessions, { SessionsCacheItem } from './session'
import { RequestContext } from './types'
import log from './log'
import { SetCookie, Request, Headers, HttpMethod, Overrides, Cookie } from 'puppeteer'
import { TimeoutError } from 'puppeteer/Errors'
import getCaptchaSolver, { CaptchaType } from './captcha'
import * as Puppeteer from "puppeteer-extra/dist/puppeteer";
const Timeout = require('await-timeout');
export interface BaseAPICall {
cmd: string
}
interface BaseSessionsAPICall extends BaseAPICall {
session?: string
}
interface SessionsCreateAPICall extends BaseSessionsAPICall {
userAgent?: string,
cookies?: SetCookie[],
headers?: Headers
maxTimeout?: number
proxy?: any
}
interface BaseRequestAPICall extends BaseAPICall {
url: string
method?: HttpMethod
postData?: string
session?: string
userAgent?: string
maxTimeout?: number
cookies?: SetCookie[],
headers?: Headers
proxy?: any, // TODO: use interface not any
download?: boolean
returnOnlyCookies?: boolean
}
interface Routes {
[key: string]: (ctx: RequestContext, params: BaseAPICall) => void | Promise<void>
}
interface ChallengeResolutionResultT {
url: string
status: number,
headers?: Headers,
response: string,
cookies: object[]
userAgent: string
}
interface ChallengeResolutionT {
status?: string
message: string
result: ChallengeResolutionResultT
}
interface OverrideResolvers {
method?: (request: Request) => HttpMethod,
postData?: (request: Request) => string,
headers?: (request: Request) => Headers
}
type OverridesProps =
'method' |
'postData' |
'headers'
// We always set a Windows User-Agent because ARM builds are detected by CloudFlare
const DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36"
const CHALLENGE_SELECTORS = ['#trk_jschal_js', '.ray_id', '.attack-box']
const TOKEN_INPUT_NAMES = ['g-recaptcha-response', 'h-captcha-response']
async function interceptResponse(page: Puppeteer.Page, callback: (payload: ChallengeResolutionT) => any) {
const client = await page.target().createCDPSession();
await client.send('Fetch.enable', {
patterns: [
{
urlPattern: '*',
resourceType: 'Document',
requestStage: 'Response',
},
],
});
client.on('Fetch.requestPaused', async (e) => {
log.debug('Fetch.requestPaused. Checking if the response has valid cookies')
let headers = e.responseHeaders || []
let cookies = await page.cookies();
log.debug(cookies)
if (cookies.filter((c: Cookie) => c.name === 'cf_clearance').length > 0) {
log.debug('Aborting request and return cookies. valid cookies found')
await client.send('Fetch.failRequest', {requestId: e.requestId, errorReason: 'Aborted'})
let status = 'ok'
let message = ''
const payload: ChallengeResolutionT = {
status,
message,
result: {
url: page.url(),
status: e.status,
headers: headers.reduce((a: any, x: { name: any; value: any }) => ({ ...a, [x.name]: x.value }), {}),
response: null,
cookies: cookies,
userAgent: ''
}
}
callback(payload);
} else {
log.debug('Continuing request. no valid cookies found')
await client.send('Fetch.continueRequest', {requestId: e.requestId})
}
});
}
async function resolveChallengeWithTimeout(ctx: RequestContext, params: BaseRequestAPICall, page: Puppeteer.Page) {
const maxTimeout = params.maxTimeout || 60000
const timer = new Timeout();
try {
const promise = resolveChallenge(ctx, params, page);
return await Promise.race([
promise,
timer.set(maxTimeout, `Maximum timeout reached. maxTimeout=${maxTimeout} (ms)`)
]);
} finally {
timer.clear();
}
}
async function resolveChallenge(ctx: RequestContext, { url, proxy, download, returnOnlyCookies }: BaseRequestAPICall, page: Puppeteer.Page): Promise<ChallengeResolutionT | void> {
let status = 'ok'
let message = ''
if (proxy) {
log.debug("Apply proxy");
if (proxy.username)
await page.authenticate({ username: proxy.username, password: proxy.password });
}
log.debug(`Navigating to... ${url}`)
let response = await page.goto(url, { waitUntil: 'domcontentloaded' })
log.html(await page.content())
// look for challenge
if (response.headers().server.startsWith('cloudflare')) {
log.info('Cloudflare detected')
if (await page.$('.cf-error-code')) {
await page.close()
return ctx.errorResponse('Cloudflare has blocked this request (Code 1020 Detected).')
}
if (response.status() > 400) {
// detect cloudflare wait 5s
let selectorFoundCount = 0
for (const selector of CHALLENGE_SELECTORS) {
const cfChallengeElem = await page.$(selector)
if (cfChallengeElem) {
selectorFoundCount++
log.debug(`'${selector}' challenge element detected.`)
log.debug('Waiting for Cloudflare challenge...')
let interceptingResult: ChallengeResolutionT;
if (returnOnlyCookies) { //If we just want to get the cookies, intercept the response before we get the content/body (just cookies and headers)
await interceptResponse(page, async function(payload){
interceptingResult = payload;
});
}
while (true) {
await page.waitFor(1000)
try {
// catch exception timeout in waitForNavigation
response = await page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 5000 })
} catch (error) { }
if (returnOnlyCookies && interceptingResult) {
await page.close();
return interceptingResult;
}
try {
// catch Execution context was destroyed
const cfChallengeElem = await page.$(selector)
if (!cfChallengeElem) { break }
log.debug('Found challenge element again...')
} catch (error)
{ }
response = await page.reload({ waitUntil: 'domcontentloaded' })
log.debug('Reloaded page...')
log.html(await page.content())
}
log.debug('Validating HTML code...')
break
} else {
log.debug(`No '${selector}' challenge element detected.`)
}
}
log.debug("Number of selector found: " + selectorFoundCount + ", total selector: " + CHALLENGE_SELECTORS.length)
if (selectorFoundCount == 0)
{
await page.close()
return ctx.errorResponse('No challenge selectors found, unable to proceed')
}
}
// it seems some captcha pages return 200 sometimes
if (await page.$('input[name="cf_captcha_kind"]')) {
const captchaSolver = getCaptchaSolver()
if (captchaSolver) {
const captchaStartTimestamp = Date.now()
const challengeForm = await page.$('#challenge-form')
if (challengeForm) {
const captchaTypeElm = await page.$('input[name="cf_captcha_kind"]')
const cfCaptchaType: string = await captchaTypeElm.evaluate((e: any) => e.value)
const captchaType: CaptchaType = (CaptchaType as any)[cfCaptchaType]
if (!captchaType) { return ctx.errorResponse('Unknown captcha type!') }
let sitekey = null
if (captchaType != 'hCaptcha' && process.env.CAPTCHA_SOLVER != 'hcaptcha-solver') {
const sitekeyElem = await page.$('*[data-sitekey]')
if (!sitekeyElem) { return ctx.errorResponse('Could not find sitekey!') }
sitekey = await sitekeyElem.evaluate((e) => e.getAttribute('data-sitekey'))
}
log.info('Waiting to receive captcha token to bypass challenge...')
const token = await captchaSolver({
url,
sitekey,
type: captchaType
})
if (!token) {
await page.close()
return ctx.errorResponse('Token solver failed to return a token.')
}
for (const name of TOKEN_INPUT_NAMES) {
const input = await page.$(`textarea[name="${name}"]`)
if (input) { await input.evaluate((e: HTMLTextAreaElement, token) => { e.value = token }, token) }
}
// ignore preset event listeners on the form
await page.evaluate(() => {
window.addEventListener('submit', (e) => { event.stopPropagation() }, true)
})
// it seems some sites obfuscate their challenge forms
// TODO: look into how they do it and come up with a more solid solution
try {
// this element is added with js and we want to wait for all the js to load before submitting
await page.waitForSelector('#challenge-form [type=submit]', { timeout: 5000 })
} catch (err) {
if (err instanceof TimeoutError) {
log.debug(`No '#challenge-form [type=submit]' element detected.`)
}
}
// calculates the time it took to solve the captcha
const captchaSolveTotalTime = Date.now() - captchaStartTimestamp
// generates a random wait time
const randomWaitTime = (Math.floor(Math.random() * 20) + 10) * 1000
// waits, if any, time remaining to appear human but stay as fast as possible
const timeLeft = randomWaitTime - captchaSolveTotalTime
if (timeLeft > 0) { await page.waitFor(timeLeft) }
let interceptingResult: ChallengeResolutionT;
if (returnOnlyCookies) { //If we just want to get the cookies, intercept the response before we get the content/body (just cookies and headers)
await interceptResponse(page, async function(payload){
interceptingResult = payload;
});
}
// submit captcha response
challengeForm.evaluate((e: HTMLFormElement) => e.submit())
response = await page.waitForNavigation({ waitUntil: 'domcontentloaded' })
if (returnOnlyCookies && interceptingResult) {
await page.close();
return interceptingResult;
}
}
} else {
status = 'warning'
message = 'Captcha detected but no automatic solver is configured.'
}
}
log.debug("Response is: " + response.status())
}
const payload: ChallengeResolutionT = {
status,
message,
result: {
url: page.url(),
status: response.status(),
headers: response.headers(),
response: null,
cookies: await page.cookies(),
userAgent: await page.evaluate(() => navigator.userAgent)
}
}
if (download) {
// for some reason we get an error unless we reload the page
// has something to do with a stale buffer and this is the quickest
// fix since I am short on time
response = await page.goto(url, { waitUntil: 'domcontentloaded' })
payload.result.response = (await response.buffer()).toString('base64')
} else {
payload.result.response = await page.content()
}
// make sure the page is closed because if it isn't and error will be thrown
// when a user uses a temporary session, the browser make be quit before
// the page is properly closed.
await page.close()
return payload
}
function mergeSessionWithParams({ defaults }: SessionsCacheItem, params: BaseRequestAPICall): BaseRequestAPICall {
const copy = { ...defaults, ...params }
// custom merging logic
copy.headers = { ...defaults.headers || {}, ...params.headers || {} } || null
return copy
}
async function setupPage(ctx: RequestContext, params: BaseRequestAPICall, browser: Puppeteer.Browser): Promise<Puppeteer.Page> {
const page = await browser.newPage()
// merge session defaults with params
const { method, postData, userAgent, headers, cookies } = params
let overrideResolvers: OverrideResolvers = {}
if (method !== 'GET') {
log.debug(`Setting method to ${method}`)
overrideResolvers.method = request => method
}
if (postData) {
log.debug(`Setting body data to ${postData}`)
overrideResolvers.postData = request => postData
}
if (userAgent) {
log.debug(`Using custom UA: ${userAgent}`)
await page.setUserAgent(userAgent)
} else {
await page.setUserAgent(DEFAULT_USER_AGENT)
}
if (headers) {
log.debug(`Adding custom headers: ${JSON.stringify(headers, null, 2)}`,)
overrideResolvers.headers = request => Object.assign(request.headers(), headers)
}
if (cookies) {
log.debug(`Setting custom cookies: ${JSON.stringify(cookies, null, 2)}`,)
await page.setCookie(...cookies)
}
// if any keys have been set on the object
if (Object.keys(overrideResolvers).length > 0) {
log.debug(overrideResolvers)
let callbackRunOnce = false
const callback = (request: Request) => {
if (callbackRunOnce || !request.isNavigationRequest()) {
request.continue()
return
}
callbackRunOnce = true
const overrides: Overrides = {}
Object.keys(overrideResolvers).forEach((key: OverridesProps) => {
// @ts-ignore
overrides[key] = overrideResolvers[key](request)
});
log.debug(overrides)
request.continue(overrides)
}
await page.setRequestInterception(true)
page.on('request', callback)
}
return page
}
const browserRequest = async (ctx: RequestContext, params: BaseRequestAPICall) => {
const oneTimeSession = params.session === undefined
const sessionId = params.session || UUIDv1()
const session = oneTimeSession
? await sessions.create(sessionId, {
userAgent: params.userAgent,
oneTimeSession
})
: sessions.get(sessionId)
if (session === false) {
return ctx.errorResponse('This session does not exist. Use \'list_sessions\' to see all the existing sessions.')
}
params = mergeSessionWithParams(session, params)
try {
const page = await setupPage(ctx, params, session.browser)
const data = await resolveChallengeWithTimeout(ctx, params, page)
if (data) {
const { status } = data
delete data.status
ctx.successResponse(data.message, {
...(oneTimeSession ? {} : { session: sessionId }),
...(status ? { status } : {}),
solution: data.result
})
}
} catch (error) {
log.error(error)
return ctx.errorResponse("Unable to process browser request. Error: " + error)
} finally {
if (oneTimeSession) { sessions.destroy(sessionId) }
}
}
export const routes: Routes = {
'sessions.create': async (ctx, { session, ...options }: SessionsCreateAPICall) => {
session = session || UUIDv1()
const { browser } = await sessions.create(session, options)
if (browser) { ctx.successResponse('Session created successfully.', { session }) }
},
'sessions.list': (ctx) => {
ctx.successResponse(null, { sessions: sessions.list() })
},
'sessions.destroy': async (ctx, { session }: BaseSessionsAPICall) => {
if (await sessions.destroy(session)) { return ctx.successResponse('The session has been removed.') }
ctx.errorResponse('This session does not exist.')
},
'request.get': async (ctx, params: BaseRequestAPICall) => {
params.method = 'GET'
if (params.postData) {
return ctx.errorResponse('Cannot use "postBody" when sending a GET request.')
}
await browserRequest(ctx, params)
},
'request.post': async (ctx, params: BaseRequestAPICall) => {
params.method = 'POST'
if (!params.postData) {
return ctx.errorResponse('Must send param "postBody" when sending a POST request.')
}
await browserRequest(ctx, params)
},
'request.cookies': async (ctx, params: BaseRequestAPICall) => {
params.returnOnlyCookies = true
params.method = 'GET'
if (params.postData) {
return ctx.errorResponse('Cannot use "postBody" when sending a GET request.')
}
await browserRequest(ctx, params)
},
}
export default async function Router(ctx: RequestContext, params: BaseAPICall): Promise<void> {
const route = routes[params.cmd]
if (route) { return await route(ctx, params) }
return ctx.errorResponse(`The command '${params.cmd}' is invalid.`)
}

63
src/server.ts Normal file
View File

@@ -0,0 +1,63 @@
import log from './services/log'
import {testWebBrowserInstallation} from "./services/sessions";
const app = require("./app");
const version: string = 'v' + require('../package.json').version
const serverPort: number = Number(process.env.PORT) || 8191
const serverHost: string = process.env.HOST || '0.0.0.0'
function validateEnvironmentVariables() {
// ip and port variables are validated by nodejs
if (process.env.LOG_LEVEL && ['error', 'warn', 'info', 'verbose', 'debug'].indexOf(process.env.LOG_LEVEL) == -1) {
log.error(`The environment variable 'LOG_LEVEL' is wrong. Check the documentation.`);
process.exit(1);
}
if (process.env.LOG_HTML && ['true', 'false'].indexOf(process.env.LOG_HTML) == -1) {
log.error(`The environment variable 'LOG_HTML' is wrong. Check the documentation.`);
process.exit(1);
}
if (process.env.HEADLESS && ['true', 'false'].indexOf(process.env.HEADLESS) == -1) {
log.error(`The environment variable 'HEADLESS' is wrong. Check the documentation.`);
process.exit(1);
}
// todo: fix resolvers
// try {
// getCaptchaSolver();
// } catch (e) {
// log.error(`The environment variable 'CAPTCHA_SOLVER' is wrong. ${e.message}`);
// process.exit(1);
// }
}
// Init
log.info(`FlareSolverr ${version}`);
log.debug('Debug log enabled');
process.on('SIGTERM', () => {
// Capture signal on Docker Stop #158
log.info("Process interrupted")
process.exit(0)
})
process.on('uncaughtException', function(err) {
// Avoid crashing in NodeJS 17 due to UnhandledPromiseRejectionWarning: Unhandled promise rejection.
log.error(err)
})
validateEnvironmentVariables();
testWebBrowserInstallation().then(() => {
// Start server
app.listen(serverPort, serverHost, () => {
log.info(`Listening on http://${serverHost}:${serverPort}`);
})
}).catch(function(e) {
log.error(e);
const msg: string = "" + e;
if (msg.includes('while trying to connect to the browser!')) {
log.error(`It seems that the system is too slow to run FlareSolverr.
If you are running with Docker, try to remove CPU limits in the container.
If not, try setting the 'BROWSER_TIMEOUT' environment variable and the 'maxTimeout' parameter to higher values.`);
}
process.exit(1);
})

41
src/services/log.ts Normal file
View File

@@ -0,0 +1,41 @@
let requests = 0
const LOG_HTML: boolean = process.env.LOG_HTML == 'true';
function toIsoString(date: Date) {
// this function fixes Date.toISOString() adding timezone
let tzo = -date.getTimezoneOffset(),
dif = tzo >= 0 ? '+' : '-',
pad = function(num: number) {
let norm = Math.floor(Math.abs(num));
return (norm < 10 ? '0' : '') + norm;
};
return date.getFullYear() +
'-' + pad(date.getMonth() + 1) +
'-' + pad(date.getDate()) +
'T' + pad(date.getHours()) +
':' + pad(date.getMinutes()) +
':' + pad(date.getSeconds()) +
dif + pad(tzo / 60) +
':' + pad(tzo % 60);
}
export default {
incRequests: () => {
requests++
},
html(html: string) {
if (LOG_HTML) {
this.debug(html)
}
},
...require('console-log-level')(
{level: process.env.LOG_LEVEL || 'info',
prefix(level: string) {
const req = (requests > 0) ? ` REQ-${requests}` : '';
return `${toIsoString(new Date())} ${level.toUpperCase()}${req}`
}
}
)
}

194
src/services/sessions.ts Normal file
View File

@@ -0,0 +1,194 @@
import {v1 as UUIDv1} from 'uuid'
import * as path from 'path'
import {Browser} from 'puppeteer'
import {Protocol} from "devtools-protocol";
import log from './log'
import {Proxy} from "../controllers/v1";
const os = require('os');
const fs = require('fs');
const puppeteer = require('puppeteer');
export interface SessionsCacheItem {
sessionId: string
browser: Browser
}
interface SessionsCache {
[key: string]: SessionsCacheItem
}
export interface SessionCreateOptions {
oneTimeSession: boolean
cookies?: Protocol.Network.CookieParam[],
maxTimeout?: number
proxy?: Proxy
}
const sessionCache: SessionsCache = {}
let webBrowserUserAgent: string;
function buildExtraPrefsFirefox(proxy: Proxy): object {
// Default configurations are defined here
// https://github.com/puppeteer/puppeteer/blob/v3.3.0/src/Launcher.ts#L481
const extraPrefsFirefox = {
// Disable newtabpage
"browser.newtabpage.enabled": false,
"browser.startup.homepage": "about:blank",
// Do not warn when closing all open tabs
"browser.tabs.warnOnClose": false,
// Disable telemetry
"toolkit.telemetry.reportingpolicy.firstRun": false,
// Disable first-run welcome page
"startup.homepage_welcome_url": "about:blank",
"startup.homepage_welcome_url.additional": "",
// Detected !
// // Disable images to speed up load
// "permissions.default.image": 2,
// Limit content processes to 1
"dom.ipc.processCount": 1
}
// proxy.url format => http://<host>:<port>
if (proxy && proxy.url) {
log.debug(`Using proxy: ${proxy.url}`)
const [host, portStr] = proxy.url.replace(/.+:\/\//g, '').split(':');
const port = parseInt(portStr);
if (!host || !portStr || !port) {
throw new Error("Proxy configuration is invalid! Use the format: protocol://ip:port")
}
const proxyPrefs = {
"network.proxy.type": 1,
"network.proxy.share_proxy_settings": true
}
if (proxy.url.indexOf("socks") != -1) {
// SOCKSv4 & SOCKSv5
Object.assign(proxyPrefs, {
"network.proxy.socks": host,
"network.proxy.socks_port": port,
"network.proxy.socks_remote_dns": true
});
if (proxy.url.indexOf("socks4") != -1) {
Object.assign(proxyPrefs, {
"network.proxy.socks_version": 4
});
} else {
Object.assign(proxyPrefs, {
"network.proxy.socks_version": 5
});
}
} else {
// HTTP
Object.assign(proxyPrefs, {
"network.proxy.ftp": host,
"network.proxy.ftp_port": port,
"network.proxy.http": host,
"network.proxy.http_port": port,
"network.proxy.ssl": host,
"network.proxy.ssl_port": port
});
}
// merge objects
Object.assign(extraPrefsFirefox, proxyPrefs);
}
return extraPrefsFirefox;
}
export function getUserAgent() {
return webBrowserUserAgent
}
export async function testWebBrowserInstallation(): Promise<void> {
log.info("Testing web browser installation...")
// check user home dir. this dir will be used by Firefox
const homeDir = os.homedir();
fs.accessSync(homeDir, fs.constants.F_OK | fs.constants.R_OK | fs.constants.W_OK | fs.constants.X_OK);
log.debug("FlareSolverr user home directory is OK: " + homeDir)
// test web browser
const testUrl = process.env.TEST_URL || "https://www.google.com";
log.debug("Test URL: " + testUrl)
const session = await create(null, {
oneTimeSession: true
})
const page = await session.browser.newPage()
const pageTimeout = Number(process.env.BROWSER_TIMEOUT) || 40000
await page.goto(testUrl, {waitUntil: 'domcontentloaded', timeout: pageTimeout})
webBrowserUserAgent = await page.evaluate(() => navigator.userAgent)
// replace Linux ARM user-agent because it's detected
if (["arm", "aarch64"].some(arch => webBrowserUserAgent.toLocaleLowerCase().includes('linux ' + arch))) {
webBrowserUserAgent = webBrowserUserAgent.replace(/linux \w+;/i, 'Linux x86_64;')
}
log.info("FlareSolverr User-Agent: " + webBrowserUserAgent)
await page.close()
await destroy(session.sessionId)
log.info("Test successful")
}
export async function create(session: string, options: SessionCreateOptions): Promise<SessionsCacheItem> {
log.debug('Creating new session...')
const sessionId = session || UUIDv1()
// NOTE: cookies can't be set in the session, you need to open the page first
const puppeteerOptions: any = {
product: 'firefox',
headless: process.env.HEADLESS !== 'false',
timeout: Number(process.env.BROWSER_TIMEOUT) || 40000
}
puppeteerOptions.extraPrefsFirefox = buildExtraPrefsFirefox(options.proxy)
// if we are running inside executable binary, change browser path
if (typeof (process as any).pkg !== 'undefined') {
const exe = process.platform === "win32" ? 'firefox.exe' : 'firefox';
puppeteerOptions.executablePath = path.join(path.dirname(process.execPath), 'firefox', exe)
}
log.debug('Launching web browser...')
let browser: Browser = await puppeteer.launch(puppeteerOptions)
if (!browser) {
throw Error(`Failed to launch web browser.`)
}
sessionCache[sessionId] = {
sessionId: sessionId,
browser: browser
}
return sessionCache[sessionId]
}
export function list(): string[] {
return Object.keys(sessionCache)
}
export async function destroy(id: string): Promise<boolean>{
if (id && sessionCache.hasOwnProperty(id)) {
const { browser } = sessionCache[id]
if (browser) {
await browser.close()
delete sessionCache[id]
return true
}
}
return false
}
export function get(id: string): SessionsCacheItem {
return sessionCache[id]
}

223
src/services/solver.ts Normal file
View File

@@ -0,0 +1,223 @@
import {Page, HTTPResponse} from 'puppeteer'
const Timeout = require('await-timeout');
import log from './log'
import {SessionCreateOptions, SessionsCacheItem} from "./sessions";
import {V1Request} from "../controllers/v1";
import cloudflareProvider from '../providers/cloudflare';
const sessions = require('./sessions')
export interface ChallengeResolutionResultT {
url: string
status: number,
headers?: Record<string, string>,
response: string,
cookies: object[]
userAgent: string
}
export interface ChallengeResolutionT {
status?: string
message: string
result: ChallengeResolutionResultT
}
async function resolveChallengeWithTimeout(params: V1Request, session: SessionsCacheItem) {
const timer = new Timeout();
try {
const promise = resolveChallenge(params, session);
return await Promise.race([
promise,
timer.set(params.maxTimeout, `Maximum timeout reached. maxTimeout=${params.maxTimeout} (ms)`)
]);
} finally {
timer.clear();
}
}
async function resolveChallenge(params: V1Request, session: SessionsCacheItem): Promise<ChallengeResolutionT | void> {
try {
let status = 'ok'
let message = ''
const page: Page = await session.browser.newPage()
// the Puppeter timeout should be half the maxTimeout because we reload the page and wait for challenge
// the user can set a really high maxTimeout if he wants to
await page.setDefaultNavigationTimeout(params.maxTimeout / 2)
// the user-agent is changed just for linux arm build
await page.setUserAgent(sessions.getUserAgent())
// set the proxy
if (params.proxy) {
log.debug(`Using proxy: ${params.proxy.url}`);
// todo: credentials are not working
// if (params.proxy.username) {
// await page.authenticate({
// username: params.proxy.username,
// password: params.proxy.password
// });
// }
}
// go to the page
log.debug(`Navigating to... ${params.url}`)
let response: HTTPResponse = await gotoPage(params, page);
// set cookies
if (params.cookies) {
for (const cookie of params.cookies) {
// the other fields in the cookie can cause issues
await page.setCookie({
"name": cookie.name,
"value": cookie.value
})
}
// reload the page
response = await gotoPage(params, page);
}
// log html in debug mode
log.html(await page.content())
// detect protection services and solve challenges
try {
response = await cloudflareProvider(params.url, page, response);
// is response is ok
// reload the page to be sure we get the real page
log.debug("Reloading the page")
try {
response = await gotoPage(params, page);
} catch (e) {
log.warn("Page not reloaded (do not report!): Cause: " + e.toString())
}
} catch (e) {
status = "error";
message = "Cloudflare " + e.toString();
}
const payload: ChallengeResolutionT = {
status,
message,
result: {
url: page.url(),
status: response.status(),
headers: response.headers(),
response: null,
cookies: await page.cookies(),
userAgent: sessions.getUserAgent()
}
}
if (params.returnOnlyCookies) {
payload.result.headers = null;
payload.result.userAgent = null;
} else {
payload.result.response = await page.content()
}
// make sure the page is closed because if it isn't and error will be thrown
// when a user uses a temporary session, the browser make be quit before
// the page is properly closed.
await page.close()
return payload
} catch (e) {
log.error("Unexpected error: " + e);
throw e;
}
}
async function gotoPage(params: V1Request, page: Page): Promise<HTTPResponse> {
let pageTimeout = params.maxTimeout / 3;
let response: HTTPResponse
try {
response = await page.goto(params.url, {waitUntil: 'domcontentloaded', timeout: pageTimeout});
} catch (e) {
// retry
response = await page.goto(params.url, {waitUntil: 'domcontentloaded', timeout: pageTimeout});
}
if (params.method == 'POST') {
// post hack
await page.setContent(
`
<!DOCTYPE html>
<html>
<body>
<script>
function parseQuery(queryString) {
var query = {};
var pairs = (queryString[0] === '?' ? queryString.substr(1) : queryString).split('&');
for (var i = 0; i < pairs.length; i++) {
var pair = pairs[i].split('=');
query[decodeURIComponent(pair[0])] = decodeURIComponent(pair[1] || '');
}
return query;
}
const form = document.createElement('form');
form.method = 'POST';
form.action = '${params.url}';
const params = parseQuery('${params.postData}');
for (const key in params) {
if (params.hasOwnProperty(key)) {
const hiddenField = document.createElement('input');
hiddenField.type = 'hidden';
hiddenField.name = key;
hiddenField.value = params[key];
form.appendChild(hiddenField);
}
}
document.body.appendChild(form);
form.submit();
</script>
</body>
</html>
`
);
await page.waitForTimeout(2000)
try {
await page.waitForNavigation({waitUntil: 'domcontentloaded', timeout: 2000})
} catch (e) {}
}
return response
}
export async function browserRequest(params: V1Request): Promise<ChallengeResolutionT> {
const oneTimeSession = params.session === undefined;
const options: SessionCreateOptions = {
oneTimeSession: oneTimeSession,
cookies: params.cookies,
maxTimeout: params.maxTimeout,
proxy: params.proxy
}
const session: SessionsCacheItem = oneTimeSession
? await sessions.create(null, options)
: sessions.get(params.session)
if (!session) {
throw Error('This session does not exist. Use \'list_sessions\' to see all the existing sessions.')
}
try {
return await resolveChallengeWithTimeout(params, session)
} catch (error) {
throw Error("Unable to process browser request. " + error)
} finally {
if (oneTimeSession) {
await sessions.destroy(session.sessionId)
}
}
}

View File

@@ -1,140 +0,0 @@
import * as os from 'os'
import * as path from 'path'
import * as fs from 'fs'
import puppeteer from 'puppeteer-extra'
import { LaunchOptions, Headers, SetCookie } from 'puppeteer'
import log from './log'
import { deleteFolderRecursive, sleep, removeEmptyFields } from './utils'
import * as Puppeteer from "puppeteer-extra/dist/puppeteer";
interface SessionPageDefaults {
headers?: Headers
userAgent?: string
}
export interface SessionsCacheItem {
browser: Puppeteer.Browser
userDataDir?: string
defaults: SessionPageDefaults
}
interface SessionsCache {
[key: string]: SessionsCacheItem
}
interface SessionCreateOptions {
oneTimeSession?: boolean
userAgent?: string
cookies?: SetCookie[]
headers?: Headers,
maxTimeout?: number
proxy?: any
}
const sessionCache: SessionsCache = {}
// setting "user-agent-override" evasion is not working for us because it can't be changed
// in each request. we set the user-agent in the browser args instead
puppeteer.use(require('puppeteer-extra-plugin-stealth')())
function userDataDirFromId(id: string): string {
return path.join(os.tmpdir(), `/puppeteer_chrome_profile_${id}`)
}
function prepareBrowserProfile(id: string): string {
// TODO: maybe pass SessionCreateOptions for loading later?
const userDataDir = userDataDirFromId(id)
if (!fs.existsSync(userDataDir)) {
fs.mkdirSync(userDataDir, { recursive: true })
}
return userDataDir
}
export default {
create: async (id: string, { cookies, oneTimeSession, userAgent, headers, maxTimeout, proxy }: SessionCreateOptions): Promise<SessionsCacheItem> => {
let args = ['--no-sandbox', '--disable-setuid-sandbox'];
if (proxy && proxy.url) {
args.push(`--proxy-server=${proxy.url}`);
}
const puppeteerOptions: LaunchOptions = {
product: 'chrome',
headless: process.env.HEADLESS !== 'false',
args
}
if (!oneTimeSession) {
log.debug('Creating userDataDir for session.')
puppeteerOptions.userDataDir = prepareBrowserProfile(id)
}
log.debug('Launching headless browser...')
// TODO: maybe access env variable?
// TODO: sometimes browser instances are created and not connected to correctly.
// how do we handle/quit those instances inside Docker?
let launchTries = 3
let browser: Puppeteer.Browser;
while (0 <= launchTries--) {
try {
browser = await puppeteer.launch(puppeteerOptions)
break
} catch (e) {
if (e.message !== 'Failed to launch the browser process!')
throw e
log.warn('Failed to open browser, trying again...')
}
}
if (!browser) { throw Error(`Failed to launch browser 3 times in a row.`) }
if (cookies) {
const page = await browser.newPage()
await page.setCookie(...cookies)
}
sessionCache[id] = {
browser: browser,
userDataDir: puppeteerOptions.userDataDir,
defaults: removeEmptyFields({
userAgent,
headers,
maxTimeout
})
}
return sessionCache[id]
},
list: (): string[] => Object.keys(sessionCache),
// TODO: create a sessions.close that doesn't rm the userDataDir
destroy: async (id: string): Promise<boolean> => {
const { browser, userDataDir } = sessionCache[id]
if (browser) {
await browser.close()
delete sessionCache[id]
if (userDataDir) {
const userDataDirPath = userDataDirFromId(id)
try {
// for some reason this keeps an error from being thrown in Windows, figures
await sleep(5000)
deleteFolderRecursive(userDataDirPath)
} catch (e) {
console.error(e)
throw Error(`Error deleting browser session folder. ${e.message}`)
}
}
return true
}
return false
},
get: (id: string): SessionsCacheItem | false => sessionCache[id] && sessionCache[id] || false
}

625
src/tests/app.test.ts Normal file
View File

@@ -0,0 +1,625 @@
// noinspection DuplicatedCode
import {Response} from "superagent";
import {V1ResponseBase, V1ResponseSession, V1ResponseSessions, V1ResponseSolution} from "../controllers/v1"
const request = require("supertest");
const app = require("../app");
const sessions = require('../services/sessions');
const version: string = 'v' + require('../../package.json').version
const proxyUrl = "http://127.0.0.1:8888"
const proxySocksUrl = "socks5://127.0.0.1:1080"
const googleUrl = "https://www.google.com";
const postUrl = "https://ptsv2.com/t/qv4j3-1634496523";
const cfUrl = "https://pirateiro.com/torrents/?search=harry";
const cfCaptchaUrl = "https://idope.se"
const cfBlockedUrl = "https://www.torrentmafya.org/table.php"
const ddgUrl = "https://www.erai-raws.info/feed/?type=magnet";
const ccfUrl = "https://www.muziekfabriek.org";
beforeAll(async () => {
// Init session
await sessions.testWebBrowserInstallation();
});
afterEach(async () => {
// Clean sessions
const sessionList = sessions.list();
for (const session of sessionList) {
await sessions.destroy(session);
}
});
describe("Test '/' path", () => {
test("GET method should return OK ", async () => {
const response: Response = await request(app).get("/");
expect(response.statusCode).toBe(200);
expect(response.body.msg).toBe("FlareSolverr is ready!");
expect(response.body.version).toBe(version);
expect(response.body.userAgent).toContain("Firefox/")
});
test("POST method should fail", async () => {
const response: Response = await request(app).post("/");
expect(response.statusCode).toBe(404);
expect(response.body.error).toBe("Unknown resource or HTTP verb");
});
});
describe("Test '/health' path", () => {
test("GET method should return OK", async () => {
const response: Response = await request(app).get("/health");
expect(response.statusCode).toBe(200);
expect(response.body.status).toBe("ok");
});
});
describe("Test '/wrong' path", () => {
test("GET method should fail", async () => {
const response: Response = await request(app).get("/wrong");
expect(response.statusCode).toBe(404);
expect(response.body.error).toBe("Unknown resource or HTTP verb");
});
});
describe("Test '/v1' path", () => {
test("Cmd 'request.bad' should fail", async () => {
const payload = {
"cmd": "request.bad",
"url": googleUrl
}
const response: Response = await request(app).post("/v1").send(payload);
expect(response.statusCode).toBe(500);
const apiResponse: V1ResponseBase = response.body;
expect(apiResponse.status).toBe("error");
expect(apiResponse.message).toBe("Error: The command 'request.bad' is invalid.");
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
expect(apiResponse.endTimestamp).toBeGreaterThanOrEqual(apiResponse.startTimestamp);
expect(apiResponse.version).toBe(version);
});
test("Cmd 'request.get' should return OK with no Cloudflare", async () => {
const payload = {
"cmd": "request.get",
"url": googleUrl
}
const response: Response = await request(app).post("/v1").send(payload);
expect(response.statusCode).toBe(200);
const apiResponse: V1ResponseSolution = response.body;
expect(apiResponse.status).toBe("ok");
expect(apiResponse.message).toBe("");
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
expect(apiResponse.version).toBe(version);
const solution = apiResponse.solution;
expect(solution.url).toContain(googleUrl)
expect(solution.status).toBe(200);
expect(Object.keys(solution.headers).length).toBeGreaterThan(0)
expect(solution.response).toContain("<!DOCTYPE html>")
expect(Object.keys(solution.cookies).length).toBeGreaterThan(0)
expect(solution.userAgent).toContain("Firefox/")
});
test("Cmd 'request.get' should return OK with Cloudflare JS", async () => {
const payload = {
"cmd": "request.get",
"url": cfUrl
}
const response: Response = await request(app).post("/v1").send(payload);
expect(response.statusCode).toBe(200);
const apiResponse: V1ResponseSolution = response.body;
expect(apiResponse.status).toBe("ok");
expect(apiResponse.message).toBe("");
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
expect(apiResponse.version).toBe(version);
const solution = apiResponse.solution;
expect(solution.url).toContain(cfUrl)
expect(solution.status).toBe(200);
expect(Object.keys(solution.headers).length).toBeGreaterThan(0)
expect(solution.response).toContain("<!DOCTYPE html>")
expect(Object.keys(solution.cookies).length).toBeGreaterThan(0)
expect(solution.userAgent).toContain("Firefox/")
const cfCookie: string = (solution.cookies as any[]).filter(function(cookie) {
return cookie.name == "cf_clearance";
})[0].value
expect(cfCookie.length).toBeGreaterThan(30)
});
test("Cmd 'request.get' should return fail with Cloudflare CAPTCHA", async () => {
const payload = {
"cmd": "request.get",
"url": cfCaptchaUrl
}
const response: Response = await request(app).post("/v1").send(payload);
expect(response.statusCode).toBe(200);
const apiResponse: V1ResponseSolution = response.body;
expect(apiResponse.status).toBe("error");
expect(apiResponse.message).toBe("Cloudflare Error: FlareSolverr can not resolve CAPTCHA challenges. Since the captcha doesn't always appear, you may have better luck with the next request.");
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
expect(apiResponse.version).toBe(version);
// solution is filled but not useful
expect(apiResponse.solution.url).toContain(cfCaptchaUrl)
});
test("Cmd 'request.post' should return fail with Cloudflare Blocked", async () => {
const payload = {
"cmd": "request.post",
"url": cfBlockedUrl,
"postData": "test1=test2"
}
const response: Response = await request(app).post("/v1").send(payload);
expect(response.statusCode).toBe(200);
const apiResponse: V1ResponseSolution = response.body;
expect(apiResponse.status).toBe("error");
expect(apiResponse.message).toBe("Cloudflare Error: Cloudflare has blocked this request. Probably your IP is banned for this site, check in your web browser.");
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
expect(apiResponse.version).toBe(version);
// solution is filled but not useful
expect(apiResponse.solution.url).toContain(cfBlockedUrl)
});
test("Cmd 'request.get' should return OK with DDoS-GUARD JS", async () => {
const payload = {
"cmd": "request.get",
"url": ddgUrl
}
const response: Response = await request(app).post("/v1").send(payload);
expect(response.statusCode).toBe(200);
const apiResponse: V1ResponseSolution = response.body;
expect(apiResponse.status).toBe("ok");
expect(apiResponse.message).toBe("");
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
expect(apiResponse.version).toBe(version);
const solution = apiResponse.solution;
expect(solution.url).toContain(ddgUrl)
expect(solution.status).toBe(200);
expect(Object.keys(solution.headers).length).toBeGreaterThan(0)
expect(solution.response).toContain("<rss version")
expect(Object.keys(solution.cookies).length).toBeGreaterThan(0)
expect(solution.userAgent).toContain("Firefox/")
const cfCookie: string = (solution.cookies as any[]).filter(function(cookie) {
return cookie.name == "__ddg1";
})[0].value
expect(cfCookie.length).toBeGreaterThan(10)
});
test("Cmd 'request.get' should return OK with Custom CloudFlare JS", async () => {
const payload = {
"cmd": "request.get",
"url": ccfUrl
}
const response: Response = await request(app).post("/v1").send(payload);
expect(response.statusCode).toBe(200);
const apiResponse: V1ResponseSolution = response.body;
expect(apiResponse.status).toBe("ok");
expect(apiResponse.message).toBe("");
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
expect(apiResponse.version).toBe(version);
const solution = apiResponse.solution;
expect(solution.url).toContain(ccfUrl)
expect(solution.status).toBe(200);
expect(Object.keys(solution.headers).length).toBeGreaterThan(0)
expect(solution.response).toContain("<html><head>")
expect(Object.keys(solution.cookies).length).toBeGreaterThan(0)
expect(solution.userAgent).toContain("Firefox/")
const cfCookie: string = (solution.cookies as any[]).filter(function(cookie) {
return cookie.name == "ct_anti_ddos_key";
})[0].value
expect(cfCookie.length).toBeGreaterThan(10)
});
test("Cmd 'request.get' should return OK with 'cookies' param", async () => {
const payload = {
"cmd": "request.get",
"url": googleUrl,
"cookies": [
{
"name": "testcookie1",
"value": "testvalue1"
},
{
"name": "testcookie2",
"value": "testvalue2"
}
]
}
const response: Response = await request(app).post("/v1").send(payload);
expect(response.statusCode).toBe(200);
const apiResponse: V1ResponseSolution = response.body;
expect(apiResponse.status).toBe("ok");
const solution = apiResponse.solution;
expect(solution.url).toContain(googleUrl)
expect(Object.keys(solution.cookies).length).toBeGreaterThan(1)
const cookie1: string = (solution.cookies as any[]).filter(function(cookie) {
return cookie.name == "testcookie1";
})[0].value
expect(cookie1).toBe("testvalue1")
const cookie2: string = (solution.cookies as any[]).filter(function(cookie) {
return cookie.name == "testcookie2";
})[0].value
expect(cookie2).toBe("testvalue2")
});
test("Cmd 'request.get' should return OK with 'returnOnlyCookies' param", async () => {
const payload = {
"cmd": "request.get",
"url": googleUrl,
"returnOnlyCookies": true
}
const response: Response = await request(app).post("/v1").send(payload);
expect(response.statusCode).toBe(200);
const apiResponse: V1ResponseSolution = response.body;
const solution = apiResponse.solution;
expect(solution.url).toContain(googleUrl)
expect(solution.status).toBe(200);
expect(solution.headers).toBe(null)
expect(solution.response).toBe(null)
expect(Object.keys(solution.cookies).length).toBeGreaterThan(0)
expect(solution.userAgent).toBe(null)
});
test("Cmd 'request.get' should return OK with HTTP 'proxy' param", async () => {
/*
To configure TinyProxy in local:
* sudo vim /etc/tinyproxy/tinyproxy.conf
* edit => LogFile "/tmp/tinyproxy.log"
* edit => Syslog Off
* sudo tinyproxy -d
* sudo tail -f /tmp/tinyproxy.log
*/
const payload = {
"cmd": "request.get",
"url": googleUrl,
"proxy": {
"url": proxyUrl
}
}
const response: Response = await request(app).post("/v1").send(payload);
expect(response.statusCode).toBe(200);
const apiResponse: V1ResponseSolution = response.body;
expect(apiResponse.status).toBe("ok");
const solution = apiResponse.solution;
expect(solution.url).toContain(googleUrl)
expect(solution.status).toBe(200);
});
// todo: credentials are not working
test.skip("Cmd 'request.get' should return OK with HTTP 'proxy' param with credentials", async () => {
/*
To configure TinyProxy in local:
* sudo vim /etc/tinyproxy/tinyproxy.conf
* edit => LogFile "/tmp/tinyproxy.log"
* edit => Syslog Off
* add => BasicAuth testuser testpass
* sudo tinyproxy -d
* sudo tail -f /tmp/tinyproxy.log
*/
const payload = {
"cmd": "request.get",
"url": googleUrl,
"proxy": {
"url": proxyUrl,
"username": "testuser",
"password": "testpass"
}
}
const response: Response = await request(app).post("/v1").send(payload);
expect(response.statusCode).toBe(200);
const apiResponse: V1ResponseSolution = response.body;
expect(apiResponse.status).toBe("ok");
const solution = apiResponse.solution;
expect(solution.url).toContain(googleUrl)
expect(solution.status).toContain(200)
});
test("Cmd 'request.get' should return OK with SOCKSv5 'proxy' param", async () => {
/*
To configure Dante in local:
* https://linuxhint.com/set-up-a-socks5-proxy-on-ubuntu-with-dante/
* sudo vim /etc/sockd.conf
* sudo systemctl restart sockd.service
* curl --socks5 socks5://127.0.0.1:1080 https://www.google.com
*/
const payload = {
"cmd": "request.get",
"url": googleUrl,
"proxy": {
"url": proxySocksUrl
}
}
const response: Response = await request(app).post("/v1").send(payload);
expect(response.statusCode).toBe(200);
const apiResponse: V1ResponseSolution = response.body;
expect(apiResponse.status).toBe("ok");
const solution = apiResponse.solution;
expect(solution.url).toContain(googleUrl)
expect(solution.status).toBe(200);
});
test("Cmd 'request.get' should fail with wrong 'proxy' param", async () => {
const payload = {
"cmd": "request.get",
"url": googleUrl,
"proxy": {
"url": "http://127.0.0.1:43210"
}
}
const response: Response = await request(app).post("/v1").send(payload);
expect(response.statusCode).toBe(500);
const apiResponse: V1ResponseSolution = response.body;
expect(apiResponse.status).toBe("error");
expect(apiResponse.message).toBe("Error: Unable to process browser request. Error: NS_ERROR_PROXY_CONNECTION_REFUSED at https://www.google.com");
});
test("Cmd 'request.get' should return fail with timeout", async () => {
const payload = {
"cmd": "request.get",
"url": googleUrl,
"maxTimeout": 10
}
const response: Response = await request(app).post("/v1").send(payload);
expect(response.statusCode).toBe(500);
const apiResponse: V1ResponseBase = response.body;
expect(apiResponse.status).toBe("error");
expect(apiResponse.message).toBe("Error: Unable to process browser request. Error: Maximum timeout reached. maxTimeout=10 (ms)");
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
expect(apiResponse.version).toBe(version);
});
test("Cmd 'request.get' should return fail with bad domain", async () => {
const payload = {
"cmd": "request.get",
"url": "https://www.google.combad"
}
const response: Response = await request(app).post("/v1").send(payload);
expect(response.statusCode).toBe(500);
const apiResponse: V1ResponseBase = response.body;
expect(apiResponse.status).toBe("error");
expect(apiResponse.message).toBe("Error: Unable to process browser request. Error: NS_ERROR_UNKNOWN_HOST at https://www.google.combad");
});
test("Cmd 'request.get' should accept deprecated params", async () => {
const payload = {
"cmd": "request.get",
"url": googleUrl,
"userAgent": "Test User-Agent" // was removed in v2, not used
}
const response: Response = await request(app).post("/v1").send(payload);
expect(response.statusCode).toBe(200);
const apiResponse: V1ResponseSolution = response.body;
expect(apiResponse.status).toBe("ok");
const solution = apiResponse.solution;
expect(solution.url).toContain(googleUrl)
expect(solution.status).toBe(200);
expect(solution.userAgent).toContain("Firefox/")
});
test("Cmd 'request.post' should return OK with no Cloudflare", async () => {
const payload = {
"cmd": "request.post",
"url": postUrl + '/post',
"postData": "param1=value1&param2=value2"
}
const response: Response = await request(app).post("/v1").send(payload);
expect(response.statusCode).toBe(200);
const apiResponse: V1ResponseSolution = response.body;
expect(apiResponse.status).toBe("ok");
expect(apiResponse.message).toBe("");
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
expect(apiResponse.version).toBe(version);
const solution = apiResponse.solution;
expect(solution.url).toContain(postUrl)
expect(solution.status).toBe(200);
expect(Object.keys(solution.headers).length).toBeGreaterThan(0)
expect(solution.response).toContain(" I hope you have a lovely day!")
expect(Object.keys(solution.cookies).length).toBe(0)
expect(solution.userAgent).toContain("Firefox/")
// check that we sent the date
const payload2 = {
"cmd": "request.get",
"url": postUrl
}
const response2: Response = await request(app).post("/v1").send(payload2);
expect(response2.statusCode).toBe(200);
const apiResponse2: V1ResponseSolution = response2.body;
expect(apiResponse2.status).toBe("ok");
const solution2 = apiResponse2.solution;
expect(solution2.status).toBe(200);
expect(solution2.response).toContain(new Date().toISOString().split(':')[0].replace('T', ' '))
});
test("Cmd 'request.post' should fail without 'postData' param", async () => {
const payload = {
"cmd": "request.post",
"url": googleUrl
}
const response: Response = await request(app).post("/v1").send(payload);
expect(response.statusCode).toBe(500);
const apiResponse: V1ResponseBase = response.body;
expect(apiResponse.status).toBe("error");
expect(apiResponse.message).toBe("Error: Must send param \"postBody\" when sending a POST request.");
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
expect(apiResponse.endTimestamp).toBeGreaterThanOrEqual(apiResponse.startTimestamp);
expect(apiResponse.version).toBe(version);
});
test("Cmd 'sessions.create' should return OK", async () => {
const payload = {
"cmd": "sessions.create"
}
const response: Response = await request(app).post("/v1").send(payload);
expect(response.statusCode).toBe(200);
const apiResponse: V1ResponseSession = response.body;
expect(apiResponse.status).toBe("ok");
expect(apiResponse.message).toBe("Session created successfully.");
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
expect(apiResponse.version).toBe(version);
expect(apiResponse.session.length).toBe(36);
});
test("Cmd 'sessions.create' should return OK with session", async () => {
const payload = {
"cmd": "sessions.create",
"session": "2bc6bb20-2f56-11ec-9543-test"
}
const response: Response = await request(app).post("/v1").send(payload);
expect(response.statusCode).toBe(200);
const apiResponse: V1ResponseSession = response.body;
expect(apiResponse.status).toBe("ok");
expect(apiResponse.message).toBe("Session created successfully.");
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
expect(apiResponse.version).toBe(version);
expect(apiResponse.session).toBe("2bc6bb20-2f56-11ec-9543-test");
});
test("Cmd 'sessions.list' should return OK", async () => {
// create one session for testing
const payload0 = {
"cmd": "sessions.create"
}
const response0: Response = await request(app).post("/v1").send(payload0);
expect(response0.statusCode).toBe(200);
const payload = {
"cmd": "sessions.list"
}
const response: Response = await request(app).post("/v1").send(payload);
expect(response.statusCode).toBe(200);
const apiResponse: V1ResponseSessions = response.body;
expect(apiResponse.status).toBe("ok");
expect(apiResponse.message).toBe("");
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
expect(apiResponse.endTimestamp).toBeGreaterThanOrEqual(apiResponse.startTimestamp);
expect(apiResponse.version).toBe(version);
expect(apiResponse.sessions.length).toBeGreaterThan(0)
});
test("Cmd 'sessions.destroy' should return OK", async () => {
// create one session for testing
const payload0 = {
"cmd": "sessions.create"
}
const response0: Response = await request(app).post("/v1").send(payload0);
expect(response0.statusCode).toBe(200);
const apiResponse0: V1ResponseSession = response0.body;
const sessionId0 = apiResponse0.session
const payload = {
"cmd": "sessions.destroy",
"session": sessionId0
}
const response: Response = await request(app).post("/v1").send(payload);
expect(response.statusCode).toBe(200);
const apiResponse: V1ResponseBase = response.body;
expect(apiResponse.status).toBe("ok");
expect(apiResponse.message).toBe("The session has been removed.");
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
expect(apiResponse.endTimestamp).toBeGreaterThanOrEqual(apiResponse.startTimestamp);
expect(apiResponse.version).toBe(version);
});
test("Cmd 'sessions.destroy' should fail", async () => {
const payload = {
"cmd": "sessions.destroy",
"session": "bad-session"
}
const response: Response = await request(app).post("/v1").send(payload);
expect(response.statusCode).toBe(500);
const apiResponse: V1ResponseBase = response.body;
expect(apiResponse.status).toBe("error");
expect(apiResponse.message).toBe("Error: This session does not exist.");
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
expect(apiResponse.version).toBe(version);
});
test("Cmd 'request.get' should use session", async () => {
// create one session for testing
const payload0 = {
"cmd": "sessions.create"
}
const response0: Response = await request(app).post("/v1").send(payload0);
expect(response0.statusCode).toBe(200);
const apiResponse0: V1ResponseSession = response0.body;
const sessionId0 = apiResponse0.session
// first request should solve the challenge
const payload = {
"cmd": "request.get",
"url": cfUrl,
"session": sessionId0
}
const response: Response = await request(app).post("/v1").send(payload);
expect(response.statusCode).toBe(200);
const apiResponse: V1ResponseSolution = response.body;
expect(apiResponse.status).toBe("ok");
const cfCookie: string = (apiResponse.solution.cookies as any[]).filter(function(cookie) {
return cookie.name == "cf_clearance";
})[0].value
expect(cfCookie.length).toBeGreaterThan(30)
// second request should have the same cookie
const response2: Response = await request(app).post("/v1").send(payload);
expect(response2.statusCode).toBe(200);
const apiResponse2: V1ResponseSolution = response2.body;
expect(apiResponse2.status).toBe("ok");
const cfCookie2: string = (apiResponse2.solution.cookies as any[]).filter(function(cookie) {
return cookie.name == "cf_clearance";
})[0].value
expect(cfCookie2.length).toBeGreaterThan(30)
expect(cfCookie2).toBe(cfCookie)
});
});

View File

@@ -1,9 +0,0 @@
import { IncomingMessage, ServerResponse } from 'http';
export interface RequestContext {
req: IncomingMessage
res: ServerResponse
startTimestamp: number
errorResponse: (msg: string) => void,
successResponse: (msg: string, extendedProperties?: object) => void
}

View File

@@ -1,31 +0,0 @@
import * as fs from 'fs'
import * as Path from 'path'
import { promisify } from 'util'
export const sleep = promisify(setTimeout)
// recursive fs.rmdir needs node version 12:
// https://github.com/ngosang/FlareSolverr/issues/5#issuecomment-655572712
export function deleteFolderRecursive(path: string) {
if (fs.existsSync(path)) {
fs.readdirSync(path).forEach((file) => {
const curPath = Path.join(path, file)
if (fs.lstatSync(curPath).isDirectory()) { // recurse
deleteFolderRecursive(curPath)
} else { // delete file
fs.unlinkSync(curPath)
}
})
fs.rmdirSync(path)
}
}
export const removeEmptyFields = (o: Record<string, any>): typeof o => {
const r: typeof o = {}
for (const k in o) {
if (o[k] !== undefined) {
r[k] = o[k]
}
}
return r
}