mirror of
https://github.com/FlareSolverr/FlareSolverr.git
synced 2025-12-05 17:18:19 +01:00
Compare commits
94 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8e518d7267 | ||
|
|
3005ba3629 | ||
|
|
176c69d1e8 | ||
|
|
7a1cf7dd80 | ||
|
|
456dfc222e | ||
|
|
23fde49f2b | ||
|
|
78daf24bc3 | ||
|
|
47c83ded58 | ||
|
|
35890cade4 | ||
|
|
753e8e1be8 | ||
|
|
a6628d0cda | ||
|
|
a79a5f2b42 | ||
|
|
1e463bb3e2 | ||
|
|
02204a84d3 | ||
|
|
95d178b37a | ||
|
|
c4f890f9a1 | ||
|
|
d16b982bb9 | ||
|
|
075b53ee24 | ||
|
|
356b893c18 | ||
|
|
a841d67745 | ||
|
|
2408a75a70 | ||
|
|
77a87c79fd | ||
|
|
cfd158462f | ||
|
|
ccfe21c15a | ||
|
|
a5b3e08e1f | ||
|
|
a0e897067a | ||
|
|
744de4d158 | ||
|
|
0459f2642d | ||
|
|
ca3f84f458 | ||
|
|
5dd563e003 | ||
|
|
78c10d6b24 | ||
|
|
3de2e44bfd | ||
|
|
7738f7a360 | ||
|
|
1b01caaa78 | ||
|
|
447c8f67a1 | ||
|
|
9dae74bc28 | ||
|
|
4199db5a41 | ||
|
|
2a4fae37c0 | ||
|
|
232ddca512 | ||
|
|
8572fab781 | ||
|
|
fdb3eae051 | ||
|
|
6dd8206a10 | ||
|
|
c4e4d28c8d | ||
|
|
543ce89eb6 | ||
|
|
0f30e17ef1 | ||
|
|
24f1b4ec6f | ||
|
|
f3b30268c3 | ||
|
|
be4354c68d | ||
|
|
5242cf3359 | ||
|
|
c6677f4d84 | ||
|
|
805a34c9d6 | ||
|
|
2f9fe05a76 | ||
|
|
8961d67a29 | ||
|
|
5da5156851 | ||
|
|
05f8ef95d9 | ||
|
|
10f8b83e83 | ||
|
|
6cf948d0e1 | ||
|
|
dcdc70273f | ||
|
|
e2dc39ee4e | ||
|
|
340638ca54 | ||
|
|
05abe69df6 | ||
|
|
e596906c19 | ||
|
|
8a1b0ea05c | ||
|
|
916fbf2c9d | ||
|
|
a85e9c2c8c | ||
|
|
71814a86bc | ||
|
|
757ec4358a | ||
|
|
f278c7cf8e | ||
|
|
b4c99d8426 | ||
|
|
8aa7723f45 | ||
|
|
c48d342b9c | ||
|
|
7c361af204 | ||
|
|
6400449344 | ||
|
|
69c4d9edfa | ||
|
|
85428a32f4 | ||
|
|
ea5e461fb4 | ||
|
|
a57510aa0d | ||
|
|
91d1f0cb4a | ||
|
|
7376ef9bc9 | ||
|
|
de9c7bcf76 | ||
|
|
bef9411e1c | ||
|
|
27ad58b2c6 | ||
|
|
d038944089 | ||
|
|
a8bc6f5468 | ||
|
|
39fdde9a74 | ||
|
|
8234cdb516 | ||
|
|
66fe775d27 | ||
|
|
ade05bb7a8 | ||
|
|
5710c08581 | ||
|
|
f1e829fd3a | ||
|
|
dfc4383b50 | ||
|
|
d140e9369d | ||
|
|
6677329842 | ||
|
|
0f40054a73 |
@@ -1,6 +1,7 @@
|
|||||||
node_modules
|
.git/
|
||||||
npm-debug.log
|
.github/
|
||||||
Dockerfile
|
.idea/
|
||||||
.dockerignore
|
bin/
|
||||||
.git
|
dist/
|
||||||
.gitignore
|
node_modules/
|
||||||
|
resources/
|
||||||
|
|||||||
25
.github/ISSUE_TEMPLATE.md
vendored
25
.github/ISSUE_TEMPLATE.md
vendored
@@ -1,31 +1,26 @@
|
|||||||
**Please use the search bar** at the top of the page and make sure you are not creating an already submitted issue.
|
**Please use the search bar** at the top of the page and make sure you are not creating an already submitted issue.
|
||||||
Check closed issues as well, because your issue may have already been fixed.
|
Check closed issues as well, because your issue may have already been fixed.
|
||||||
|
|
||||||
### Instruction on how to enable debug and html trace
|
### How to enable debug and html traces
|
||||||
|
|
||||||
[Follow the instructions from this wiki page](https://github.com/FlareSolverr/FlareSolverr/wiki/How-to-enable-debug-and-html-trace)
|
[Follow the instructions from this wiki page](https://github.com/FlareSolverr/FlareSolverr/wiki/How-to-enable-debug-and-html-trace)
|
||||||
|
|
||||||
### Environment
|
### Environment
|
||||||
|
|
||||||
**FlareSolverr Version**:
|
* **FlareSolverr version**:
|
||||||
|
* **Last working FlareSolverr version**:
|
||||||
**Docker**: [yes/no]
|
* **Operating system**:
|
||||||
|
* **Are you using Docker**: [yes/no]
|
||||||
**OS**:
|
* **FlareSolverr User-Agent (see log traces or / endpoint)**:
|
||||||
|
* **Are you using a proxy or VPN?** [yes/no]
|
||||||
**Last Working FlareSolverr Version**:
|
* **Are you using Captcha Solver:** [yes/no]
|
||||||
|
* **If using captcha solver, which one:**
|
||||||
**Are you using a proxy or VPN?** [yes/no]
|
* **URL to test this issue:**
|
||||||
|
|
||||||
**Using Captcha Solver:** [yse/no]
|
|
||||||
|
|
||||||
**If using captcha solver, which one:**
|
|
||||||
|
|
||||||
### Description
|
### Description
|
||||||
|
|
||||||
[List steps to reproduce the error and details on what happens and what you expected to happen]
|
[List steps to reproduce the error and details on what happens and what you expected to happen]
|
||||||
|
|
||||||
|
|
||||||
### Logged Error Messages
|
### Logged Error Messages
|
||||||
|
|
||||||
[Place any relevant error messages you noticed from the logs here.]
|
[Place any relevant error messages you noticed from the logs here.]
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
name: publish
|
name: release-docker
|
||||||
|
|
||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
@@ -24,19 +24,19 @@ jobs:
|
|||||||
tag-sha: false
|
tag-sha: false
|
||||||
-
|
-
|
||||||
name: Set up QEMU
|
name: Set up QEMU
|
||||||
uses: docker/setup-qemu-action@v1
|
uses: docker/setup-qemu-action@v1.0.1
|
||||||
-
|
-
|
||||||
name: Set up Docker Buildx
|
name: Set up Docker Buildx
|
||||||
uses: docker/setup-buildx-action@v1
|
uses: docker/setup-buildx-action@v1
|
||||||
-
|
-
|
||||||
name: Login to DockerHub
|
name: Login to DockerHub
|
||||||
uses: docker/login-action@v1
|
uses: docker/login-action@v1
|
||||||
with:
|
with:
|
||||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||||
-
|
-
|
||||||
name: Login to GitHub Container Registry
|
name: Login to GitHub Container Registry
|
||||||
uses: docker/login-action@v1
|
uses: docker/login-action@v1
|
||||||
with:
|
with:
|
||||||
registry: ghcr.io
|
registry: ghcr.io
|
||||||
username: ${{ github.repository_owner }}
|
username: ${{ github.repository_owner }}
|
||||||
39
.github/workflows/release.yml
vendored
39
.github/workflows/release.yml
vendored
@@ -7,19 +7,35 @@ on:
|
|||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
build:
|
build:
|
||||||
name: Create Release
|
name: Create release
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout code
|
- name: Checkout code
|
||||||
uses: actions/checkout@v2
|
uses: actions/checkout@v2
|
||||||
|
with:
|
||||||
|
fetch-depth: 0 # get all commits, branches and tags (required for the changelog)
|
||||||
|
|
||||||
- name: Build Changelog
|
- name: Setup Node
|
||||||
id: github_release
|
uses: actions/setup-node@v2
|
||||||
uses: mikepenz/release-changelog-builder-action@main
|
with:
|
||||||
env:
|
node-version: '14'
|
||||||
GITHUB_TOKEN: ${{ secrets.GH_PAT }}
|
|
||||||
|
|
||||||
- name: Create Release
|
- name: Build artifacts
|
||||||
|
run: |
|
||||||
|
npm install
|
||||||
|
npm run build
|
||||||
|
npm run package
|
||||||
|
|
||||||
|
- name: Build changelog
|
||||||
|
id: github_changelog
|
||||||
|
run: |
|
||||||
|
changelog=$(git log $(git tag | tail -2 | head -1)..HEAD --no-merges --oneline)
|
||||||
|
changelog="${changelog//'%'/'%25'}"
|
||||||
|
changelog="${changelog//$'\n'/'%0A'}"
|
||||||
|
changelog="${changelog//$'\r'/'%0D'}"
|
||||||
|
echo "##[set-output name=changelog;]${changelog}"
|
||||||
|
|
||||||
|
- name: Create release
|
||||||
id: create_release
|
id: create_release
|
||||||
uses: actions/create-release@v1
|
uses: actions/create-release@v1
|
||||||
env:
|
env:
|
||||||
@@ -27,6 +43,13 @@ jobs:
|
|||||||
with:
|
with:
|
||||||
tag_name: ${{ github.ref }}
|
tag_name: ${{ github.ref }}
|
||||||
release_name: ${{ github.ref }}
|
release_name: ${{ github.ref }}
|
||||||
body: ${{ steps.github_release.outputs.changelog }}
|
body: ${{ steps.github_changelog.outputs.changelog }}
|
||||||
draft: false
|
draft: false
|
||||||
prerelease: false
|
prerelease: false
|
||||||
|
|
||||||
|
- name: Upload release artifacts
|
||||||
|
uses: alexellis/upload-assets@0.2.2
|
||||||
|
env:
|
||||||
|
GITHUB_TOKEN: ${{ secrets.GH_PAT }}
|
||||||
|
with:
|
||||||
|
asset_paths: '["./bin/*.zip"]'
|
||||||
|
|||||||
3
.gitignore
vendored
3
.gitignore
vendored
@@ -121,3 +121,6 @@ dist
|
|||||||
|
|
||||||
# Project Development
|
# Project Development
|
||||||
testing/
|
testing/
|
||||||
|
|
||||||
|
# Binaries
|
||||||
|
bin/
|
||||||
|
|||||||
24
Dockerfile
24
Dockerfile
@@ -1,13 +1,14 @@
|
|||||||
FROM --platform=${TARGETPLATFORM:-linux/amd64} node:15.2.1-alpine3.11
|
FROM --platform=${TARGETPLATFORM:-linux/amd64} node:16-alpine3.14
|
||||||
|
|
||||||
# Print build information
|
# Print build information
|
||||||
ARG TARGETPLATFORM
|
ARG TARGETPLATFORM
|
||||||
ARG BUILDPLATFORM
|
ARG BUILDPLATFORM
|
||||||
RUN printf "I am running on ${BUILDPLATFORM:-linux/amd64}, building for ${TARGETPLATFORM:-linux/amd64}\n$(uname -a)\n"
|
RUN printf "I am running on ${BUILDPLATFORM:-linux/amd64}, building for ${TARGETPLATFORM:-linux/amd64}\n$(uname -a)\n"
|
||||||
|
|
||||||
# Install Chromium, dumb-init and remove all locales but en-US
|
# Install the web browser (package firefox-esr is available too)
|
||||||
RUN apk add --no-cache chromium dumb-init && \
|
RUN apk update && \
|
||||||
find /usr/lib/chromium/locales -type f ! -name 'en-US.*' -delete
|
apk add --no-cache firefox dumb-init && \
|
||||||
|
rm -Rf /var/cache
|
||||||
|
|
||||||
# Copy FlareSolverr code
|
# Copy FlareSolverr code
|
||||||
USER node
|
USER node
|
||||||
@@ -16,15 +17,18 @@ WORKDIR /home/node/flaresolverr
|
|||||||
COPY --chown=node:node package.json package-lock.json tsconfig.json ./
|
COPY --chown=node:node package.json package-lock.json tsconfig.json ./
|
||||||
COPY --chown=node:node src ./src/
|
COPY --chown=node:node src ./src/
|
||||||
|
|
||||||
# Install package. Skip installing Chrome, we will use the installed package.
|
# Install package. Skip installing the browser, we will use the installed package.
|
||||||
ENV PUPPETEER_PRODUCT=chrome \
|
ENV PUPPETEER_PRODUCT=firefox \
|
||||||
PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
|
PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
|
||||||
PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
|
PUPPETEER_EXECUTABLE_PATH=/usr/bin/firefox
|
||||||
RUN npm install && \
|
RUN npm install && \
|
||||||
npm run build && \
|
npm run build && \
|
||||||
rm -rf src tsconfig.json && \
|
npm prune --production && \
|
||||||
npm prune --production
|
rm -rf /home/node/.npm
|
||||||
|
|
||||||
EXPOSE 8191
|
EXPOSE 8191
|
||||||
ENTRYPOINT ["/usr/bin/dumb-init", "--"]
|
ENTRYPOINT ["/usr/bin/dumb-init", "--"]
|
||||||
CMD ["npm", "start"]
|
CMD ["node", "./dist/server.js"]
|
||||||
|
|
||||||
|
# docker build -t flaresolverr:custom .
|
||||||
|
# docker run -p 8191:8191 -e LOG_LEVEL=debug flaresolverr:custom
|
||||||
|
|||||||
125
README.md
125
README.md
@@ -1,19 +1,21 @@
|
|||||||
# FlareSolverr
|
# FlareSolverr
|
||||||
|
|
||||||
[](https://github.com/FlareSolverr/FlareSolverr/issues)
|
[](https://github.com/FlareSolverr/FlareSolverr/releases)
|
||||||
[](https://github.com/FlareSolverr/FlareSolverr/pulls)
|
[](https://hub.docker.com/r/flaresolverr/flaresolverr/)
|
||||||
[](https://hub.docker.com/r/flaresolverr/flaresolverr/)
|
[](https://github.com/FlareSolverr/FlareSolverr/issues)
|
||||||
|
[](https://github.com/FlareSolverr/FlareSolverr/pulls)
|
||||||
|
[](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=X5NJLLX5GLTV6&source=url)
|
||||||
|
[](https://www.buymeacoffee.com/ngosang)
|
||||||
|
[](https://en.cryptobadges.io/donate/13Hcv77AdnFWEUZ9qUpoPBttQsUT7q9TTh)
|
||||||
|
|
||||||
FlareSolverr is a proxy server to bypass Cloudflare protection
|
FlareSolverr is a proxy server to bypass Cloudflare protection.
|
||||||
|
|
||||||
:warning: This project is in beta state. Some things may not work and the API can change at any time.
|
|
||||||
|
|
||||||
## How it works
|
## How it works
|
||||||
|
|
||||||
FlareSolverr starts a proxy server and it waits for user requests in an idle state using few resources.
|
FlareSolverr starts a proxy server and it waits for user requests in an idle state using few resources.
|
||||||
When some request arrives, it uses [puppeteer](https://github.com/puppeteer/puppeteer) with the
|
When some request arrives, it uses [puppeteer](https://github.com/puppeteer/puppeteer) with the
|
||||||
[stealth plugin](https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin-stealth)
|
[stealth plugin](https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin-stealth)
|
||||||
to create a headless browser (Chrome). It opens the URL with user parameters and waits until the Cloudflare challenge
|
to create a headless browser (Firefox). It opens the URL with user parameters and waits until the Cloudflare challenge
|
||||||
is solved (or timeout). The HTML code and the cookies are sent back to the user, and those cookies can be used to
|
is solved (or timeout). The HTML code and the cookies are sent back to the user, and those cookies can be used to
|
||||||
bypass Cloudflare using other HTTP clients.
|
bypass Cloudflare using other HTTP clients.
|
||||||
|
|
||||||
@@ -48,19 +50,33 @@ If you prefer the `docker cli` execute the following command.
|
|||||||
```bash
|
```bash
|
||||||
docker run -d \
|
docker run -d \
|
||||||
--name=flaresolverr \
|
--name=flaresolverr \
|
||||||
|
-p 8191:8191 \
|
||||||
-e LOG_LEVEL=info \
|
-e LOG_LEVEL=info \
|
||||||
--restart unless-stopped \
|
--restart unless-stopped \
|
||||||
ghcr.io/flaresolverr/flaresolverr:latest
|
ghcr.io/flaresolverr/flaresolverr:latest
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Precompiled binaries
|
||||||
|
|
||||||
|
This is the recommended way for Windows users.
|
||||||
|
* Download the [FlareSolverr zip](https://github.com/FlareSolverr/FlareSolverr/releases) from the release's assets. It is available for Windows and Linux.
|
||||||
|
* Extract the zip file. FlareSolverr executable and firefox folder must be in the same directory.
|
||||||
|
* Execute FlareSolverr binary. In the environment variables section you can find how to change the configuration.
|
||||||
|
|
||||||
### From source code
|
### From source code
|
||||||
|
|
||||||
This is the recommended way for Windows / MacOS users and for developers.
|
This is the recommended way for macOS users and for developers.
|
||||||
* Install [NodeJS](https://nodejs.org/)
|
* Install [NodeJS](https://nodejs.org/).
|
||||||
* Clone this repository and open a shell in that path
|
* Clone this repository and open a shell in that path.
|
||||||
* Run `npm install` command to install FlareSolverr dependencies
|
* Run `export PUPPETEER_PRODUCT=firefox` (Linux/macOS) or `set PUPPETEER_PRODUCT=firefox` (Windows).
|
||||||
* Run `npm run build` command to compile TypeScript code
|
* Run `npm install` command to install FlareSolverr dependencies.
|
||||||
* Run `npm start` command to start FlareSolverr
|
* Run `node node_modules/puppeteer/install.js` to install Firefox.
|
||||||
|
* Run `npm run build` command to compile TypeScript code.
|
||||||
|
* Run `npm start` command to start FlareSolverr.
|
||||||
|
|
||||||
|
### Systemd service
|
||||||
|
|
||||||
|
We provide an example Systemd unit file `flaresolverr.service` as reference. You have to modify the file to suit your needs: paths, user and environment variables.
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
@@ -71,11 +87,7 @@ curl -L -X POST 'http://localhost:8191/v1' \
|
|||||||
--data-raw '{
|
--data-raw '{
|
||||||
"cmd": "request.get",
|
"cmd": "request.get",
|
||||||
"url":"http://www.google.com/",
|
"url":"http://www.google.com/",
|
||||||
"userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleW...",
|
"maxTimeout": 60000
|
||||||
"maxTimeout": 60000,
|
|
||||||
"headers": {
|
|
||||||
"X-Test": "Testing 123..."
|
|
||||||
}
|
|
||||||
}'
|
}'
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -92,7 +104,6 @@ This also speeds up the requests since it won't have to launch a new browser ins
|
|||||||
Parameter | Notes
|
Parameter | Notes
|
||||||
|--|--|
|
|--|--|
|
||||||
session | Optional. The session ID that you want to be assigned to the instance. If isn't set a random UUID will be assigned.
|
session | Optional. The session ID that you want to be assigned to the instance. If isn't set a random UUID will be assigned.
|
||||||
userAgent | Optional. Will be used by the headless browser.
|
|
||||||
|
|
||||||
#### + `sessions.list`
|
#### + `sessions.list`
|
||||||
|
|
||||||
@@ -127,9 +138,12 @@ Parameter | Notes
|
|||||||
|--|--|
|
|--|--|
|
||||||
url | Mandatory
|
url | Mandatory
|
||||||
session | Optional. Will send the request from and existing browser instance. If one is not sent it will create a temporary instance that will be destroyed immediately after the request is completed.
|
session | Optional. Will send the request from and existing browser instance. If one is not sent it will create a temporary instance that will be destroyed immediately after the request is completed.
|
||||||
headers | Optional. To specify user headers.
|
maxTimeout | Optional, default value 60000. Max timeout to solve the challenge in milliseconds.
|
||||||
maxTimeout | Optional. Max timeout to solve the challenge
|
cookies | Optional. Will be used by the headless browser. Follow [this](https://github.com/puppeteer/puppeteer/blob/v3.3.0/docs/api.md#pagesetcookiecookies) format.
|
||||||
cookies | Optional. Will be used by the headless browser. Follow [this](https://github.com/puppeteer/puppeteer/blob/v3.3.0/docs/api.md#pagesetcookiecookies) format
|
returnOnlyCookies | Optional, default false. Only returns the cookies. Response data, headers and other parts of the response are removed.
|
||||||
|
proxy | Optional, default disabled. Eg: `"proxy": {"url": "http://127.0.0.1:8888"}`. Authorization (username/password) is not supported.
|
||||||
|
|
||||||
|
:warning: If you want to use Cloudflare clearance cookie in your scripts, make sure you use the FlareSolverr User-Agent too. If they don't match you will see the challenge.
|
||||||
|
|
||||||
Example response from running the `curl` above:
|
Example response from running the `curl` above:
|
||||||
|
|
||||||
@@ -196,66 +210,37 @@ This is the same as `request.get` but it takes one more param:
|
|||||||
|
|
||||||
Parameter | Notes
|
Parameter | Notes
|
||||||
|--|--|
|
|--|--|
|
||||||
postData | Must be a string. If you want to POST a form, don't forget to set the `Content-Type` header to `application/x-www-form-urlencoded` or the server might not understand your request.
|
postData | Must be a string with `application/x-www-form-urlencoded`. Eg: `postData": "a=b&c=d"`
|
||||||
|
|
||||||
### Download small files
|
|
||||||
|
|
||||||
If you need to access an image/pdf or small file, you should pass the `download` parameter to `request.get` setting it
|
|
||||||
to `true`. Rather than access the html and return text it will return the buffer **base64** encoded which you will be
|
|
||||||
able to decode and save the image/pdf.
|
|
||||||
|
|
||||||
This method isn't recommended for videos or anything larger. As that should be streamed back to the client and at the
|
|
||||||
moment there is nothing setup to do so. If this is something you need feel free to create an issue and/or submit a PR.
|
|
||||||
|
|
||||||
## Environment variables
|
## Environment variables
|
||||||
|
|
||||||
To set the environment vars in Linux run `export LOG_LEVEL=debug` and then start FlareSolverr in the same shell.
|
|
||||||
|
|
||||||
Name | Default | Notes
|
Name | Default | Notes
|
||||||
|--|--|--|
|
|--|--|--|
|
||||||
LOG_LEVEL | info | Used to change the verbosity of the logging.
|
LOG_LEVEL | info | Verbosity of the logging. Use `LOG_LEVEL=debug` for more information.
|
||||||
LOG_HTML | false | Used for debugging. If `true` all HTML that passes through the proxy will be logged to the console in `debug` level.
|
LOG_HTML | false | Only for debugging. If `true` all HTML that passes through the proxy will be logged to the console in `debug` level.
|
||||||
PORT | 8191 | Change this if you already have a process running on port `8191`.
|
CAPTCHA_SOLVER | none | Captcha solving method. It is used when a captcha is encountered. See the Captcha Solvers section.
|
||||||
HOST | 0.0.0.0 | This shouldn't need to be messed with but if you insist, it's here!
|
TZ | UTC | Timezone used in the logs and the web browser. Example: `TZ=Europe/London`.
|
||||||
CAPTCHA_SOLVER | None | This is used to select which captcha solving method it used when a captcha is encountered.
|
HEADLESS | true | Only for debugging. To run the web browser in headless mode or visible.
|
||||||
HEADLESS | true | This is used to debug the browser by not running it in headless mode.
|
BROWSER_TIMEOUT | 30000 | If you are experiencing errors/timeouts because your system is slow, you can try to increase this value. Remember to increase the `maxTimeout` parameter too.
|
||||||
|
PORT | 8191 | Listening port. You don't need to change this if you are running on Docker.
|
||||||
|
HOST | 0.0.0.0 | Listening interface. You don't need to change this if you are running on Docker.
|
||||||
|
|
||||||
|
Environment variables are set differently depending on the operating system. Some examples:
|
||||||
|
* Docker: Take a look at the Docker section in this document. Environment variables can be set in the `docker-compose.yml` file or in the Docker CLI command.
|
||||||
|
* Linux: Run `export LOG_LEVEL=debug` and then start FlareSolverr in the same shell.
|
||||||
|
* Windows: Open `cmd.exe`, run `set LOG_LEVEL=debug` and then start FlareSolverr in the same shell.
|
||||||
|
|
||||||
## Captcha Solvers
|
## Captcha Solvers
|
||||||
|
|
||||||
Sometimes CF not only gives mathematical computations and browser tests, sometimes they also require the user to solve
|
:warning: At this time none of the captcha solvers work. You can check the status in the open issues. Any help is welcome.
|
||||||
a captcha. If this is the case, FlareSolverr will return the captcha page. But that's not very helpful to you is it?
|
|
||||||
|
Sometimes CloudFlare not only gives mathematical computations and browser tests, sometimes they also require the user to
|
||||||
|
solve a captcha.
|
||||||
|
If this is the case, FlareSolverr will return the error `Captcha detected but no automatic solver is configured.`
|
||||||
|
|
||||||
FlareSolverr can be customized to solve the captchas automatically by setting the environment variable `CAPTCHA_SOLVER`
|
FlareSolverr can be customized to solve the captchas automatically by setting the environment variable `CAPTCHA_SOLVER`
|
||||||
to the file name of one of the adapters inside the [/captcha](src/captcha) directory.
|
to the file name of one of the adapters inside the [/captcha](src/captcha) directory.
|
||||||
|
|
||||||
### hcaptcha-solver
|
|
||||||
|
|
||||||
This method makes use of the [hcaptcha-solver](https://github.com/JimmyLaurent/hcaptcha-solver) project which attempts
|
|
||||||
to solve hCaptcha by randomly selecting images.
|
|
||||||
|
|
||||||
To use this solver you must first install it and then set it as the `CAPTCHA_SOLVER`.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
npm i hcaptcha-solver
|
|
||||||
CAPTCHA_SOLVER=hcaptcha-solver
|
|
||||||
```
|
|
||||||
|
|
||||||
### CaptchaHarvester
|
|
||||||
|
|
||||||
This method makes use of the [CaptchaHarvester](https://github.com/NoahCardoza/CaptchaHarvester) project which allows
|
|
||||||
users to collect thier own tokens from ReCaptcha V2/V3 and hCaptcha for free.
|
|
||||||
|
|
||||||
To use this method you must set these ENV variables:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
CAPTCHA_SOLVER=harvester
|
|
||||||
HARVESTER_ENDPOINT=https://127.0.0.1:5000/token
|
|
||||||
```
|
|
||||||
|
|
||||||
**Note**: above I set `HARVESTER_ENDPOINT` to the default configuration of the captcha harvester's server, but that
|
|
||||||
could change if you customize the command line flags. Simply put, `HARVESTER_ENDPOINT` should be set to the URI of the
|
|
||||||
route that returns a token in plain text when called.
|
|
||||||
|
|
||||||
## Related projects
|
## Related projects
|
||||||
|
|
||||||
* C# implementation => https://github.com/FlareSolverr/FlareSolverrSharp
|
* C# implementation => https://github.com/FlareSolverr/FlareSolverrSharp
|
||||||
|
|||||||
111
build-binaries.js
Normal file
111
build-binaries.js
Normal file
@@ -0,0 +1,111 @@
|
|||||||
|
const fs = require('fs')
|
||||||
|
const path = require('path')
|
||||||
|
const { execSync } = require('child_process')
|
||||||
|
const archiver = require('archiver')
|
||||||
|
const https = require('https')
|
||||||
|
const puppeteer = require('puppeteer')
|
||||||
|
const version = 'v' + require('./package.json').version;
|
||||||
|
|
||||||
|
function getFirefoxNightlyVersion() {
|
||||||
|
const firefoxVersions = 'https://product-details.mozilla.org/1.0/firefox_versions.json';
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
let data = '';
|
||||||
|
https
|
||||||
|
.get(firefoxVersions, (r) => {
|
||||||
|
if (r.statusCode >= 400)
|
||||||
|
return reject(new Error(`Got status code ${r.statusCode}`));
|
||||||
|
r.on('data', (chunk) => {
|
||||||
|
data += chunk;
|
||||||
|
});
|
||||||
|
r.on('end', () => {
|
||||||
|
try {
|
||||||
|
const versions = JSON.parse(data);
|
||||||
|
return resolve(versions.FIREFOX_NIGHTLY);
|
||||||
|
} catch {
|
||||||
|
return reject(new Error('Firefox version not found'));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
})
|
||||||
|
.on('error', reject);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
(async () => {
|
||||||
|
const builds = [
|
||||||
|
{
|
||||||
|
platform: 'linux',
|
||||||
|
firefoxFolder: 'firefox',
|
||||||
|
fsExec: 'flaresolverr-linux',
|
||||||
|
fsZipExec: 'flaresolverr',
|
||||||
|
fsZipName: 'linux-x64',
|
||||||
|
fsLicenseName: 'LICENSE'
|
||||||
|
},
|
||||||
|
{
|
||||||
|
platform: 'win64',
|
||||||
|
firefoxFolder: 'firefox',
|
||||||
|
fsExec: 'flaresolverr-win.exe',
|
||||||
|
fsZipExec: 'flaresolverr.exe',
|
||||||
|
fsZipName: 'windows-x64',
|
||||||
|
fsLicenseName: 'LICENSE.txt'
|
||||||
|
}
|
||||||
|
// todo: this has to be build in macOS (hdiutil is required). changes required in sessions.ts too
|
||||||
|
// {
|
||||||
|
// platform: 'mac',
|
||||||
|
// firefoxFolder: 'firefox',
|
||||||
|
// fsExec: 'flaresolverr-macos',
|
||||||
|
// fsZipExec: 'flaresolverr',
|
||||||
|
// fsZipName: 'macos',
|
||||||
|
// fsLicenseName: 'LICENSE'
|
||||||
|
// }
|
||||||
|
]
|
||||||
|
|
||||||
|
// generate executables
|
||||||
|
console.log('Generating executables...')
|
||||||
|
if (fs.existsSync('bin')) {
|
||||||
|
fs.rmSync('bin', { recursive: true })
|
||||||
|
}
|
||||||
|
execSync('./node_modules/.bin/pkg -t node14-win-x64,node14-linux-x64 --out-path bin .')
|
||||||
|
// execSync('./node_modules/.bin/pkg -t node14-win-x64,node14-mac-x64,node14-linux-x64 --out-path bin .')
|
||||||
|
|
||||||
|
// get firefox revision
|
||||||
|
const revision = await getFirefoxNightlyVersion();
|
||||||
|
|
||||||
|
// download firefox and zip together
|
||||||
|
for (const os of builds) {
|
||||||
|
console.log('Building ' + os.fsZipName + ' artifact')
|
||||||
|
|
||||||
|
// download firefox
|
||||||
|
console.log(`Downloading firefox ${revision} for ${os.platform} ...`)
|
||||||
|
const f = puppeteer.createBrowserFetcher({
|
||||||
|
product: 'firefox',
|
||||||
|
platform: os.platform,
|
||||||
|
path: path.join(__dirname, 'bin', 'puppeteer')
|
||||||
|
})
|
||||||
|
await f.download(revision)
|
||||||
|
|
||||||
|
// compress in zip
|
||||||
|
console.log('Compressing zip file...')
|
||||||
|
const zipName = 'bin/flaresolverr-' + version + '-' + os.fsZipName + '.zip'
|
||||||
|
const output = fs.createWriteStream(zipName)
|
||||||
|
const archive = archiver('zip')
|
||||||
|
|
||||||
|
output.on('close', function () {
|
||||||
|
console.log('File ' + zipName + ' created. Size: ' + archive.pointer() + ' bytes')
|
||||||
|
})
|
||||||
|
|
||||||
|
archive.on('error', function (err) {
|
||||||
|
throw err
|
||||||
|
})
|
||||||
|
|
||||||
|
archive.pipe(output)
|
||||||
|
|
||||||
|
archive.file('LICENSE', { name: 'flaresolverr/' + os.fsLicenseName })
|
||||||
|
archive.file('bin/' + os.fsExec, { name: 'flaresolverr/' + os.fsZipExec })
|
||||||
|
archive.directory('bin/puppeteer/' + os.platform + '-' + revision + '/' + os.firefoxFolder, 'flaresolverr/firefox')
|
||||||
|
if (os.platform === 'linux') {
|
||||||
|
archive.file('flaresolverr.service', { name: 'flaresolverr/flaresolverr.service' })
|
||||||
|
}
|
||||||
|
|
||||||
|
await archive.finalize()
|
||||||
|
}
|
||||||
|
})()
|
||||||
@@ -6,13 +6,10 @@ services:
|
|||||||
image: ghcr.io/flaresolverr/flaresolverr:latest
|
image: ghcr.io/flaresolverr/flaresolverr:latest
|
||||||
container_name: flaresolverr
|
container_name: flaresolverr
|
||||||
environment:
|
environment:
|
||||||
# Used to change the verbosity of the logging
|
- LOG_LEVEL=${LOG_LEVEL:-info}
|
||||||
- LOG_LEVEL=info
|
- LOG_HTML=${LOG_HTML:-false}
|
||||||
# Enables hcaptcha-solver => https://github.com/JimmyLaurent/hcaptcha-solver
|
- CAPTCHA_SOLVER=${CAPTCHA_SOLVER:-none}
|
||||||
#- CAPTCHA_SOLVER=hcaptcha-solver
|
- TZ=Europe/London
|
||||||
# Enables CaptchaHarvester => https://github.com/NoahCardoza/CaptchaHarvester
|
|
||||||
#- CAPTCHA_SOLVER=harvester
|
|
||||||
#- HARVESTER_ENDPOINT=https://127.0.0.1:5000/token
|
|
||||||
ports:
|
ports:
|
||||||
- 8191:8191
|
- "${PORT:-8191}:8191"
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
|||||||
19
flaresolverr.service
Normal file
19
flaresolverr.service
Normal file
@@ -0,0 +1,19 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=FlareSolverr
|
||||||
|
After=network.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
SyslogIdentifier=flaresolverr
|
||||||
|
Restart=always
|
||||||
|
RestartSec=5
|
||||||
|
Type=simple
|
||||||
|
User=flaresolverr
|
||||||
|
Group=flaresolverr
|
||||||
|
Environment="LOG_LEVEL=info"
|
||||||
|
Environment="CAPTCHA_SOLVER=none"
|
||||||
|
WorkingDirectory=/opt/flaresolverr
|
||||||
|
ExecStart=/opt/flaresolverr/flaresolverr
|
||||||
|
TimeoutStopSec=30
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
12
jest.config.js
Normal file
12
jest.config.js
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
module.exports = {
|
||||||
|
// A list of paths to directories that Jest should use to search for files in
|
||||||
|
roots: [
|
||||||
|
"./src/"
|
||||||
|
],
|
||||||
|
// Compile Typescript
|
||||||
|
transform: {
|
||||||
|
'^.+\\.(ts|tsx)$': 'ts-jest'
|
||||||
|
},
|
||||||
|
// Default value for FlareSolverr maxTimeout is 60000
|
||||||
|
testTimeout: 70000
|
||||||
|
}
|
||||||
14994
package-lock.json
generated
14994
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
52
package.json
52
package.json
@@ -1,48 +1,46 @@
|
|||||||
{
|
{
|
||||||
"name": "flaresolverr",
|
"name": "flaresolverr",
|
||||||
"version": "1.2.1",
|
"version": "2.0.2",
|
||||||
"description": "Proxy server to bypass Cloudflare protection.",
|
"description": "Proxy server to bypass Cloudflare protection.",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"start": "node ./dist/index.js",
|
"start": "node ./dist/server.js",
|
||||||
"build": "tsc",
|
"build": "tsc",
|
||||||
"dev": "nodemon -e ts --exec ts-node src/index.ts"
|
"dev": "nodemon -e ts --exec ts-node src/server.ts",
|
||||||
|
"package": "node build-binaries.js",
|
||||||
|
"test": "jest --runInBand"
|
||||||
},
|
},
|
||||||
"author": "Diego Heras (ngosang)",
|
"author": "Diego Heras (ngosang)",
|
||||||
"contributors": [
|
|
||||||
{
|
|
||||||
"name": "Noah Cardoza",
|
|
||||||
"url": "https://github.com/NoahCardoza/CloudProxy.git"
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"repository": {
|
"repository": {
|
||||||
"type": "git",
|
"type": "git",
|
||||||
"url": "https://github.com/ngosang/FlareSolverr"
|
"url": "https://github.com/ngosang/FlareSolverr"
|
||||||
},
|
},
|
||||||
|
"bin": {
|
||||||
|
"flaresolverr": "dist/server.js"
|
||||||
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"await-timeout": "^1.1.1",
|
"await-timeout": "^1.1.1",
|
||||||
|
"body-parser": "^1.19.0",
|
||||||
"console-log-level": "^1.4.1",
|
"console-log-level": "^1.4.1",
|
||||||
"got": "^11.5.1",
|
"express": "^4.17.1",
|
||||||
"hcaptcha-solver": "^1.0.2",
|
|
||||||
"puppeteer": "^3.3.0",
|
"puppeteer": "^3.3.0",
|
||||||
"puppeteer-extra": "^3.1.15",
|
"uuid": "^8.3.2"
|
||||||
"puppeteer-extra-plugin-stealth": "^2.6.5",
|
|
||||||
"uuid": "^8.2.0"
|
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@types/await-timeout": "^0.3.1",
|
"@types/await-timeout": "^0.3.1",
|
||||||
"@types/node": "^14.0.23",
|
"@types/body-parser": "^1.19.1",
|
||||||
"@types/puppeteer": "^3.0.1",
|
"@types/express": "^4.17.13",
|
||||||
"@types/uuid": "^8.0.0",
|
"@types/jest": "^27.0.2",
|
||||||
"eslint": "^7.5.0",
|
"@types/node": "^14.17.27",
|
||||||
"eslint-config-airbnb-base": "^14.2.0",
|
"@types/puppeteer": "^3.0.6",
|
||||||
"eslint-config-standard": "^14.1.1",
|
"@types/supertest": "^2.0.11",
|
||||||
"eslint-plugin-import": "^2.22.0",
|
"@types/uuid": "^8.3.1",
|
||||||
"eslint-plugin-node": "^11.1.0",
|
"archiver": "^5.3.0",
|
||||||
"eslint-plugin-promise": "^4.2.1",
|
"nodemon": "^2.0.13",
|
||||||
"eslint-plugin-standard": "^4.0.1",
|
"pkg": "^5.3.3",
|
||||||
"nodemon": "^2.0.4",
|
"supertest": "^6.1.6",
|
||||||
"ts-node": "^8.10.2",
|
"ts-jest": "^27.0.7",
|
||||||
"typescript": "^3.9.7"
|
"ts-node": "^10.3.0",
|
||||||
|
"typescript": "^4.4.4"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
BIN
resources/flaresolverr_logo.png
Normal file
BIN
resources/flaresolverr_logo.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 9.1 KiB |
180
resources/flaresolverr_logo.svg
Normal file
180
resources/flaresolverr_logo.svg
Normal file
@@ -0,0 +1,180 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||||
|
<!-- Created with Inkscape (http://www.inkscape.org/) -->
|
||||||
|
|
||||||
|
<svg
|
||||||
|
xmlns:dc="http://purl.org/dc/elements/1.1/"
|
||||||
|
xmlns:cc="http://creativecommons.org/ns#"
|
||||||
|
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
|
||||||
|
xmlns:svg="http://www.w3.org/2000/svg"
|
||||||
|
xmlns="http://www.w3.org/2000/svg"
|
||||||
|
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
|
||||||
|
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
|
||||||
|
width="256"
|
||||||
|
height="256"
|
||||||
|
viewBox="0 0 256 256"
|
||||||
|
id="svg2"
|
||||||
|
version="1.1"
|
||||||
|
inkscape:version="0.91 r13725"
|
||||||
|
sodipodi:docname="flaresolverr_logo.svg"
|
||||||
|
inkscape:export-filename="C:\Users\Diego\Desktop\flaresolverr_logo.png"
|
||||||
|
inkscape:export-xdpi="90"
|
||||||
|
inkscape:export-ydpi="90">
|
||||||
|
<defs
|
||||||
|
id="defs4" />
|
||||||
|
<sodipodi:namedview
|
||||||
|
id="base"
|
||||||
|
pagecolor="#ffffff"
|
||||||
|
bordercolor="#666666"
|
||||||
|
borderopacity="1.0"
|
||||||
|
inkscape:pageopacity="0.0"
|
||||||
|
inkscape:pageshadow="2"
|
||||||
|
inkscape:zoom="0.64"
|
||||||
|
inkscape:cx="-88.263072"
|
||||||
|
inkscape:cy="-93.571587"
|
||||||
|
inkscape:document-units="px"
|
||||||
|
inkscape:current-layer="layer1"
|
||||||
|
showgrid="false"
|
||||||
|
inkscape:window-width="2560"
|
||||||
|
inkscape:window-height="1377"
|
||||||
|
inkscape:window-x="-8"
|
||||||
|
inkscape:window-y="-8"
|
||||||
|
inkscape:window-maximized="1"
|
||||||
|
units="px"
|
||||||
|
showborder="true" />
|
||||||
|
<metadata
|
||||||
|
id="metadata7">
|
||||||
|
<rdf:RDF>
|
||||||
|
<cc:Work
|
||||||
|
rdf:about="">
|
||||||
|
<dc:format>image/svg+xml</dc:format>
|
||||||
|
<dc:type
|
||||||
|
rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
|
||||||
|
<dc:title></dc:title>
|
||||||
|
</cc:Work>
|
||||||
|
</rdf:RDF>
|
||||||
|
</metadata>
|
||||||
|
<g
|
||||||
|
inkscape:label="Capa 1"
|
||||||
|
inkscape:groupmode="layer"
|
||||||
|
id="layer1"
|
||||||
|
transform="translate(0,-796.36219)">
|
||||||
|
<g
|
||||||
|
id="g4177"
|
||||||
|
transform="matrix(0.51436047,0,0,0.59495735,-334.60687,650.43877)">
|
||||||
|
<g
|
||||||
|
id="g4141" />
|
||||||
|
<g
|
||||||
|
id="g4143" />
|
||||||
|
<g
|
||||||
|
id="g4145" />
|
||||||
|
<g
|
||||||
|
id="g4147" />
|
||||||
|
<g
|
||||||
|
id="g4149" />
|
||||||
|
<g
|
||||||
|
id="g4151" />
|
||||||
|
<g
|
||||||
|
id="g4153" />
|
||||||
|
<g
|
||||||
|
id="g4155" />
|
||||||
|
<g
|
||||||
|
id="g4157" />
|
||||||
|
<g
|
||||||
|
id="g4159" />
|
||||||
|
<g
|
||||||
|
id="g4161" />
|
||||||
|
<g
|
||||||
|
id="g4163" />
|
||||||
|
<g
|
||||||
|
id="g4165" />
|
||||||
|
<g
|
||||||
|
id="g4167" />
|
||||||
|
<g
|
||||||
|
id="g4169" />
|
||||||
|
<g
|
||||||
|
id="g4263"
|
||||||
|
transform="matrix(0.94954959,0,0,0.94954959,-111.49858,393.65111)">
|
||||||
|
<g
|
||||||
|
id="g4269" />
|
||||||
|
<g
|
||||||
|
id="g4342"
|
||||||
|
transform="translate(736.24631,-345.97247)">
|
||||||
|
<path
|
||||||
|
style="fill:#9dc6fb"
|
||||||
|
inkscape:connector-curvature="0"
|
||||||
|
d="m 584.32729,454.42324 c -0.995,-51.995 -44.49,-93.257 -96.488,-91.376 -7.616,0.273 -14.792,-3.862 -18.446,-10.55 -22.605,-41.376 -66.519,-69.441 -116.989,-69.441 -51.757,0 -96.596,29.528 -118.647,72.648 -6.423,12.56 -19.224,10.9 -24.689,10.9 -40.126,0 -74.199,25.852 -86.512,61.804 -2.046,5.973 -6.938,10.463 -12.894,12.556 -22.389998,7.87 -38.250998,29.605 -37.275998,54.902 1.163,30.174 26.849,53.631 57.044998,53.631 l 359.817,0 c 52.291,0 96.08,-42.793 95.079,-95.074 z"
|
||||||
|
id="path4285" />
|
||||||
|
<path
|
||||||
|
style="fill:#80b4fb"
|
||||||
|
inkscape:connector-curvature="0"
|
||||||
|
d="m 190.59629,495.86724 c -0.975,-25.298 14.885,-47.033 37.276,-54.902 5.956,-2.094 10.848,-6.584 12.894,-12.556 12.313,-35.952 46.385,-61.804 86.512,-61.804 5.465,0 18.265,1.66 24.688,-10.9 13.005,-25.43 33.94,-46.125 59.541,-58.832 -17.812,-8.834 -37.873,-13.816 -59.103,-13.816 -51.757,0 -96.596,29.528 -118.647,72.648 -6.423,12.56 -19.224,10.9 -24.689,10.9 -40.126,0 -74.199,25.852 -86.512,61.804 -2.046,5.973 -6.938,10.463 -12.894,12.556 -22.389998,7.87 -38.250998,29.605 -37.275998,54.902 1.163,30.174 26.849,53.63 57.044998,53.63 l 118.21,0 c -30.196,0 -55.881,-23.457 -57.045,-53.63 z"
|
||||||
|
id="path4287" />
|
||||||
|
</g>
|
||||||
|
</g>
|
||||||
|
</g>
|
||||||
|
<g
|
||||||
|
id="g4241"
|
||||||
|
transform="matrix(0.1453379,0,0,0.1453379,47.012211,854.83732)"
|
||||||
|
style="fill:#4d4d4d">
|
||||||
|
<g
|
||||||
|
id="g4197"
|
||||||
|
style="fill:#4d4d4d;fill-opacity:1">
|
||||||
|
<path
|
||||||
|
id="path4201"
|
||||||
|
d="m 867.699,356.238 -31.5,-26.6 c -9.699,-8.2 -24,-7.8 -33.199,0.9 l -17.4,16.3 c -14.699,-7.1 -30.299,-12.1 -46.4,-15 l -4.898,-24 c -2.5,-12.4 -14,-21 -26.602,-20 l -41.1,3.5 c -12.6,1.1 -22.5,11.4 -22.9,24.1 l -0.799,24.4 c -15.801,5.7 -30.701,13.5 -44.301,23.3 l -20.799,-13.8 c -10.602,-7 -24.701,-5 -32.9,4.7 l -26.6,31.7 c -8.201,9.7 -7.801,24 0.898,33.2 l 18.201,19.399 c -6.301,14.2 -10.801,29.101 -13.4,44.4 l -26,5.3 c -12.4,2.5 -21,14 -20,26.601 l 3.5,41.1 c 1.1,12.6 11.4,22.5 24.1,22.9 l 28.1,0.899 c 5.102,13.4 11.801,26.101 19.9,38 l -15.699,23.7 c -7,10.6 -5,24.7 4.699,32.9 l 31.5,26.6 c 9.701,8.2 24,7.8 33.201,-0.9 l 20.6,-19.3 c 13.5,6.3 27.699,11 42.299,13.8 l 5.701,28.2 c 2.5,12.4 14,21 26.6,20 l 41.1,-3.5 c 12.6,-1.1 22.5,-11.399 22.9,-24.1 l 0.9,-27.601 c 15,-5.3 29.199,-12.5 42.299,-21.399 l 22.701,15 c 10.6,7 24.699,5 32.9,-4.7 l 26.6,-31.5 c 8.199,-9.7 7.799,-24 -0.9,-33.2 L 872.7,592.138 c 6.701,-14.2 11.602,-29.2 14.4,-44.601 l 25,-5.1 c 12.4,-2.5 21,-14 20,-26.601 l -3.5,-41.1 c -1.1,-12.6 -11.4,-22.5 -24.1,-22.9 l -25.1,-0.8 c -5.201,-14.6 -12.201,-28.399 -20.9,-41.2 l 13.699,-20.6 c 7.201,-10.598 5.201,-24.798 -4.5,-32.998 z M 712.801,593.837 c -44.4,3.801 -83.602,-29.3 -87.301,-73.699 -3.801,-44.4 29.301,-83.601 73.699,-87.301 44.4,-3.8 83.602,29.301 87.301,73.7 3.801,44.401 -29.301,83.601 -73.699,87.3 z"
|
||||||
|
inkscape:connector-curvature="0"
|
||||||
|
style="fill:#4d4d4d;fill-opacity:1" />
|
||||||
|
<path
|
||||||
|
id="path4203"
|
||||||
|
d="m 205,704.438 c -12.6,1.3 -22.3,11.899 -22.4,24.6 l -0.3,25.3 c -0.2,12.7 9.2,23.5 21.8,25.101 l 18.6,2.399 c 3.1,11.301 7.5,22.101 13.2,32.301 l -12,14.8 c -8,9.899 -7.4,24.1 1.5,33.2 l 17.7,18.1 c 8.9,9.1 23.1,10.1 33.2,2.3 l 14.899,-11.5 c 10.5,6.2 21.601,11.101 33.2,14.5 l 2,19.2 c 1.3,12.6 11.9,22.3 24.6,22.4 l 25.301,0.3 c 12.699,0.2 23.5,-9.2 25.1,-21.8 l 2.3,-18.2 c 12.601,-3.101 24.601,-7.8 36,-14 l 14,11.3 c 9.9,8 24.101,7.4 33.201,-1.5 l 18.1,-17.7 c 9.1,-8.899 10.1,-23.1 2.301,-33.2 L 496.6,818.438 c 6.6,-11 11.701,-22.7 15.201,-35 l 16.6,-1.7 c 12.6,-1.3 22.299,-11.9 22.4,-24.6 l 0.299,-25.301 c 0.201,-12.699 -9.199,-23.5 -21.799,-25.1 l -16.201,-2.1 c -3.1,-12.2 -7.699,-24 -13.699,-35 l 10.1,-12.4 c 8,-9.9 7.4,-24.1 -1.5,-33.2 l -17.699,-18.1 c -8.9,-9.101 -23.102,-10.101 -33.201,-2.3 l -12.101,9.3 c -11.399,-6.9 -23.6,-12.2 -36.399,-15.8 L 407,581.437 c -1.3,-12.601 -11.899,-22.3 -24.6,-22.4 l -25.3,-0.3 c -12.7,-0.2 -23.5,9.2 -25.101,21.8 l -2,15.601 c -13.199,3.399 -25.899,8.6 -37.699,15.399 l -12.5,-10.2 c -9.9,-8 -24.101,-7.399 -33.201,1.5 l -18.2,17.801 c -9.1,8.899 -10.1,23.1 -2.3,33.199 l 10.7,13.801 c -6.2,11 -11.1,22.699 -14.3,35 l -17.499,1.8 z m 163.3,-28.601 c 36.3,0.4 65.399,30.301 65,66.601 -0.4,36.3 -30.301,65.399 -66.601,65 -36.3,-0.4 -65.399,-30.3 -65,-66.601 0.401,-36.299 30.301,-65.399 66.601,-65 z"
|
||||||
|
inkscape:connector-curvature="0"
|
||||||
|
style="fill:#4d4d4d;fill-opacity:1" />
|
||||||
|
</g>
|
||||||
|
<g
|
||||||
|
id="g4205"
|
||||||
|
style="fill:#4d4d4d" />
|
||||||
|
<g
|
||||||
|
id="g4207"
|
||||||
|
style="fill:#4d4d4d" />
|
||||||
|
<g
|
||||||
|
id="g4209"
|
||||||
|
style="fill:#4d4d4d" />
|
||||||
|
<g
|
||||||
|
id="g4211"
|
||||||
|
style="fill:#4d4d4d" />
|
||||||
|
<g
|
||||||
|
id="g4213"
|
||||||
|
style="fill:#4d4d4d" />
|
||||||
|
<g
|
||||||
|
id="g4215"
|
||||||
|
style="fill:#4d4d4d" />
|
||||||
|
<g
|
||||||
|
id="g4217"
|
||||||
|
style="fill:#4d4d4d" />
|
||||||
|
<g
|
||||||
|
id="g4219"
|
||||||
|
style="fill:#4d4d4d" />
|
||||||
|
<g
|
||||||
|
id="g4221"
|
||||||
|
style="fill:#4d4d4d" />
|
||||||
|
<g
|
||||||
|
id="g4223"
|
||||||
|
style="fill:#4d4d4d" />
|
||||||
|
<g
|
||||||
|
id="g4225"
|
||||||
|
style="fill:#4d4d4d" />
|
||||||
|
<g
|
||||||
|
id="g4227"
|
||||||
|
style="fill:#4d4d4d" />
|
||||||
|
<g
|
||||||
|
id="g4229"
|
||||||
|
style="fill:#4d4d4d" />
|
||||||
|
<g
|
||||||
|
id="g4231"
|
||||||
|
style="fill:#4d4d4d" />
|
||||||
|
<g
|
||||||
|
id="g4233"
|
||||||
|
style="fill:#4d4d4d" />
|
||||||
|
</g>
|
||||||
|
</g>
|
||||||
|
</svg>
|
||||||
|
After Width: | Height: | Size: 8.0 KiB |
83
src/app.ts
Normal file
83
src/app.ts
Normal file
@@ -0,0 +1,83 @@
|
|||||||
|
import log from './services/log'
|
||||||
|
import {NextFunction, Request, Response} from 'express';
|
||||||
|
import {getUserAgent} from "./services/sessions";
|
||||||
|
import {controllerV1} from "./controllers/v1";
|
||||||
|
|
||||||
|
const express = require('express');
|
||||||
|
const app = express();
|
||||||
|
const bodyParser = require('body-parser');
|
||||||
|
const version: string = 'v' + require('../package.json').version
|
||||||
|
|
||||||
|
// Convert request objects to JSON
|
||||||
|
app.use(bodyParser.json({
|
||||||
|
limit: '50mb',
|
||||||
|
verify(req: Request, res: Response, buf: any) {
|
||||||
|
req.body = buf;
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
|
||||||
|
// Access log
|
||||||
|
app.use(function(req: Request, res: Response, next: NextFunction) {
|
||||||
|
if (req.url != '/health') {
|
||||||
|
// count the request for the log prefix
|
||||||
|
log.incRequests()
|
||||||
|
// build access message
|
||||||
|
let body = "";
|
||||||
|
if (req.method == 'POST' && req.body) {
|
||||||
|
body += " body: "
|
||||||
|
try {
|
||||||
|
body += JSON.stringify(req.body)
|
||||||
|
} catch(e) {
|
||||||
|
body += req.body
|
||||||
|
}
|
||||||
|
}
|
||||||
|
log.info(`Incoming request => ${req.method} ${req.url}${body}`);
|
||||||
|
}
|
||||||
|
next();
|
||||||
|
});
|
||||||
|
|
||||||
|
// *********************************************************************************************************************
|
||||||
|
// Routes
|
||||||
|
|
||||||
|
// Show welcome message
|
||||||
|
app.get("/", ( req: Request, res: Response ) => {
|
||||||
|
res.send({
|
||||||
|
"msg": "FlareSolverr is ready!",
|
||||||
|
"version": version,
|
||||||
|
"userAgent": getUserAgent()
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// Health endpoint. this endpoint is special because it doesn't print traces
|
||||||
|
app.get("/health", ( req: Request, res: Response ) => {
|
||||||
|
res.send({
|
||||||
|
"status": "ok"
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// Controller v1
|
||||||
|
app.post("/v1", async( req: Request, res: Response ) => {
|
||||||
|
await controllerV1(req, res);
|
||||||
|
});
|
||||||
|
|
||||||
|
// *********************************************************************************************************************
|
||||||
|
|
||||||
|
// Unknown paths or verbs
|
||||||
|
app.use(function (req : Request, res : Response) {
|
||||||
|
res.status(404)
|
||||||
|
.send({"error": "Unknown resource or HTTP verb"})
|
||||||
|
})
|
||||||
|
|
||||||
|
// Errors
|
||||||
|
app.use(function (err: any, req: Request, res: Response, next: NextFunction) {
|
||||||
|
if (err) {
|
||||||
|
let msg = 'Invalid request: ' + err;
|
||||||
|
msg = msg.replace("\n", "").replace("\r", "")
|
||||||
|
log.error(msg)
|
||||||
|
res.send({"error": msg})
|
||||||
|
} else {
|
||||||
|
next()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
module.exports = app;
|
||||||
@@ -1,31 +0,0 @@
|
|||||||
import got from 'got'
|
|
||||||
import { sleep } from '../utils'
|
|
||||||
|
|
||||||
/*
|
|
||||||
This method uses the captcha-harvester project:
|
|
||||||
https://github.com/NoahCardoza/CaptchaHarvester
|
|
||||||
|
|
||||||
While the function must take url/sitekey/type args,
|
|
||||||
they aren't used because the harvester server must
|
|
||||||
be preconfigured.
|
|
||||||
|
|
||||||
ENV:
|
|
||||||
HARVESTER_ENDPOINT: This must be the full path
|
|
||||||
to the /token endpoint of the harvester.
|
|
||||||
E.G. "https://127.0.0.1:5000/token"
|
|
||||||
*/
|
|
||||||
|
|
||||||
export default async function solve(): Promise<string> {
|
|
||||||
const endpoint = process.env.HARVESTER_ENDPOINT
|
|
||||||
if (!endpoint) { throw Error('ENV variable `HARVESTER_ENDPOINT` must be set.') }
|
|
||||||
while (true) {
|
|
||||||
try {
|
|
||||||
return (await got.get(process.env.HARVESTER_ENDPOINT, {
|
|
||||||
https: { rejectUnauthorized: false }
|
|
||||||
})).body
|
|
||||||
} catch (e) {
|
|
||||||
if (e.response.statusCode !== 418) { throw e }
|
|
||||||
}
|
|
||||||
await sleep(3000)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,21 +0,0 @@
|
|||||||
const solveCaptcha = require('hcaptcha-solver');
|
|
||||||
import { SolverOptions } from '.'
|
|
||||||
/*
|
|
||||||
This method uses the hcaptcha-solver project:
|
|
||||||
https://github.com/JimmyLaurent/hcaptcha-solver
|
|
||||||
|
|
||||||
TODO: allow user pass custom options to the solver.
|
|
||||||
|
|
||||||
ENV:
|
|
||||||
There are no other variables that must be set to get this to work
|
|
||||||
*/
|
|
||||||
|
|
||||||
export default async function solve({ url }: SolverOptions): Promise<string> {
|
|
||||||
try {
|
|
||||||
const token = await solveCaptcha(url)
|
|
||||||
return token
|
|
||||||
} catch (e) {
|
|
||||||
console.error(e)
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,3 +1,5 @@
|
|||||||
|
import log from "../services/log";
|
||||||
|
|
||||||
export enum CaptchaType {
|
export enum CaptchaType {
|
||||||
re = 'reCaptcha',
|
re = 'reCaptcha',
|
||||||
h = 'hCaptcha'
|
h = 'hCaptcha'
|
||||||
@@ -16,7 +18,9 @@ const captchaSolvers: { [key: string]: Solver } = {}
|
|||||||
export default (): Solver => {
|
export default (): Solver => {
|
||||||
const method = process.env.CAPTCHA_SOLVER
|
const method = process.env.CAPTCHA_SOLVER
|
||||||
|
|
||||||
if (!method) { return null }
|
if (!method || method.toLowerCase() == 'none') {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
if (!(method in captchaSolvers)) {
|
if (!(method in captchaSolvers)) {
|
||||||
try {
|
try {
|
||||||
@@ -26,10 +30,12 @@ export default (): Solver => {
|
|||||||
throw Error(`The solver '${method}' is not a valid captcha solving method.`)
|
throw Error(`The solver '${method}' is not a valid captcha solving method.`)
|
||||||
} else {
|
} else {
|
||||||
console.error(e)
|
console.error(e)
|
||||||
throw Error(`An error occured loading the solver '${method}'.`)
|
throw Error(`An error occurred loading the solver '${method}'.`)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
log.info(`Using '${method}' to solve the captcha.`);
|
||||||
|
|
||||||
return captchaSolvers[method]
|
return captchaSolvers[method]
|
||||||
}
|
}
|
||||||
|
|||||||
179
src/controllers/v1.ts
Normal file
179
src/controllers/v1.ts
Normal file
@@ -0,0 +1,179 @@
|
|||||||
|
// todo: avoid puppeter objects
|
||||||
|
import {SetCookie, Headers, HttpMethod} from 'puppeteer'
|
||||||
|
import {Request, Response} from 'express';
|
||||||
|
|
||||||
|
import log from '../services/log'
|
||||||
|
import {browserRequest, ChallengeResolutionResultT, ChallengeResolutionT} from "../services/solver";
|
||||||
|
import {SessionCreateOptions} from "../services/sessions";
|
||||||
|
const sessions = require('../services/sessions')
|
||||||
|
const version: string = 'v' + require('../../package.json').version
|
||||||
|
|
||||||
|
interface V1Routes {
|
||||||
|
[key: string]: (params: V1RequestBase, response: V1ResponseBase) => Promise<void>
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface Proxy {
|
||||||
|
url?: string
|
||||||
|
username?: string
|
||||||
|
password?: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface V1RequestBase {
|
||||||
|
cmd: string
|
||||||
|
cookies?: SetCookie[],
|
||||||
|
maxTimeout?: number
|
||||||
|
proxy?: Proxy
|
||||||
|
session: string
|
||||||
|
headers?: Headers // deprecated v2, not used
|
||||||
|
userAgent?: string // deprecated v2, not used
|
||||||
|
}
|
||||||
|
|
||||||
|
interface V1RequestSession extends V1RequestBase {
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface V1Request extends V1RequestBase {
|
||||||
|
url: string
|
||||||
|
method?: HttpMethod
|
||||||
|
postData?: string
|
||||||
|
returnOnlyCookies?: boolean
|
||||||
|
download?: boolean // deprecated v2, not used
|
||||||
|
returnRawHtml?: boolean // deprecated v2, not used
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface V1ResponseBase {
|
||||||
|
status: string
|
||||||
|
message: string
|
||||||
|
startTimestamp: number
|
||||||
|
endTimestamp: number
|
||||||
|
version: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface V1ResponseSolution extends V1ResponseBase {
|
||||||
|
solution: ChallengeResolutionResultT
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface V1ResponseSession extends V1ResponseBase {
|
||||||
|
session: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface V1ResponseSessions extends V1ResponseBase {
|
||||||
|
sessions: string[]
|
||||||
|
}
|
||||||
|
|
||||||
|
export const routes: V1Routes = {
|
||||||
|
'sessions.create': async (params: V1RequestSession, response: V1ResponseSession): Promise<void> => {
|
||||||
|
const options: SessionCreateOptions = {
|
||||||
|
oneTimeSession: false,
|
||||||
|
cookies: params.cookies,
|
||||||
|
maxTimeout: params.maxTimeout,
|
||||||
|
proxy: params.proxy
|
||||||
|
}
|
||||||
|
const { sessionId, browser } = await sessions.create(params.session, options)
|
||||||
|
if (browser) {
|
||||||
|
response.status = "ok";
|
||||||
|
response.message = "Session created successfully.";
|
||||||
|
response.session = sessionId
|
||||||
|
} else {
|
||||||
|
throw Error('Error creating session.')
|
||||||
|
}
|
||||||
|
},
|
||||||
|
'sessions.list': async (params: V1RequestSession, response: V1ResponseSessions): Promise<void> => {
|
||||||
|
response.status = "ok";
|
||||||
|
response.message = "";
|
||||||
|
response.sessions = sessions.list();
|
||||||
|
},
|
||||||
|
'sessions.destroy': async (params: V1RequestSession, response: V1ResponseBase): Promise<void> => {
|
||||||
|
if (await sessions.destroy(params.session)) {
|
||||||
|
response.status = "ok";
|
||||||
|
response.message = "The session has been removed.";
|
||||||
|
} else {
|
||||||
|
throw Error('This session does not exist.')
|
||||||
|
}
|
||||||
|
},
|
||||||
|
'request.get': async (params: V1Request, response: V1ResponseSolution): Promise<void> => {
|
||||||
|
params.method = 'GET'
|
||||||
|
if (params.postData) {
|
||||||
|
throw Error('Cannot use "postBody" when sending a GET request.')
|
||||||
|
}
|
||||||
|
if (params.returnRawHtml) {
|
||||||
|
log.warn("Request parameter 'returnRawHtml' was removed in FlareSolverr v2.")
|
||||||
|
}
|
||||||
|
if (params.download) {
|
||||||
|
log.warn("Request parameter 'download' was removed in FlareSolverr v2.")
|
||||||
|
}
|
||||||
|
const result: ChallengeResolutionT = await browserRequest(params)
|
||||||
|
|
||||||
|
response.status = result.status;
|
||||||
|
response.message = result.message;
|
||||||
|
response.solution = result.result;
|
||||||
|
if (response.message) {
|
||||||
|
log.info(response.message)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
'request.post': async (params: V1Request, response: V1ResponseSolution): Promise<void> => {
|
||||||
|
params.method = 'POST'
|
||||||
|
if (!params.postData) {
|
||||||
|
throw Error('Must send param "postBody" when sending a POST request.')
|
||||||
|
}
|
||||||
|
if (params.returnRawHtml) {
|
||||||
|
log.warn("Request parameter 'returnRawHtml' was removed in FlareSolverr v2.")
|
||||||
|
}
|
||||||
|
if (params.download) {
|
||||||
|
log.warn("Request parameter 'download' was removed in FlareSolverr v2.")
|
||||||
|
}
|
||||||
|
const result: ChallengeResolutionT = await browserRequest(params)
|
||||||
|
|
||||||
|
response.status = result.status;
|
||||||
|
response.message = result.message;
|
||||||
|
response.solution = result.result;
|
||||||
|
if (response.message) {
|
||||||
|
log.info(response.message)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function controllerV1(req: Request, res: Response): Promise<void> {
|
||||||
|
const response: V1ResponseBase = {
|
||||||
|
status: null,
|
||||||
|
message: null,
|
||||||
|
startTimestamp: Date.now(),
|
||||||
|
endTimestamp: 0,
|
||||||
|
version: version
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const params: V1RequestBase = req.body
|
||||||
|
// do some validations
|
||||||
|
if (!params.cmd) {
|
||||||
|
throw Error("Request parameter 'cmd' is mandatory.")
|
||||||
|
}
|
||||||
|
if (params.headers) {
|
||||||
|
log.warn("Request parameter 'headers' was removed in FlareSolverr v2.")
|
||||||
|
}
|
||||||
|
if (params.userAgent) {
|
||||||
|
log.warn("Request parameter 'userAgent' was removed in FlareSolverr v2.")
|
||||||
|
}
|
||||||
|
|
||||||
|
// set default values
|
||||||
|
if (!params.maxTimeout || params.maxTimeout < 1) {
|
||||||
|
params.maxTimeout = 60000;
|
||||||
|
}
|
||||||
|
|
||||||
|
// execute the command
|
||||||
|
const route = routes[params.cmd]
|
||||||
|
if (route) {
|
||||||
|
await route(params, response)
|
||||||
|
} else {
|
||||||
|
throw Error(`The command '${params.cmd}' is invalid.`)
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
res.status(500)
|
||||||
|
response.status = "error";
|
||||||
|
response.message = e.toString();
|
||||||
|
log.error(response.message)
|
||||||
|
}
|
||||||
|
|
||||||
|
response.endTimestamp = Date.now()
|
||||||
|
log.info(`Response in ${(response.endTimestamp - response.startTimestamp) / 1000} s`)
|
||||||
|
res.send(response)
|
||||||
|
}
|
||||||
114
src/index.ts
114
src/index.ts
@@ -1,114 +0,0 @@
|
|||||||
import log from './log'
|
|
||||||
import { createServer, IncomingMessage, ServerResponse } from 'http';
|
|
||||||
import { RequestContext } from './types'
|
|
||||||
import Router, { BaseAPICall } from './routes'
|
|
||||||
|
|
||||||
const version: string = "v" + require('../package.json').version
|
|
||||||
const serverPort: number = Number(process.env.PORT) || 8191
|
|
||||||
const serverHost: string = process.env.HOST || '0.0.0.0'
|
|
||||||
|
|
||||||
|
|
||||||
function errorResponse(errorMsg: string, res: ServerResponse, startTimestamp: number) {
|
|
||||||
log.error(errorMsg)
|
|
||||||
const response = {
|
|
||||||
status: 'error',
|
|
||||||
message: errorMsg,
|
|
||||||
startTimestamp,
|
|
||||||
endTimestamp: Date.now(),
|
|
||||||
version
|
|
||||||
}
|
|
||||||
res.writeHead(500, {
|
|
||||||
'Content-Type': 'application/json'
|
|
||||||
})
|
|
||||||
res.write(JSON.stringify(response))
|
|
||||||
res.end()
|
|
||||||
}
|
|
||||||
|
|
||||||
function successResponse(successMsg: string, extendedProperties: object, res: ServerResponse, startTimestamp: number) {
|
|
||||||
const endTimestamp = Date.now()
|
|
||||||
log.info(`Successful response in ${(endTimestamp - startTimestamp) / 1000} s`)
|
|
||||||
if (successMsg) { log.info(successMsg) }
|
|
||||||
|
|
||||||
const response = Object.assign({
|
|
||||||
status: 'ok',
|
|
||||||
message: successMsg || '',
|
|
||||||
startTimestamp,
|
|
||||||
endTimestamp,
|
|
||||||
version
|
|
||||||
}, extendedProperties || {})
|
|
||||||
res.writeHead(200, {
|
|
||||||
'Content-Type': 'application/json'
|
|
||||||
})
|
|
||||||
res.write(JSON.stringify(response))
|
|
||||||
res.end()
|
|
||||||
}
|
|
||||||
|
|
||||||
function validateIncomingRequest(ctx: RequestContext, params: BaseAPICall) {
|
|
||||||
log.info(`Params: ${JSON.stringify(params)}`)
|
|
||||||
|
|
||||||
if (ctx.req.method !== 'POST') {
|
|
||||||
ctx.errorResponse('Only the POST method is allowed')
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
if (ctx.req.url !== '/v1') {
|
|
||||||
ctx.errorResponse('Only /v1 endpoint is allowed')
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!params.cmd) {
|
|
||||||
ctx.errorResponse("Parameter 'cmd' is mandatory")
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
createServer((req: IncomingMessage, res: ServerResponse) => {
|
|
||||||
const startTimestamp = Date.now()
|
|
||||||
|
|
||||||
// count the request for the log prefix
|
|
||||||
log.incRequests()
|
|
||||||
log.info(`Incoming request: ${req.method} ${req.url}`)
|
|
||||||
|
|
||||||
// show welcome message
|
|
||||||
if (req.url == '/') {
|
|
||||||
successResponse("FlareSolverr is ready!", null, res, startTimestamp);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// get request body
|
|
||||||
const bodyParts: any[] = []
|
|
||||||
req.on('data', chunk => {
|
|
||||||
bodyParts.push(chunk)
|
|
||||||
}).on('end', () => {
|
|
||||||
// parse params
|
|
||||||
const body = Buffer.concat(bodyParts).toString()
|
|
||||||
let params: BaseAPICall = null
|
|
||||||
try {
|
|
||||||
params = JSON.parse(body)
|
|
||||||
} catch (err) {
|
|
||||||
errorResponse('Body must be in JSON format', res, startTimestamp)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
const ctx: RequestContext = {
|
|
||||||
req,
|
|
||||||
res,
|
|
||||||
startTimestamp,
|
|
||||||
errorResponse: (msg) => errorResponse(msg, res, startTimestamp),
|
|
||||||
successResponse: (msg, extendedProperties) => successResponse(msg, extendedProperties, res, startTimestamp)
|
|
||||||
}
|
|
||||||
|
|
||||||
// validate params
|
|
||||||
if (!validateIncomingRequest(ctx, params)) { return }
|
|
||||||
|
|
||||||
// process request
|
|
||||||
Router(ctx, params).catch(e => {
|
|
||||||
console.error(e)
|
|
||||||
ctx.errorResponse(e.message)
|
|
||||||
})
|
|
||||||
})
|
|
||||||
}).listen(serverPort, serverHost, () => {
|
|
||||||
log.info(`FlareSolverr ${version} listening on http://${serverHost}:${serverPort}`)
|
|
||||||
})
|
|
||||||
19
src/log.ts
19
src/log.ts
@@ -1,19 +0,0 @@
|
|||||||
let requests = 0
|
|
||||||
|
|
||||||
const LOG_HTML: boolean = Boolean(process.env.LOG_HTML) || false
|
|
||||||
|
|
||||||
export default {
|
|
||||||
incRequests: () => { requests++ },
|
|
||||||
html(html: string) {
|
|
||||||
if (LOG_HTML)
|
|
||||||
this.debug(html)
|
|
||||||
},
|
|
||||||
...require('console-log-level')(
|
|
||||||
{
|
|
||||||
level: process.env.LOG_LEVEL || 'info',
|
|
||||||
prefix(level: string) {
|
|
||||||
return `${new Date().toISOString()} ${level.toUpperCase()} REQ-${requests}`
|
|
||||||
}
|
|
||||||
}
|
|
||||||
)
|
|
||||||
}
|
|
||||||
141
src/providers/cloudflare.ts
Normal file
141
src/providers/cloudflare.ts
Normal file
@@ -0,0 +1,141 @@
|
|||||||
|
import {Page, Response} from 'puppeteer'
|
||||||
|
|
||||||
|
import log from "../services/log";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This class contains the logic to solve protections provided by CloudFlare
|
||||||
|
**/
|
||||||
|
|
||||||
|
const BAN_SELECTORS = ['span[data-translate="error"]'];
|
||||||
|
const CHALLENGE_SELECTORS = ['#trk_jschal_js', '.ray_id', '.attack-box', '#cf-please-wait'];
|
||||||
|
const CAPTCHA_SELECTORS = ['input[name="cf_captcha_kind"]'];
|
||||||
|
|
||||||
|
export default async function resolveChallenge(url: string, page: Page, response: Response): Promise<Response> {
|
||||||
|
|
||||||
|
// look for challenge and return fast if not detected
|
||||||
|
if (response.headers().server &&
|
||||||
|
response.headers().server.startsWith('cloudflare') &&
|
||||||
|
(response.status() == 403 || response.status() == 503)) {
|
||||||
|
log.info('Cloudflare detected');
|
||||||
|
} else {
|
||||||
|
log.info('Cloudflare not detected');
|
||||||
|
return response;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (await findAnySelector(page, BAN_SELECTORS)) {
|
||||||
|
throw new Error('Cloudflare has blocked this request. Probably your IP is banned for this site, check in your web browser.')
|
||||||
|
}
|
||||||
|
|
||||||
|
let selectorFound = false;
|
||||||
|
if (response.status() > 400) {
|
||||||
|
|
||||||
|
// find Cloudflare selectors
|
||||||
|
let selector: string = await findAnySelector(page, CHALLENGE_SELECTORS)
|
||||||
|
if (selector) {
|
||||||
|
selectorFound = true;
|
||||||
|
log.debug(`Javascript challenge element '${selector}' detected.`)
|
||||||
|
log.debug('Waiting for Cloudflare challenge...')
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
try {
|
||||||
|
|
||||||
|
selector = await findAnySelector(page, CHALLENGE_SELECTORS)
|
||||||
|
if (!selector) {
|
||||||
|
// solved!
|
||||||
|
log.debug('Challenge element not found')
|
||||||
|
break
|
||||||
|
} else {
|
||||||
|
log.debug(`Javascript challenge element '${selector}' detected.`)
|
||||||
|
|
||||||
|
// new Cloudflare Challenge #cf-please-wait
|
||||||
|
const displayStyle = await page.evaluate((selector) => {
|
||||||
|
return getComputedStyle(document.querySelector(selector)).getPropertyValue("display");
|
||||||
|
}, selector);
|
||||||
|
if (displayStyle == "none") {
|
||||||
|
// spinner is hidden, could be a captcha or not
|
||||||
|
log.debug('Challenge element is hidden')
|
||||||
|
// wait until redirecting disappears
|
||||||
|
while (true) {
|
||||||
|
try {
|
||||||
|
await page.waitFor(1000)
|
||||||
|
const displayStyle2 = await page.evaluate(() => {
|
||||||
|
return getComputedStyle(document.querySelector('#cf-spinner-redirecting')).getPropertyValue("display");
|
||||||
|
});
|
||||||
|
if (displayStyle2 == "none") {
|
||||||
|
break // hCaptcha detected
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
break // redirection completed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
break
|
||||||
|
} else {
|
||||||
|
log.debug('Challenge element is visible')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
log.debug('Found challenge element again')
|
||||||
|
|
||||||
|
} catch (error)
|
||||||
|
{
|
||||||
|
log.debug("Unexpected error: " + error);
|
||||||
|
if (!error.toString().includes("Execution context was destroyed")) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
log.debug('Waiting for Cloudflare challenge...')
|
||||||
|
await page.waitFor(1000)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.debug('Validating HTML code...')
|
||||||
|
} else {
|
||||||
|
log.debug(`No challenge element detected.`)
|
||||||
|
}
|
||||||
|
|
||||||
|
} else {
|
||||||
|
// some sites use cloudflare but there is no challenge
|
||||||
|
log.debug(`Javascript challenge not detected. Status code: ${response.status()}`);
|
||||||
|
selectorFound = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// check for CAPTCHA challenge
|
||||||
|
if (await findAnySelector(page, CAPTCHA_SELECTORS)) {
|
||||||
|
log.info('CAPTCHA challenge detected');
|
||||||
|
throw new Error('FlareSolverr can not resolve CAPTCHA challenges. Since the captcha doesn\'t always appear, you may have better luck with the next request.');
|
||||||
|
|
||||||
|
// const captchaSolver = getCaptchaSolver()
|
||||||
|
// if (captchaSolver) {
|
||||||
|
// // to-do: get the params
|
||||||
|
// log.info('Waiting to receive captcha token to bypass challenge...')
|
||||||
|
// const token = await captchaSolver({
|
||||||
|
// url,
|
||||||
|
// sitekey,
|
||||||
|
// type: captchaType
|
||||||
|
// })
|
||||||
|
// log.debug(`Token received: ${token}`);
|
||||||
|
// // to-do: send the token
|
||||||
|
// }
|
||||||
|
// } else {
|
||||||
|
// throw new Error('Captcha detected but no automatic solver is configured.');
|
||||||
|
// }
|
||||||
|
} else {
|
||||||
|
if (!selectorFound)
|
||||||
|
{
|
||||||
|
throw new Error('No challenge selectors found, unable to proceed.')
|
||||||
|
} else {
|
||||||
|
log.info('Challenge solved');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return response;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function findAnySelector(page: Page, selectors: string[]) {
|
||||||
|
for (const selector of selectors) {
|
||||||
|
const cfChallengeElem = await page.$(selector)
|
||||||
|
if (cfChallengeElem) {
|
||||||
|
return selector;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
491
src/routes.ts
491
src/routes.ts
@@ -1,491 +0,0 @@
|
|||||||
import { v1 as UUIDv1 } from 'uuid'
|
|
||||||
import sessions, { SessionsCacheItem } from './session'
|
|
||||||
import { RequestContext } from './types'
|
|
||||||
import log from './log'
|
|
||||||
import { SetCookie, Request, Headers, HttpMethod, Overrides, Cookie } from 'puppeteer'
|
|
||||||
import { TimeoutError } from 'puppeteer/Errors'
|
|
||||||
import getCaptchaSolver, { CaptchaType } from './captcha'
|
|
||||||
import * as Puppeteer from "puppeteer-extra/dist/puppeteer";
|
|
||||||
const Timeout = require('await-timeout');
|
|
||||||
|
|
||||||
export interface BaseAPICall {
|
|
||||||
cmd: string
|
|
||||||
}
|
|
||||||
|
|
||||||
interface BaseSessionsAPICall extends BaseAPICall {
|
|
||||||
session?: string
|
|
||||||
}
|
|
||||||
|
|
||||||
interface SessionsCreateAPICall extends BaseSessionsAPICall {
|
|
||||||
userAgent?: string,
|
|
||||||
cookies?: SetCookie[],
|
|
||||||
headers?: Headers
|
|
||||||
maxTimeout?: number
|
|
||||||
proxy?: any
|
|
||||||
}
|
|
||||||
|
|
||||||
interface BaseRequestAPICall extends BaseAPICall {
|
|
||||||
url: string
|
|
||||||
method?: HttpMethod
|
|
||||||
postData?: string
|
|
||||||
session?: string
|
|
||||||
userAgent?: string
|
|
||||||
maxTimeout?: number
|
|
||||||
cookies?: SetCookie[],
|
|
||||||
headers?: Headers
|
|
||||||
proxy?: any, // TODO: use interface not any
|
|
||||||
download?: boolean
|
|
||||||
returnOnlyCookies?: boolean
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
interface Routes {
|
|
||||||
[key: string]: (ctx: RequestContext, params: BaseAPICall) => void | Promise<void>
|
|
||||||
}
|
|
||||||
|
|
||||||
interface ChallengeResolutionResultT {
|
|
||||||
url: string
|
|
||||||
status: number,
|
|
||||||
headers?: Headers,
|
|
||||||
response: string,
|
|
||||||
cookies: object[]
|
|
||||||
userAgent: string
|
|
||||||
}
|
|
||||||
|
|
||||||
interface ChallengeResolutionT {
|
|
||||||
status?: string
|
|
||||||
message: string
|
|
||||||
result: ChallengeResolutionResultT
|
|
||||||
}
|
|
||||||
|
|
||||||
interface OverrideResolvers {
|
|
||||||
method?: (request: Request) => HttpMethod,
|
|
||||||
postData?: (request: Request) => string,
|
|
||||||
headers?: (request: Request) => Headers
|
|
||||||
}
|
|
||||||
|
|
||||||
type OverridesProps =
|
|
||||||
'method' |
|
|
||||||
'postData' |
|
|
||||||
'headers'
|
|
||||||
|
|
||||||
// We always set a Windows User-Agent because ARM builds are detected by CloudFlare
|
|
||||||
const DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36"
|
|
||||||
const CHALLENGE_SELECTORS = ['#trk_jschal_js', '.ray_id', '.attack-box']
|
|
||||||
const TOKEN_INPUT_NAMES = ['g-recaptcha-response', 'h-captcha-response']
|
|
||||||
|
|
||||||
async function interceptResponse(page: Puppeteer.Page, callback: (payload: ChallengeResolutionT) => any) {
|
|
||||||
const client = await page.target().createCDPSession();
|
|
||||||
await client.send('Fetch.enable', {
|
|
||||||
patterns: [
|
|
||||||
{
|
|
||||||
urlPattern: '*',
|
|
||||||
resourceType: 'Document',
|
|
||||||
requestStage: 'Response',
|
|
||||||
},
|
|
||||||
],
|
|
||||||
});
|
|
||||||
|
|
||||||
client.on('Fetch.requestPaused', async (e) => {
|
|
||||||
log.debug('Fetch.requestPaused. Checking if the response has valid cookies')
|
|
||||||
let headers = e.responseHeaders || []
|
|
||||||
|
|
||||||
let cookies = await page.cookies();
|
|
||||||
log.debug(cookies)
|
|
||||||
|
|
||||||
if (cookies.filter((c: Cookie) => c.name === 'cf_clearance').length > 0) {
|
|
||||||
log.debug('Aborting request and return cookies. valid cookies found')
|
|
||||||
await client.send('Fetch.failRequest', {requestId: e.requestId, errorReason: 'Aborted'})
|
|
||||||
|
|
||||||
let status = 'ok'
|
|
||||||
let message = ''
|
|
||||||
const payload: ChallengeResolutionT = {
|
|
||||||
status,
|
|
||||||
message,
|
|
||||||
result: {
|
|
||||||
url: page.url(),
|
|
||||||
status: e.status,
|
|
||||||
headers: headers.reduce((a: any, x: { name: any; value: any }) => ({ ...a, [x.name]: x.value }), {}),
|
|
||||||
response: null,
|
|
||||||
cookies: cookies,
|
|
||||||
userAgent: ''
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
callback(payload);
|
|
||||||
} else {
|
|
||||||
log.debug('Continuing request. no valid cookies found')
|
|
||||||
await client.send('Fetch.continueRequest', {requestId: e.requestId})
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
async function resolveChallengeWithTimeout(ctx: RequestContext, params: BaseRequestAPICall, page: Puppeteer.Page) {
|
|
||||||
const maxTimeout = params.maxTimeout || 60000
|
|
||||||
const timer = new Timeout();
|
|
||||||
try {
|
|
||||||
const promise = resolveChallenge(ctx, params, page);
|
|
||||||
return await Promise.race([
|
|
||||||
promise,
|
|
||||||
timer.set(maxTimeout, `Maximum timeout reached. maxTimeout=${maxTimeout} (ms)`)
|
|
||||||
]);
|
|
||||||
} finally {
|
|
||||||
timer.clear();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function resolveChallenge(ctx: RequestContext, { url, proxy, download, returnOnlyCookies }: BaseRequestAPICall, page: Puppeteer.Page): Promise<ChallengeResolutionT | void> {
|
|
||||||
|
|
||||||
let status = 'ok'
|
|
||||||
let message = ''
|
|
||||||
|
|
||||||
if (proxy) {
|
|
||||||
log.debug("Apply proxy");
|
|
||||||
if (proxy.username)
|
|
||||||
await page.authenticate({ username: proxy.username, password: proxy.password });
|
|
||||||
}
|
|
||||||
|
|
||||||
log.debug(`Navigating to... ${url}`)
|
|
||||||
let response = await page.goto(url, { waitUntil: 'domcontentloaded' })
|
|
||||||
|
|
||||||
log.html(await page.content())
|
|
||||||
|
|
||||||
// look for challenge
|
|
||||||
if (response.headers().server.startsWith('cloudflare')) {
|
|
||||||
log.info('Cloudflare detected')
|
|
||||||
|
|
||||||
if (await page.$('.cf-error-code')) {
|
|
||||||
await page.close()
|
|
||||||
return ctx.errorResponse('Cloudflare has blocked this request (Code 1020 Detected).')
|
|
||||||
}
|
|
||||||
|
|
||||||
if (response.status() > 400) {
|
|
||||||
// detect cloudflare wait 5s
|
|
||||||
let selectorFoundCount = 0
|
|
||||||
for (const selector of CHALLENGE_SELECTORS) {
|
|
||||||
const cfChallengeElem = await page.$(selector)
|
|
||||||
if (cfChallengeElem) {
|
|
||||||
selectorFoundCount++
|
|
||||||
log.debug(`'${selector}' challenge element detected.`)
|
|
||||||
log.debug('Waiting for Cloudflare challenge...')
|
|
||||||
|
|
||||||
let interceptingResult: ChallengeResolutionT;
|
|
||||||
if (returnOnlyCookies) { //If we just want to get the cookies, intercept the response before we get the content/body (just cookies and headers)
|
|
||||||
await interceptResponse(page, async function(payload){
|
|
||||||
interceptingResult = payload;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
while (true) {
|
|
||||||
await page.waitFor(1000)
|
|
||||||
try {
|
|
||||||
// catch exception timeout in waitForNavigation
|
|
||||||
response = await page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 5000 })
|
|
||||||
} catch (error) { }
|
|
||||||
|
|
||||||
if (returnOnlyCookies && interceptingResult) {
|
|
||||||
await page.close();
|
|
||||||
return interceptingResult;
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
// catch Execution context was destroyed
|
|
||||||
const cfChallengeElem = await page.$(selector)
|
|
||||||
if (!cfChallengeElem) { break }
|
|
||||||
log.debug('Found challenge element again...')
|
|
||||||
} catch (error)
|
|
||||||
{ }
|
|
||||||
|
|
||||||
response = await page.reload({ waitUntil: 'domcontentloaded' })
|
|
||||||
log.debug('Reloaded page...')
|
|
||||||
log.html(await page.content())
|
|
||||||
}
|
|
||||||
|
|
||||||
log.debug('Validating HTML code...')
|
|
||||||
break
|
|
||||||
} else {
|
|
||||||
log.debug(`No '${selector}' challenge element detected.`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
log.debug("Number of selector found: " + selectorFoundCount + ", total selector: " + CHALLENGE_SELECTORS.length)
|
|
||||||
if (selectorFoundCount == 0)
|
|
||||||
{
|
|
||||||
await page.close()
|
|
||||||
return ctx.errorResponse('No challenge selectors found, unable to proceed')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// it seems some captcha pages return 200 sometimes
|
|
||||||
if (await page.$('input[name="cf_captcha_kind"]')) {
|
|
||||||
const captchaSolver = getCaptchaSolver()
|
|
||||||
if (captchaSolver) {
|
|
||||||
const captchaStartTimestamp = Date.now()
|
|
||||||
const challengeForm = await page.$('#challenge-form')
|
|
||||||
if (challengeForm) {
|
|
||||||
const captchaTypeElm = await page.$('input[name="cf_captcha_kind"]')
|
|
||||||
const cfCaptchaType: string = await captchaTypeElm.evaluate((e: any) => e.value)
|
|
||||||
const captchaType: CaptchaType = (CaptchaType as any)[cfCaptchaType]
|
|
||||||
if (!captchaType) { return ctx.errorResponse('Unknown captcha type!') }
|
|
||||||
|
|
||||||
let sitekey = null
|
|
||||||
if (captchaType != 'hCaptcha' && process.env.CAPTCHA_SOLVER != 'hcaptcha-solver') {
|
|
||||||
const sitekeyElem = await page.$('*[data-sitekey]')
|
|
||||||
if (!sitekeyElem) { return ctx.errorResponse('Could not find sitekey!') }
|
|
||||||
sitekey = await sitekeyElem.evaluate((e) => e.getAttribute('data-sitekey'))
|
|
||||||
}
|
|
||||||
|
|
||||||
log.info('Waiting to receive captcha token to bypass challenge...')
|
|
||||||
const token = await captchaSolver({
|
|
||||||
url,
|
|
||||||
sitekey,
|
|
||||||
type: captchaType
|
|
||||||
})
|
|
||||||
|
|
||||||
if (!token) {
|
|
||||||
await page.close()
|
|
||||||
return ctx.errorResponse('Token solver failed to return a token.')
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const name of TOKEN_INPUT_NAMES) {
|
|
||||||
const input = await page.$(`textarea[name="${name}"]`)
|
|
||||||
if (input) { await input.evaluate((e: HTMLTextAreaElement, token) => { e.value = token }, token) }
|
|
||||||
}
|
|
||||||
|
|
||||||
// ignore preset event listeners on the form
|
|
||||||
await page.evaluate(() => {
|
|
||||||
window.addEventListener('submit', (e) => { event.stopPropagation() }, true)
|
|
||||||
})
|
|
||||||
|
|
||||||
// it seems some sites obfuscate their challenge forms
|
|
||||||
// TODO: look into how they do it and come up with a more solid solution
|
|
||||||
try {
|
|
||||||
// this element is added with js and we want to wait for all the js to load before submitting
|
|
||||||
await page.waitForSelector('#challenge-form [type=submit]', { timeout: 5000 })
|
|
||||||
} catch (err) {
|
|
||||||
if (err instanceof TimeoutError) {
|
|
||||||
log.debug(`No '#challenge-form [type=submit]' element detected.`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// calculates the time it took to solve the captcha
|
|
||||||
const captchaSolveTotalTime = Date.now() - captchaStartTimestamp
|
|
||||||
|
|
||||||
// generates a random wait time
|
|
||||||
const randomWaitTime = (Math.floor(Math.random() * 20) + 10) * 1000
|
|
||||||
|
|
||||||
// waits, if any, time remaining to appear human but stay as fast as possible
|
|
||||||
const timeLeft = randomWaitTime - captchaSolveTotalTime
|
|
||||||
if (timeLeft > 0) { await page.waitFor(timeLeft) }
|
|
||||||
|
|
||||||
let interceptingResult: ChallengeResolutionT;
|
|
||||||
if (returnOnlyCookies) { //If we just want to get the cookies, intercept the response before we get the content/body (just cookies and headers)
|
|
||||||
await interceptResponse(page, async function(payload){
|
|
||||||
interceptingResult = payload;
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// submit captcha response
|
|
||||||
challengeForm.evaluate((e: HTMLFormElement) => e.submit())
|
|
||||||
response = await page.waitForNavigation({ waitUntil: 'domcontentloaded' })
|
|
||||||
|
|
||||||
if (returnOnlyCookies && interceptingResult) {
|
|
||||||
await page.close();
|
|
||||||
return interceptingResult;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
status = 'warning'
|
|
||||||
message = 'Captcha detected but no automatic solver is configured.'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
log.debug("Response is: " + response.status())
|
|
||||||
}
|
|
||||||
|
|
||||||
const payload: ChallengeResolutionT = {
|
|
||||||
status,
|
|
||||||
message,
|
|
||||||
result: {
|
|
||||||
url: page.url(),
|
|
||||||
status: response.status(),
|
|
||||||
headers: response.headers(),
|
|
||||||
response: null,
|
|
||||||
cookies: await page.cookies(),
|
|
||||||
userAgent: await page.evaluate(() => navigator.userAgent)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (download) {
|
|
||||||
// for some reason we get an error unless we reload the page
|
|
||||||
// has something to do with a stale buffer and this is the quickest
|
|
||||||
// fix since I am short on time
|
|
||||||
response = await page.goto(url, { waitUntil: 'domcontentloaded' })
|
|
||||||
payload.result.response = (await response.buffer()).toString('base64')
|
|
||||||
} else {
|
|
||||||
payload.result.response = await page.content()
|
|
||||||
}
|
|
||||||
|
|
||||||
// make sure the page is closed because if it isn't and error will be thrown
|
|
||||||
// when a user uses a temporary session, the browser make be quit before
|
|
||||||
// the page is properly closed.
|
|
||||||
await page.close()
|
|
||||||
|
|
||||||
return payload
|
|
||||||
}
|
|
||||||
|
|
||||||
function mergeSessionWithParams({ defaults }: SessionsCacheItem, params: BaseRequestAPICall): BaseRequestAPICall {
|
|
||||||
const copy = { ...defaults, ...params }
|
|
||||||
|
|
||||||
// custom merging logic
|
|
||||||
copy.headers = { ...defaults.headers || {}, ...params.headers || {} } || null
|
|
||||||
|
|
||||||
return copy
|
|
||||||
}
|
|
||||||
|
|
||||||
async function setupPage(ctx: RequestContext, params: BaseRequestAPICall, browser: Puppeteer.Browser): Promise<Puppeteer.Page> {
|
|
||||||
const page = await browser.newPage()
|
|
||||||
|
|
||||||
// merge session defaults with params
|
|
||||||
const { method, postData, userAgent, headers, cookies } = params
|
|
||||||
|
|
||||||
let overrideResolvers: OverrideResolvers = {}
|
|
||||||
|
|
||||||
if (method !== 'GET') {
|
|
||||||
log.debug(`Setting method to ${method}`)
|
|
||||||
overrideResolvers.method = request => method
|
|
||||||
}
|
|
||||||
|
|
||||||
if (postData) {
|
|
||||||
log.debug(`Setting body data to ${postData}`)
|
|
||||||
overrideResolvers.postData = request => postData
|
|
||||||
}
|
|
||||||
|
|
||||||
if (userAgent) {
|
|
||||||
log.debug(`Using custom UA: ${userAgent}`)
|
|
||||||
await page.setUserAgent(userAgent)
|
|
||||||
} else {
|
|
||||||
await page.setUserAgent(DEFAULT_USER_AGENT)
|
|
||||||
}
|
|
||||||
|
|
||||||
if (headers) {
|
|
||||||
log.debug(`Adding custom headers: ${JSON.stringify(headers, null, 2)}`,)
|
|
||||||
overrideResolvers.headers = request => Object.assign(request.headers(), headers)
|
|
||||||
}
|
|
||||||
|
|
||||||
if (cookies) {
|
|
||||||
log.debug(`Setting custom cookies: ${JSON.stringify(cookies, null, 2)}`,)
|
|
||||||
await page.setCookie(...cookies)
|
|
||||||
}
|
|
||||||
|
|
||||||
// if any keys have been set on the object
|
|
||||||
if (Object.keys(overrideResolvers).length > 0) {
|
|
||||||
log.debug(overrideResolvers)
|
|
||||||
let callbackRunOnce = false
|
|
||||||
const callback = (request: Request) => {
|
|
||||||
|
|
||||||
if (callbackRunOnce || !request.isNavigationRequest()) {
|
|
||||||
request.continue()
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
callbackRunOnce = true
|
|
||||||
const overrides: Overrides = {}
|
|
||||||
|
|
||||||
Object.keys(overrideResolvers).forEach((key: OverridesProps) => {
|
|
||||||
// @ts-ignore
|
|
||||||
overrides[key] = overrideResolvers[key](request)
|
|
||||||
});
|
|
||||||
|
|
||||||
log.debug(overrides)
|
|
||||||
|
|
||||||
request.continue(overrides)
|
|
||||||
}
|
|
||||||
|
|
||||||
await page.setRequestInterception(true)
|
|
||||||
page.on('request', callback)
|
|
||||||
}
|
|
||||||
|
|
||||||
return page
|
|
||||||
}
|
|
||||||
|
|
||||||
const browserRequest = async (ctx: RequestContext, params: BaseRequestAPICall) => {
|
|
||||||
const oneTimeSession = params.session === undefined
|
|
||||||
const sessionId = params.session || UUIDv1()
|
|
||||||
const session = oneTimeSession
|
|
||||||
? await sessions.create(sessionId, {
|
|
||||||
userAgent: params.userAgent,
|
|
||||||
oneTimeSession
|
|
||||||
})
|
|
||||||
: sessions.get(sessionId)
|
|
||||||
|
|
||||||
if (session === false) {
|
|
||||||
return ctx.errorResponse('This session does not exist. Use \'list_sessions\' to see all the existing sessions.')
|
|
||||||
}
|
|
||||||
|
|
||||||
params = mergeSessionWithParams(session, params)
|
|
||||||
|
|
||||||
try {
|
|
||||||
const page = await setupPage(ctx, params, session.browser)
|
|
||||||
const data = await resolveChallengeWithTimeout(ctx, params, page)
|
|
||||||
|
|
||||||
if (data) {
|
|
||||||
const { status } = data
|
|
||||||
delete data.status
|
|
||||||
ctx.successResponse(data.message, {
|
|
||||||
...(oneTimeSession ? {} : { session: sessionId }),
|
|
||||||
...(status ? { status } : {}),
|
|
||||||
solution: data.result
|
|
||||||
})
|
|
||||||
}
|
|
||||||
} catch (error) {
|
|
||||||
log.error(error)
|
|
||||||
return ctx.errorResponse("Unable to process browser request. Error: " + error)
|
|
||||||
} finally {
|
|
||||||
if (oneTimeSession) { sessions.destroy(sessionId) }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export const routes: Routes = {
|
|
||||||
'sessions.create': async (ctx, { session, ...options }: SessionsCreateAPICall) => {
|
|
||||||
session = session || UUIDv1()
|
|
||||||
const { browser } = await sessions.create(session, options)
|
|
||||||
if (browser) { ctx.successResponse('Session created successfully.', { session }) }
|
|
||||||
},
|
|
||||||
'sessions.list': (ctx) => {
|
|
||||||
ctx.successResponse(null, { sessions: sessions.list() })
|
|
||||||
},
|
|
||||||
'sessions.destroy': async (ctx, { session }: BaseSessionsAPICall) => {
|
|
||||||
if (await sessions.destroy(session)) { return ctx.successResponse('The session has been removed.') }
|
|
||||||
ctx.errorResponse('This session does not exist.')
|
|
||||||
},
|
|
||||||
'request.get': async (ctx, params: BaseRequestAPICall) => {
|
|
||||||
params.method = 'GET'
|
|
||||||
if (params.postData) {
|
|
||||||
return ctx.errorResponse('Cannot use "postBody" when sending a GET request.')
|
|
||||||
}
|
|
||||||
await browserRequest(ctx, params)
|
|
||||||
},
|
|
||||||
'request.post': async (ctx, params: BaseRequestAPICall) => {
|
|
||||||
params.method = 'POST'
|
|
||||||
|
|
||||||
if (!params.postData) {
|
|
||||||
return ctx.errorResponse('Must send param "postBody" when sending a POST request.')
|
|
||||||
}
|
|
||||||
|
|
||||||
await browserRequest(ctx, params)
|
|
||||||
},
|
|
||||||
'request.cookies': async (ctx, params: BaseRequestAPICall) => {
|
|
||||||
params.returnOnlyCookies = true
|
|
||||||
params.method = 'GET'
|
|
||||||
if (params.postData) {
|
|
||||||
return ctx.errorResponse('Cannot use "postBody" when sending a GET request.')
|
|
||||||
}
|
|
||||||
await browserRequest(ctx, params)
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
export default async function Router(ctx: RequestContext, params: BaseAPICall): Promise<void> {
|
|
||||||
const route = routes[params.cmd]
|
|
||||||
if (route) { return await route(ctx, params) }
|
|
||||||
return ctx.errorResponse(`The command '${params.cmd}' is invalid.`)
|
|
||||||
}
|
|
||||||
58
src/server.ts
Normal file
58
src/server.ts
Normal file
@@ -0,0 +1,58 @@
|
|||||||
|
import log from './services/log'
|
||||||
|
import {testWebBrowserInstallation} from "./services/sessions";
|
||||||
|
|
||||||
|
const app = require("./app");
|
||||||
|
const version: string = 'v' + require('../package.json').version
|
||||||
|
const serverPort: number = Number(process.env.PORT) || 8191
|
||||||
|
const serverHost: string = process.env.HOST || '0.0.0.0'
|
||||||
|
|
||||||
|
function validateEnvironmentVariables() {
|
||||||
|
// ip and port variables are validated by nodejs
|
||||||
|
if (process.env.LOG_LEVEL && ['error', 'warn', 'info', 'verbose', 'debug'].indexOf(process.env.LOG_LEVEL) == -1) {
|
||||||
|
log.error(`The environment variable 'LOG_LEVEL' is wrong. Check the documentation.`);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
if (process.env.LOG_HTML && ['true', 'false'].indexOf(process.env.LOG_HTML) == -1) {
|
||||||
|
log.error(`The environment variable 'LOG_HTML' is wrong. Check the documentation.`);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
if (process.env.HEADLESS && ['true', 'false'].indexOf(process.env.HEADLESS) == -1) {
|
||||||
|
log.error(`The environment variable 'HEADLESS' is wrong. Check the documentation.`);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
// todo: fix resolvers
|
||||||
|
// try {
|
||||||
|
// getCaptchaSolver();
|
||||||
|
// } catch (e) {
|
||||||
|
// log.error(`The environment variable 'CAPTCHA_SOLVER' is wrong. ${e.message}`);
|
||||||
|
// process.exit(1);
|
||||||
|
// }
|
||||||
|
}
|
||||||
|
|
||||||
|
// Init
|
||||||
|
log.info(`FlareSolverr ${version}`);
|
||||||
|
log.debug('Debug log enabled');
|
||||||
|
|
||||||
|
process.on('SIGTERM', () => {
|
||||||
|
// Capture signal on Docker Stop #158
|
||||||
|
log.info("Process interrupted")
|
||||||
|
process.exit(0)
|
||||||
|
})
|
||||||
|
|
||||||
|
validateEnvironmentVariables();
|
||||||
|
|
||||||
|
testWebBrowserInstallation().then(() => {
|
||||||
|
// Start server
|
||||||
|
app.listen(serverPort, serverHost, () => {
|
||||||
|
log.info(`Listening on http://${serverHost}:${serverPort}`);
|
||||||
|
})
|
||||||
|
}).catch(function(e) {
|
||||||
|
log.error(e);
|
||||||
|
const msg: string = "" + e;
|
||||||
|
if (msg.includes('while trying to connect to the browser!')) {
|
||||||
|
log.error(`It seems that the system is too slow to run FlareSolverr.
|
||||||
|
If you are running with Docker, try to remove CPU limits in the container.
|
||||||
|
If not, try setting the 'BROWSER_TIMEOUT' environment variable and the 'maxTimeout' parameter to higher values.`);
|
||||||
|
}
|
||||||
|
process.exit(1);
|
||||||
|
})
|
||||||
41
src/services/log.ts
Normal file
41
src/services/log.ts
Normal file
@@ -0,0 +1,41 @@
|
|||||||
|
let requests = 0
|
||||||
|
|
||||||
|
const LOG_HTML: boolean = process.env.LOG_HTML == 'true';
|
||||||
|
|
||||||
|
function toIsoString(date: Date) {
|
||||||
|
// this function fixes Date.toISOString() adding timezone
|
||||||
|
let tzo = -date.getTimezoneOffset(),
|
||||||
|
dif = tzo >= 0 ? '+' : '-',
|
||||||
|
pad = function(num: number) {
|
||||||
|
let norm = Math.floor(Math.abs(num));
|
||||||
|
return (norm < 10 ? '0' : '') + norm;
|
||||||
|
};
|
||||||
|
|
||||||
|
return date.getFullYear() +
|
||||||
|
'-' + pad(date.getMonth() + 1) +
|
||||||
|
'-' + pad(date.getDate()) +
|
||||||
|
'T' + pad(date.getHours()) +
|
||||||
|
':' + pad(date.getMinutes()) +
|
||||||
|
':' + pad(date.getSeconds()) +
|
||||||
|
dif + pad(tzo / 60) +
|
||||||
|
':' + pad(tzo % 60);
|
||||||
|
}
|
||||||
|
|
||||||
|
export default {
|
||||||
|
incRequests: () => {
|
||||||
|
requests++
|
||||||
|
},
|
||||||
|
html(html: string) {
|
||||||
|
if (LOG_HTML) {
|
||||||
|
this.debug(html)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
...require('console-log-level')(
|
||||||
|
{level: process.env.LOG_LEVEL || 'info',
|
||||||
|
prefix(level: string) {
|
||||||
|
const req = (requests > 0) ? ` REQ-${requests}` : '';
|
||||||
|
return `${toIsoString(new Date())} ${level.toUpperCase()}${req}`
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
}
|
||||||
166
src/services/sessions.ts
Normal file
166
src/services/sessions.ts
Normal file
@@ -0,0 +1,166 @@
|
|||||||
|
import {v1 as UUIDv1} from 'uuid'
|
||||||
|
import * as path from 'path'
|
||||||
|
import {SetCookie, Browser} from 'puppeteer'
|
||||||
|
|
||||||
|
import log from './log'
|
||||||
|
import {Proxy} from "../controllers/v1";
|
||||||
|
|
||||||
|
const os = require('os');
|
||||||
|
const fs = require('fs');
|
||||||
|
const puppeteer = require('puppeteer');
|
||||||
|
|
||||||
|
export interface SessionsCacheItem {
|
||||||
|
sessionId: string
|
||||||
|
browser: Browser
|
||||||
|
}
|
||||||
|
|
||||||
|
interface SessionsCache {
|
||||||
|
[key: string]: SessionsCacheItem
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SessionCreateOptions {
|
||||||
|
oneTimeSession: boolean
|
||||||
|
cookies?: SetCookie[],
|
||||||
|
maxTimeout?: number
|
||||||
|
proxy?: Proxy
|
||||||
|
}
|
||||||
|
|
||||||
|
const sessionCache: SessionsCache = {}
|
||||||
|
let webBrowserUserAgent: string;
|
||||||
|
|
||||||
|
function buildExtraPrefsFirefox(proxy: Proxy): object {
|
||||||
|
// Default configurations are defined here
|
||||||
|
// https://github.com/puppeteer/puppeteer/blob/v3.3.0/src/Launcher.ts#L481
|
||||||
|
const extraPrefsFirefox = {
|
||||||
|
// Disable newtabpage
|
||||||
|
"browser.newtabpage.enabled": false,
|
||||||
|
"browser.startup.homepage": "about:blank",
|
||||||
|
|
||||||
|
// Do not warn when closing all open tabs
|
||||||
|
"browser.tabs.warnOnClose": false,
|
||||||
|
|
||||||
|
// Disable telemetry
|
||||||
|
"toolkit.telemetry.reportingpolicy.firstRun": false,
|
||||||
|
|
||||||
|
// Disable first-run welcome page
|
||||||
|
"startup.homepage_welcome_url": "about:blank",
|
||||||
|
"startup.homepage_welcome_url.additional": "",
|
||||||
|
|
||||||
|
// Disable images to speed up load
|
||||||
|
"permissions.default.image": 2,
|
||||||
|
|
||||||
|
// Limit content processes to 1
|
||||||
|
"dom.ipc.processCount": 1
|
||||||
|
}
|
||||||
|
|
||||||
|
// proxy.url format => http://<host>:<port>
|
||||||
|
if (proxy && proxy.url) {
|
||||||
|
const [host, portStr] = proxy.url.replace(/.+:\/\//g, '').split(':');
|
||||||
|
const port = parseInt(portStr);
|
||||||
|
|
||||||
|
const proxyPrefs = {
|
||||||
|
// Proxy configuration
|
||||||
|
"network.proxy.ftp": host,
|
||||||
|
"network.proxy.ftp_port": port,
|
||||||
|
"network.proxy.http": host,
|
||||||
|
"network.proxy.http_port": port,
|
||||||
|
"network.proxy.share_proxy_settings": true,
|
||||||
|
"network.proxy.socks": host,
|
||||||
|
"network.proxy.socks_port": port,
|
||||||
|
"network.proxy.socks_remote_dns": true,
|
||||||
|
"network.proxy.ssl": host,
|
||||||
|
"network.proxy.ssl_port": port,
|
||||||
|
"network.proxy.type": 1
|
||||||
|
}
|
||||||
|
|
||||||
|
// merge objects
|
||||||
|
Object.assign(extraPrefsFirefox, proxyPrefs);
|
||||||
|
}
|
||||||
|
|
||||||
|
return extraPrefsFirefox;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getUserAgent() {
|
||||||
|
return webBrowserUserAgent
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function testWebBrowserInstallation(): Promise<void> {
|
||||||
|
log.info("Testing web browser installation...")
|
||||||
|
|
||||||
|
// check user home dir. this dir will be used by Firefox
|
||||||
|
const homeDir = os.homedir();
|
||||||
|
fs.accessSync(homeDir, fs.constants.F_OK | fs.constants.R_OK | fs.constants.W_OK | fs.constants.X_OK);
|
||||||
|
log.debug("FlareSolverr user home directory is OK: " + homeDir)
|
||||||
|
|
||||||
|
// test web browser
|
||||||
|
const session = await create(null, {
|
||||||
|
oneTimeSession: true
|
||||||
|
})
|
||||||
|
const page = await session.browser.newPage()
|
||||||
|
await page.goto("https://www.google.com")
|
||||||
|
webBrowserUserAgent = await page.evaluate(() => navigator.userAgent)
|
||||||
|
|
||||||
|
// replace Linux ARM user-agent because it's detected
|
||||||
|
if (webBrowserUserAgent.toLocaleLowerCase().includes('linux arm')) {
|
||||||
|
webBrowserUserAgent = webBrowserUserAgent.replace(/linux arm[^;]+;/i, 'Linux x86_64;')
|
||||||
|
}
|
||||||
|
|
||||||
|
log.info("FlareSolverr User-Agent: " + webBrowserUserAgent)
|
||||||
|
await page.close()
|
||||||
|
await destroy(session.sessionId)
|
||||||
|
|
||||||
|
log.info("Test successful")
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function create(session: string, options: SessionCreateOptions): Promise<SessionsCacheItem> {
|
||||||
|
const sessionId = session || UUIDv1()
|
||||||
|
|
||||||
|
// NOTE: cookies can't be set in the session, you need to open the page first
|
||||||
|
|
||||||
|
const puppeteerOptions: any = {
|
||||||
|
product: 'firefox',
|
||||||
|
headless: process.env.HEADLESS !== 'false',
|
||||||
|
timeout: process.env.BROWSER_TIMEOUT || 30000
|
||||||
|
}
|
||||||
|
|
||||||
|
puppeteerOptions.extraPrefsFirefox = buildExtraPrefsFirefox(options.proxy)
|
||||||
|
|
||||||
|
// if we are running inside executable binary, change browser path
|
||||||
|
if (typeof (process as any).pkg !== 'undefined') {
|
||||||
|
const exe = process.platform === "win32" ? 'firefox.exe' : 'firefox';
|
||||||
|
puppeteerOptions.executablePath = path.join(path.dirname(process.execPath), 'firefox', exe)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.debug('Launching web browser...')
|
||||||
|
let browser: Browser = await puppeteer.launch(puppeteerOptions)
|
||||||
|
if (!browser) {
|
||||||
|
throw Error(`Failed to launch web browser.`)
|
||||||
|
}
|
||||||
|
|
||||||
|
sessionCache[sessionId] = {
|
||||||
|
sessionId: sessionId,
|
||||||
|
browser: browser
|
||||||
|
}
|
||||||
|
|
||||||
|
return sessionCache[sessionId]
|
||||||
|
}
|
||||||
|
|
||||||
|
export function list(): string[] {
|
||||||
|
return Object.keys(sessionCache)
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function destroy(id: string): Promise<boolean>{
|
||||||
|
if (id && sessionCache.hasOwnProperty(id)) {
|
||||||
|
const { browser } = sessionCache[id]
|
||||||
|
if (browser) {
|
||||||
|
await browser.close()
|
||||||
|
delete sessionCache[id]
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
export function get(id: string): SessionsCacheItem {
|
||||||
|
return sessionCache[id]
|
||||||
|
}
|
||||||
216
src/services/solver.ts
Normal file
216
src/services/solver.ts
Normal file
@@ -0,0 +1,216 @@
|
|||||||
|
import {Response, Headers, Page} from 'puppeteer'
|
||||||
|
const Timeout = require('await-timeout');
|
||||||
|
|
||||||
|
import log from './log'
|
||||||
|
import {SessionCreateOptions, SessionsCacheItem} from "./sessions";
|
||||||
|
import {V1Request} from "../controllers/v1";
|
||||||
|
import cloudflareProvider from '../providers/cloudflare';
|
||||||
|
|
||||||
|
const sessions = require('./sessions')
|
||||||
|
|
||||||
|
export interface ChallengeResolutionResultT {
|
||||||
|
url: string
|
||||||
|
status: number,
|
||||||
|
headers?: Headers,
|
||||||
|
response: string,
|
||||||
|
cookies: object[]
|
||||||
|
userAgent: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ChallengeResolutionT {
|
||||||
|
status?: string
|
||||||
|
message: string
|
||||||
|
result: ChallengeResolutionResultT
|
||||||
|
}
|
||||||
|
|
||||||
|
async function resolveChallengeWithTimeout(params: V1Request, session: SessionsCacheItem) {
|
||||||
|
const timer = new Timeout();
|
||||||
|
try {
|
||||||
|
const promise = resolveChallenge(params, session);
|
||||||
|
return await Promise.race([
|
||||||
|
promise,
|
||||||
|
timer.set(params.maxTimeout, `Maximum timeout reached. maxTimeout=${params.maxTimeout} (ms)`)
|
||||||
|
]);
|
||||||
|
} finally {
|
||||||
|
timer.clear();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function resolveChallenge(params: V1Request, session: SessionsCacheItem): Promise<ChallengeResolutionT | void> {
|
||||||
|
try {
|
||||||
|
let status = 'ok'
|
||||||
|
let message = ''
|
||||||
|
|
||||||
|
const page: Page = await session.browser.newPage()
|
||||||
|
|
||||||
|
// the Puppeter timeout should be half the maxTimeout because we reload the page and wait for challenge
|
||||||
|
// the user can set a really high maxTimeout if he wants to
|
||||||
|
await page.setDefaultNavigationTimeout(params.maxTimeout / 2)
|
||||||
|
|
||||||
|
// the user-agent is changed just for linux arm build
|
||||||
|
await page.setUserAgent(sessions.getUserAgent())
|
||||||
|
|
||||||
|
// set the proxy
|
||||||
|
if (params.proxy) {
|
||||||
|
log.debug(`Using proxy: ${params.proxy.url}`);
|
||||||
|
// todo: credentials are not working
|
||||||
|
// if (params.proxy.username) {
|
||||||
|
// await page.authenticate({
|
||||||
|
// username: params.proxy.username,
|
||||||
|
// password: params.proxy.password
|
||||||
|
// });
|
||||||
|
// }
|
||||||
|
}
|
||||||
|
|
||||||
|
// go to the page
|
||||||
|
log.debug(`Navigating to... ${params.url}`)
|
||||||
|
let response: Response = await gotoPage(params, page);
|
||||||
|
|
||||||
|
// set cookies
|
||||||
|
if (params.cookies) {
|
||||||
|
for (const cookie of params.cookies) {
|
||||||
|
// the other fields in the cookie can cause issues
|
||||||
|
await page.setCookie({
|
||||||
|
"name": cookie.name,
|
||||||
|
"value": cookie.value
|
||||||
|
})
|
||||||
|
}
|
||||||
|
// reload the page
|
||||||
|
response = await gotoPage(params, page);
|
||||||
|
}
|
||||||
|
|
||||||
|
// log html in debug mode
|
||||||
|
log.html(await page.content())
|
||||||
|
|
||||||
|
// detect protection services and solve challenges
|
||||||
|
try {
|
||||||
|
response = await cloudflareProvider(params.url, page, response);
|
||||||
|
|
||||||
|
// is response is ok
|
||||||
|
// reload the page to be sure we get the real page
|
||||||
|
log.debug("Reloading the page")
|
||||||
|
response = await gotoPage(params, page);
|
||||||
|
|
||||||
|
} catch (e) {
|
||||||
|
status = "error";
|
||||||
|
message = "Cloudflare " + e.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
const payload: ChallengeResolutionT = {
|
||||||
|
status,
|
||||||
|
message,
|
||||||
|
result: {
|
||||||
|
url: page.url(),
|
||||||
|
status: response.status(),
|
||||||
|
headers: response.headers(),
|
||||||
|
response: null,
|
||||||
|
cookies: await page.cookies(),
|
||||||
|
userAgent: sessions.getUserAgent()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (params.returnOnlyCookies) {
|
||||||
|
payload.result.headers = null;
|
||||||
|
payload.result.userAgent = null;
|
||||||
|
} else {
|
||||||
|
payload.result.response = await page.content()
|
||||||
|
}
|
||||||
|
|
||||||
|
// make sure the page is closed because if it isn't and error will be thrown
|
||||||
|
// when a user uses a temporary session, the browser make be quit before
|
||||||
|
// the page is properly closed.
|
||||||
|
await page.close()
|
||||||
|
|
||||||
|
return payload
|
||||||
|
} catch (e) {
|
||||||
|
log.error("Unexpected error: " + e);
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function gotoPage(params: V1Request, page: Page): Promise<Response> {
|
||||||
|
let response: Response;
|
||||||
|
if (params.method != 'POST') {
|
||||||
|
response = await page.goto(params.url, {waitUntil: 'domcontentloaded'});
|
||||||
|
|
||||||
|
} else {
|
||||||
|
// post hack
|
||||||
|
// first request a page without cloudflare
|
||||||
|
response = await page.goto(params.url, {waitUntil: 'domcontentloaded'});
|
||||||
|
await page.setContent(
|
||||||
|
`
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<html>
|
||||||
|
<body>
|
||||||
|
<script>
|
||||||
|
|
||||||
|
function parseQuery(queryString) {
|
||||||
|
var query = {};
|
||||||
|
var pairs = (queryString[0] === '?' ? queryString.substr(1) : queryString).split('&');
|
||||||
|
for (var i = 0; i < pairs.length; i++) {
|
||||||
|
var pair = pairs[i].split('=');
|
||||||
|
query[decodeURIComponent(pair[0])] = decodeURIComponent(pair[1] || '');
|
||||||
|
}
|
||||||
|
return query;
|
||||||
|
}
|
||||||
|
|
||||||
|
const form = document.createElement('form');
|
||||||
|
form.method = 'POST';
|
||||||
|
form.action = '${params.url}';
|
||||||
|
|
||||||
|
const params = parseQuery('${params.postData}');
|
||||||
|
for (const key in params) {
|
||||||
|
if (params.hasOwnProperty(key)) {
|
||||||
|
const hiddenField = document.createElement('input');
|
||||||
|
hiddenField.type = 'hidden';
|
||||||
|
hiddenField.name = key;
|
||||||
|
hiddenField.value = params[key];
|
||||||
|
form.appendChild(hiddenField);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
document.body.appendChild(form);
|
||||||
|
form.submit();
|
||||||
|
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
`
|
||||||
|
);
|
||||||
|
await page.waitFor(2000)
|
||||||
|
try {
|
||||||
|
await page.waitForNavigation({waitUntil: 'domcontentloaded', timeout: 2000})
|
||||||
|
} catch (e) {}
|
||||||
|
|
||||||
|
}
|
||||||
|
return response
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function browserRequest(params: V1Request): Promise<ChallengeResolutionT> {
|
||||||
|
const oneTimeSession = params.session === undefined;
|
||||||
|
|
||||||
|
const options: SessionCreateOptions = {
|
||||||
|
oneTimeSession: oneTimeSession,
|
||||||
|
cookies: params.cookies,
|
||||||
|
maxTimeout: params.maxTimeout,
|
||||||
|
proxy: params.proxy
|
||||||
|
}
|
||||||
|
|
||||||
|
const session: SessionsCacheItem = oneTimeSession
|
||||||
|
? await sessions.create(null, options)
|
||||||
|
: sessions.get(params.session)
|
||||||
|
|
||||||
|
if (!session) {
|
||||||
|
throw Error('This session does not exist. Use \'list_sessions\' to see all the existing sessions.')
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
return await resolveChallengeWithTimeout(params, session)
|
||||||
|
} catch (error) {
|
||||||
|
throw Error("Unable to process browser request. " + error)
|
||||||
|
} finally {
|
||||||
|
if (oneTimeSession) {
|
||||||
|
await sessions.destroy(session.sessionId)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
140
src/session.ts
140
src/session.ts
@@ -1,140 +0,0 @@
|
|||||||
import * as os from 'os'
|
|
||||||
import * as path from 'path'
|
|
||||||
import * as fs from 'fs'
|
|
||||||
|
|
||||||
import puppeteer from 'puppeteer-extra'
|
|
||||||
import { LaunchOptions, Headers, SetCookie } from 'puppeteer'
|
|
||||||
|
|
||||||
import log from './log'
|
|
||||||
import { deleteFolderRecursive, sleep, removeEmptyFields } from './utils'
|
|
||||||
import * as Puppeteer from "puppeteer-extra/dist/puppeteer";
|
|
||||||
|
|
||||||
interface SessionPageDefaults {
|
|
||||||
headers?: Headers
|
|
||||||
userAgent?: string
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface SessionsCacheItem {
|
|
||||||
browser: Puppeteer.Browser
|
|
||||||
userDataDir?: string
|
|
||||||
defaults: SessionPageDefaults
|
|
||||||
}
|
|
||||||
|
|
||||||
interface SessionsCache {
|
|
||||||
[key: string]: SessionsCacheItem
|
|
||||||
}
|
|
||||||
|
|
||||||
interface SessionCreateOptions {
|
|
||||||
oneTimeSession?: boolean
|
|
||||||
userAgent?: string
|
|
||||||
cookies?: SetCookie[]
|
|
||||||
headers?: Headers,
|
|
||||||
maxTimeout?: number
|
|
||||||
proxy?: any
|
|
||||||
}
|
|
||||||
|
|
||||||
const sessionCache: SessionsCache = {}
|
|
||||||
|
|
||||||
// setting "user-agent-override" evasion is not working for us because it can't be changed
|
|
||||||
// in each request. we set the user-agent in the browser args instead
|
|
||||||
puppeteer.use(require('puppeteer-extra-plugin-stealth')())
|
|
||||||
|
|
||||||
function userDataDirFromId(id: string): string {
|
|
||||||
return path.join(os.tmpdir(), `/puppeteer_chrome_profile_${id}`)
|
|
||||||
}
|
|
||||||
|
|
||||||
function prepareBrowserProfile(id: string): string {
|
|
||||||
// TODO: maybe pass SessionCreateOptions for loading later?
|
|
||||||
const userDataDir = userDataDirFromId(id)
|
|
||||||
|
|
||||||
if (!fs.existsSync(userDataDir)) {
|
|
||||||
fs.mkdirSync(userDataDir, { recursive: true })
|
|
||||||
}
|
|
||||||
|
|
||||||
return userDataDir
|
|
||||||
}
|
|
||||||
|
|
||||||
export default {
|
|
||||||
create: async (id: string, { cookies, oneTimeSession, userAgent, headers, maxTimeout, proxy }: SessionCreateOptions): Promise<SessionsCacheItem> => {
|
|
||||||
let args = ['--no-sandbox', '--disable-setuid-sandbox'];
|
|
||||||
if (proxy && proxy.url) {
|
|
||||||
args.push(`--proxy-server=${proxy.url}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const puppeteerOptions: LaunchOptions = {
|
|
||||||
product: 'chrome',
|
|
||||||
headless: process.env.HEADLESS !== 'false',
|
|
||||||
args
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!oneTimeSession) {
|
|
||||||
log.debug('Creating userDataDir for session.')
|
|
||||||
puppeteerOptions.userDataDir = prepareBrowserProfile(id)
|
|
||||||
}
|
|
||||||
|
|
||||||
log.debug('Launching headless browser...')
|
|
||||||
|
|
||||||
// TODO: maybe access env variable?
|
|
||||||
// TODO: sometimes browser instances are created and not connected to correctly.
|
|
||||||
// how do we handle/quit those instances inside Docker?
|
|
||||||
let launchTries = 3
|
|
||||||
let browser: Puppeteer.Browser;
|
|
||||||
|
|
||||||
while (0 <= launchTries--) {
|
|
||||||
try {
|
|
||||||
browser = await puppeteer.launch(puppeteerOptions)
|
|
||||||
break
|
|
||||||
} catch (e) {
|
|
||||||
if (e.message !== 'Failed to launch the browser process!')
|
|
||||||
throw e
|
|
||||||
log.warn('Failed to open browser, trying again...')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!browser) { throw Error(`Failed to launch browser 3 times in a row.`) }
|
|
||||||
|
|
||||||
if (cookies) {
|
|
||||||
const page = await browser.newPage()
|
|
||||||
await page.setCookie(...cookies)
|
|
||||||
}
|
|
||||||
|
|
||||||
sessionCache[id] = {
|
|
||||||
browser: browser,
|
|
||||||
userDataDir: puppeteerOptions.userDataDir,
|
|
||||||
defaults: removeEmptyFields({
|
|
||||||
userAgent,
|
|
||||||
headers,
|
|
||||||
maxTimeout
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
return sessionCache[id]
|
|
||||||
},
|
|
||||||
|
|
||||||
list: (): string[] => Object.keys(sessionCache),
|
|
||||||
|
|
||||||
// TODO: create a sessions.close that doesn't rm the userDataDir
|
|
||||||
|
|
||||||
destroy: async (id: string): Promise<boolean> => {
|
|
||||||
const { browser, userDataDir } = sessionCache[id]
|
|
||||||
if (browser) {
|
|
||||||
await browser.close()
|
|
||||||
delete sessionCache[id]
|
|
||||||
if (userDataDir) {
|
|
||||||
const userDataDirPath = userDataDirFromId(id)
|
|
||||||
try {
|
|
||||||
// for some reason this keeps an error from being thrown in Windows, figures
|
|
||||||
await sleep(5000)
|
|
||||||
deleteFolderRecursive(userDataDirPath)
|
|
||||||
} catch (e) {
|
|
||||||
console.error(e)
|
|
||||||
throw Error(`Error deleting browser session folder. ${e.message}`)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
},
|
|
||||||
|
|
||||||
get: (id: string): SessionsCacheItem | false => sessionCache[id] && sessionCache[id] || false
|
|
||||||
}
|
|
||||||
538
src/tests/app.test.ts
Normal file
538
src/tests/app.test.ts
Normal file
@@ -0,0 +1,538 @@
|
|||||||
|
// noinspection DuplicatedCode
|
||||||
|
|
||||||
|
import {Response} from "superagent";
|
||||||
|
import {V1ResponseBase, V1ResponseSession, V1ResponseSessions, V1ResponseSolution} from "../controllers/v1"
|
||||||
|
|
||||||
|
const request = require("supertest");
|
||||||
|
const app = require("../app");
|
||||||
|
const sessions = require('../services/sessions');
|
||||||
|
const version: string = 'v' + require('../../package.json').version
|
||||||
|
|
||||||
|
const proxyUrl = "http://127.0.0.1:8888"
|
||||||
|
const googleUrl = "https://www.google.com";
|
||||||
|
const postUrl = "https://ptsv2.com/t/qv4j3-1634496523";
|
||||||
|
const cfUrl = "https://pirateiro.com/torrents/?search=harry";
|
||||||
|
const cfCaptchaUrl = "https://idope.se"
|
||||||
|
const cfBlockedUrl = "https://www.torrentmafya.org/table.php"
|
||||||
|
|
||||||
|
beforeAll(async () => {
|
||||||
|
// Init session
|
||||||
|
await sessions.testWebBrowserInstallation();
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(async () => {
|
||||||
|
// Clean sessions
|
||||||
|
const sessionList = sessions.list();
|
||||||
|
for (const session of sessionList) {
|
||||||
|
await sessions.destroy(session);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("Test '/' path", () => {
|
||||||
|
test("GET method should return OK ", async () => {
|
||||||
|
const response: Response = await request(app).get("/");
|
||||||
|
expect(response.statusCode).toBe(200);
|
||||||
|
expect(response.body.msg).toBe("FlareSolverr is ready!");
|
||||||
|
expect(response.body.version).toBe(version);
|
||||||
|
expect(response.body.userAgent).toContain("Firefox/")
|
||||||
|
});
|
||||||
|
|
||||||
|
test("POST method should fail", async () => {
|
||||||
|
const response: Response = await request(app).post("/");
|
||||||
|
expect(response.statusCode).toBe(404);
|
||||||
|
expect(response.body.error).toBe("Unknown resource or HTTP verb");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("Test '/health' path", () => {
|
||||||
|
test("GET method should return OK", async () => {
|
||||||
|
const response: Response = await request(app).get("/health");
|
||||||
|
expect(response.statusCode).toBe(200);
|
||||||
|
expect(response.body.status).toBe("ok");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("Test '/wrong' path", () => {
|
||||||
|
test("GET method should fail", async () => {
|
||||||
|
const response: Response = await request(app).get("/wrong");
|
||||||
|
expect(response.statusCode).toBe(404);
|
||||||
|
expect(response.body.error).toBe("Unknown resource or HTTP verb");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("Test '/v1' path", () => {
|
||||||
|
test("Cmd 'request.bad' should fail", async () => {
|
||||||
|
const payload = {
|
||||||
|
"cmd": "request.bad",
|
||||||
|
"url": googleUrl
|
||||||
|
}
|
||||||
|
const response: Response = await request(app).post("/v1").send(payload);
|
||||||
|
expect(response.statusCode).toBe(500);
|
||||||
|
|
||||||
|
const apiResponse: V1ResponseBase = response.body;
|
||||||
|
expect(apiResponse.status).toBe("error");
|
||||||
|
expect(apiResponse.message).toBe("Error: The command 'request.bad' is invalid.");
|
||||||
|
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
|
||||||
|
expect(apiResponse.endTimestamp).toBeGreaterThanOrEqual(apiResponse.startTimestamp);
|
||||||
|
expect(apiResponse.version).toBe(version);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Cmd 'request.get' should return OK with no Cloudflare", async () => {
|
||||||
|
const payload = {
|
||||||
|
"cmd": "request.get",
|
||||||
|
"url": googleUrl
|
||||||
|
}
|
||||||
|
const response: Response = await request(app).post("/v1").send(payload);
|
||||||
|
expect(response.statusCode).toBe(200);
|
||||||
|
|
||||||
|
const apiResponse: V1ResponseSolution = response.body;
|
||||||
|
expect(apiResponse.status).toBe("ok");
|
||||||
|
expect(apiResponse.message).toBe("");
|
||||||
|
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
|
||||||
|
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
|
||||||
|
expect(apiResponse.version).toBe(version);
|
||||||
|
|
||||||
|
const solution = apiResponse.solution;
|
||||||
|
expect(solution.url).toContain(googleUrl)
|
||||||
|
expect(solution.status).toBe(200);
|
||||||
|
expect(Object.keys(solution.headers).length).toBeGreaterThan(0)
|
||||||
|
expect(solution.response).toContain("<!DOCTYPE html>")
|
||||||
|
expect(Object.keys(solution.cookies).length).toBeGreaterThan(0)
|
||||||
|
expect(solution.userAgent).toContain("Firefox/")
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Cmd 'request.get' should return OK with Cloudflare JS", async () => {
|
||||||
|
const payload = {
|
||||||
|
"cmd": "request.get",
|
||||||
|
"url": cfUrl
|
||||||
|
}
|
||||||
|
const response: Response = await request(app).post("/v1").send(payload);
|
||||||
|
expect(response.statusCode).toBe(200);
|
||||||
|
|
||||||
|
const apiResponse: V1ResponseSolution = response.body;
|
||||||
|
expect(apiResponse.status).toBe("ok");
|
||||||
|
expect(apiResponse.message).toBe("");
|
||||||
|
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
|
||||||
|
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
|
||||||
|
expect(apiResponse.version).toBe(version);
|
||||||
|
|
||||||
|
const solution = apiResponse.solution;
|
||||||
|
expect(solution.url).toContain(cfUrl)
|
||||||
|
expect(solution.status).toBe(200);
|
||||||
|
expect(Object.keys(solution.headers).length).toBeGreaterThan(0)
|
||||||
|
expect(solution.response).toContain("<!DOCTYPE html>")
|
||||||
|
expect(Object.keys(solution.cookies).length).toBeGreaterThan(0)
|
||||||
|
expect(solution.userAgent).toContain("Firefox/")
|
||||||
|
|
||||||
|
const cfCookie: string = (solution.cookies as any[]).filter(function(cookie) {
|
||||||
|
return cookie.name == "cf_clearance";
|
||||||
|
})[0].value
|
||||||
|
expect(cfCookie.length).toBeGreaterThan(30)
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Cmd 'request.get' should return fail with Cloudflare CAPTCHA", async () => {
|
||||||
|
const payload = {
|
||||||
|
"cmd": "request.get",
|
||||||
|
"url": cfCaptchaUrl
|
||||||
|
}
|
||||||
|
const response: Response = await request(app).post("/v1").send(payload);
|
||||||
|
expect(response.statusCode).toBe(200);
|
||||||
|
|
||||||
|
const apiResponse: V1ResponseSolution = response.body;
|
||||||
|
expect(apiResponse.status).toBe("error");
|
||||||
|
expect(apiResponse.message).toBe("Cloudflare Error: FlareSolverr can not resolve CAPTCHA challenges. Since the captcha doesn't always appear, you may have better luck with the next request.");
|
||||||
|
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
|
||||||
|
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
|
||||||
|
expect(apiResponse.version).toBe(version);
|
||||||
|
// solution is filled but not useful
|
||||||
|
expect(apiResponse.solution.url).toContain(cfCaptchaUrl)
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Cmd 'request.post' should return fail with Cloudflare Blocked", async () => {
|
||||||
|
const payload = {
|
||||||
|
"cmd": "request.post",
|
||||||
|
"url": cfBlockedUrl,
|
||||||
|
"postData": "test1=test2"
|
||||||
|
}
|
||||||
|
const response: Response = await request(app).post("/v1").send(payload);
|
||||||
|
expect(response.statusCode).toBe(200);
|
||||||
|
|
||||||
|
const apiResponse: V1ResponseSolution = response.body;
|
||||||
|
expect(apiResponse.status).toBe("error");
|
||||||
|
expect(apiResponse.message).toBe("Cloudflare Error: Cloudflare has blocked this request. Probably your IP is banned for this site, check in your web browser.");
|
||||||
|
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
|
||||||
|
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
|
||||||
|
expect(apiResponse.version).toBe(version);
|
||||||
|
// solution is filled but not useful
|
||||||
|
expect(apiResponse.solution.url).toContain(cfBlockedUrl)
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Cmd 'request.get' should return OK with 'cookies' param", async () => {
|
||||||
|
const payload = {
|
||||||
|
"cmd": "request.get",
|
||||||
|
"url": googleUrl,
|
||||||
|
"cookies": [
|
||||||
|
{
|
||||||
|
"name": "testcookie1",
|
||||||
|
"value": "testvalue1"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "testcookie2",
|
||||||
|
"value": "testvalue2"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
const response: Response = await request(app).post("/v1").send(payload);
|
||||||
|
expect(response.statusCode).toBe(200);
|
||||||
|
|
||||||
|
const apiResponse: V1ResponseSolution = response.body;
|
||||||
|
expect(apiResponse.status).toBe("ok");
|
||||||
|
|
||||||
|
const solution = apiResponse.solution;
|
||||||
|
expect(solution.url).toContain(googleUrl)
|
||||||
|
expect(Object.keys(solution.cookies).length).toBeGreaterThan(1)
|
||||||
|
const cookie1: string = (solution.cookies as any[]).filter(function(cookie) {
|
||||||
|
return cookie.name == "testcookie1";
|
||||||
|
})[0].value
|
||||||
|
expect(cookie1).toBe("testvalue1")
|
||||||
|
const cookie2: string = (solution.cookies as any[]).filter(function(cookie) {
|
||||||
|
return cookie.name == "testcookie2";
|
||||||
|
})[0].value
|
||||||
|
expect(cookie2).toBe("testvalue2")
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Cmd 'request.get' should return OK with 'returnOnlyCookies' param", async () => {
|
||||||
|
const payload = {
|
||||||
|
"cmd": "request.get",
|
||||||
|
"url": googleUrl,
|
||||||
|
"returnOnlyCookies": true
|
||||||
|
}
|
||||||
|
const response: Response = await request(app).post("/v1").send(payload);
|
||||||
|
expect(response.statusCode).toBe(200);
|
||||||
|
|
||||||
|
const apiResponse: V1ResponseSolution = response.body;
|
||||||
|
|
||||||
|
const solution = apiResponse.solution;
|
||||||
|
expect(solution.url).toContain(googleUrl)
|
||||||
|
expect(solution.status).toBe(200);
|
||||||
|
expect(solution.headers).toBe(null)
|
||||||
|
expect(solution.response).toBe(null)
|
||||||
|
expect(Object.keys(solution.cookies).length).toBeGreaterThan(0)
|
||||||
|
expect(solution.userAgent).toBe(null)
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Cmd 'request.get' should return OK with 'proxy' param", async () => {
|
||||||
|
/*
|
||||||
|
To configure TinyProxy in local:
|
||||||
|
* sudo vim /etc/tinyproxy/tinyproxy.conf
|
||||||
|
* edit => LogFile "/tmp/tinyproxy.log"
|
||||||
|
* edit => Syslog Off
|
||||||
|
* sudo tinyproxy -d
|
||||||
|
* sudo tail -f /tmp/tinyproxy.log
|
||||||
|
*/
|
||||||
|
const payload = {
|
||||||
|
"cmd": "request.get",
|
||||||
|
"url": googleUrl,
|
||||||
|
"proxy": {
|
||||||
|
"url": proxyUrl
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const response: Response = await request(app).post("/v1").send(payload);
|
||||||
|
expect(response.statusCode).toBe(200);
|
||||||
|
|
||||||
|
const apiResponse: V1ResponseSolution = response.body;
|
||||||
|
expect(apiResponse.status).toBe("ok");
|
||||||
|
|
||||||
|
const solution = apiResponse.solution;
|
||||||
|
expect(solution.url).toContain(googleUrl)
|
||||||
|
expect(solution.status).toBe(200);
|
||||||
|
});
|
||||||
|
|
||||||
|
// todo: credentials are not working
|
||||||
|
test.skip("Cmd 'request.get' should return OK with 'proxy' param with credentials", async () => {
|
||||||
|
/*
|
||||||
|
To configure TinyProxy in local:
|
||||||
|
* sudo vim /etc/tinyproxy/tinyproxy.conf
|
||||||
|
* edit => LogFile "/tmp/tinyproxy.log"
|
||||||
|
* edit => Syslog Off
|
||||||
|
* add => BasicAuth testuser testpass
|
||||||
|
* sudo tinyproxy -d
|
||||||
|
* sudo tail -f /tmp/tinyproxy.log
|
||||||
|
*/
|
||||||
|
const payload = {
|
||||||
|
"cmd": "request.get",
|
||||||
|
"url": googleUrl,
|
||||||
|
"proxy": {
|
||||||
|
"url": proxyUrl,
|
||||||
|
"username": "testuser",
|
||||||
|
"password": "testpass"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const response: Response = await request(app).post("/v1").send(payload);
|
||||||
|
expect(response.statusCode).toBe(200);
|
||||||
|
|
||||||
|
const apiResponse: V1ResponseSolution = response.body;
|
||||||
|
expect(apiResponse.status).toBe("ok");
|
||||||
|
|
||||||
|
const solution = apiResponse.solution;
|
||||||
|
expect(solution.url).toContain(googleUrl)
|
||||||
|
expect(solution.status).toContain(200)
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Cmd 'request.get' should fail with wrong 'proxy' param", async () => {
|
||||||
|
const payload = {
|
||||||
|
"cmd": "request.get",
|
||||||
|
"url": googleUrl,
|
||||||
|
"proxy": {
|
||||||
|
"url": "http://127.0.0.1:43210"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const response: Response = await request(app).post("/v1").send(payload);
|
||||||
|
expect(response.statusCode).toBe(500);
|
||||||
|
|
||||||
|
const apiResponse: V1ResponseSolution = response.body;
|
||||||
|
expect(apiResponse.status).toBe("error");
|
||||||
|
expect(apiResponse.message).toBe("Error: Unable to process browser request. Error: NS_ERROR_PROXY_CONNECTION_REFUSED at https://www.google.com");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Cmd 'request.get' should return fail with timeout", async () => {
|
||||||
|
const payload = {
|
||||||
|
"cmd": "request.get",
|
||||||
|
"url": googleUrl,
|
||||||
|
"maxTimeout": 10
|
||||||
|
}
|
||||||
|
const response: Response = await request(app).post("/v1").send(payload);
|
||||||
|
expect(response.statusCode).toBe(500);
|
||||||
|
|
||||||
|
const apiResponse: V1ResponseBase = response.body;
|
||||||
|
expect(apiResponse.status).toBe("error");
|
||||||
|
expect(apiResponse.message).toBe("Error: Unable to process browser request. Error: Maximum timeout reached. maxTimeout=10 (ms)");
|
||||||
|
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
|
||||||
|
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
|
||||||
|
expect(apiResponse.version).toBe(version);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Cmd 'request.get' should return fail with bad domain", async () => {
|
||||||
|
const payload = {
|
||||||
|
"cmd": "request.get",
|
||||||
|
"url": "https://www.google.combad"
|
||||||
|
}
|
||||||
|
const response: Response = await request(app).post("/v1").send(payload);
|
||||||
|
expect(response.statusCode).toBe(500);
|
||||||
|
|
||||||
|
const apiResponse: V1ResponseBase = response.body;
|
||||||
|
expect(apiResponse.status).toBe("error");
|
||||||
|
expect(apiResponse.message).toBe("Error: Unable to process browser request. Error: NS_ERROR_UNKNOWN_HOST at https://www.google.combad");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Cmd 'request.get' should accept deprecated params", async () => {
|
||||||
|
const payload = {
|
||||||
|
"cmd": "request.get",
|
||||||
|
"url": googleUrl,
|
||||||
|
"userAgent": "Test User-Agent" // was removed in v2, not used
|
||||||
|
}
|
||||||
|
const response: Response = await request(app).post("/v1").send(payload);
|
||||||
|
expect(response.statusCode).toBe(200);
|
||||||
|
|
||||||
|
const apiResponse: V1ResponseSolution = response.body;
|
||||||
|
expect(apiResponse.status).toBe("ok");
|
||||||
|
|
||||||
|
const solution = apiResponse.solution;
|
||||||
|
expect(solution.url).toContain(googleUrl)
|
||||||
|
expect(solution.status).toBe(200);
|
||||||
|
expect(solution.userAgent).toContain("Firefox/")
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Cmd 'request.post' should return OK with no Cloudflare", async () => {
|
||||||
|
const payload = {
|
||||||
|
"cmd": "request.post",
|
||||||
|
"url": postUrl + '/post',
|
||||||
|
"postData": "param1=value1¶m2=value2"
|
||||||
|
}
|
||||||
|
const response: Response = await request(app).post("/v1").send(payload);
|
||||||
|
expect(response.statusCode).toBe(200);
|
||||||
|
|
||||||
|
const apiResponse: V1ResponseSolution = response.body;
|
||||||
|
expect(apiResponse.status).toBe("ok");
|
||||||
|
expect(apiResponse.message).toBe("");
|
||||||
|
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
|
||||||
|
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
|
||||||
|
expect(apiResponse.version).toBe(version);
|
||||||
|
|
||||||
|
const solution = apiResponse.solution;
|
||||||
|
expect(solution.url).toContain(postUrl)
|
||||||
|
expect(solution.status).toBe(200);
|
||||||
|
expect(Object.keys(solution.headers).length).toBeGreaterThan(0)
|
||||||
|
expect(solution.response).toContain(" I hope you have a lovely day!")
|
||||||
|
expect(Object.keys(solution.cookies).length).toBe(0)
|
||||||
|
expect(solution.userAgent).toContain("Firefox/")
|
||||||
|
|
||||||
|
// check that we sent the date
|
||||||
|
const payload2 = {
|
||||||
|
"cmd": "request.get",
|
||||||
|
"url": postUrl
|
||||||
|
}
|
||||||
|
const response2: Response = await request(app).post("/v1").send(payload2);
|
||||||
|
expect(response2.statusCode).toBe(200);
|
||||||
|
|
||||||
|
const apiResponse2: V1ResponseSolution = response2.body;
|
||||||
|
expect(apiResponse2.status).toBe("ok");
|
||||||
|
|
||||||
|
const solution2 = apiResponse2.solution;
|
||||||
|
expect(solution2.status).toBe(200);
|
||||||
|
expect(solution2.response).toContain(new Date().toISOString().split(':')[0].replace('T', ' '))
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Cmd 'request.post' should fail without 'postData' param", async () => {
|
||||||
|
const payload = {
|
||||||
|
"cmd": "request.post",
|
||||||
|
"url": googleUrl
|
||||||
|
}
|
||||||
|
const response: Response = await request(app).post("/v1").send(payload);
|
||||||
|
expect(response.statusCode).toBe(500);
|
||||||
|
|
||||||
|
const apiResponse: V1ResponseBase = response.body;
|
||||||
|
expect(apiResponse.status).toBe("error");
|
||||||
|
expect(apiResponse.message).toBe("Error: Must send param \"postBody\" when sending a POST request.");
|
||||||
|
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
|
||||||
|
expect(apiResponse.endTimestamp).toBeGreaterThanOrEqual(apiResponse.startTimestamp);
|
||||||
|
expect(apiResponse.version).toBe(version);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Cmd 'sessions.create' should return OK", async () => {
|
||||||
|
const payload = {
|
||||||
|
"cmd": "sessions.create"
|
||||||
|
}
|
||||||
|
const response: Response = await request(app).post("/v1").send(payload);
|
||||||
|
expect(response.statusCode).toBe(200);
|
||||||
|
|
||||||
|
const apiResponse: V1ResponseSession = response.body;
|
||||||
|
expect(apiResponse.status).toBe("ok");
|
||||||
|
expect(apiResponse.message).toBe("Session created successfully.");
|
||||||
|
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
|
||||||
|
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
|
||||||
|
expect(apiResponse.version).toBe(version);
|
||||||
|
expect(apiResponse.session.length).toBe(36);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Cmd 'sessions.create' should return OK with session", async () => {
|
||||||
|
const payload = {
|
||||||
|
"cmd": "sessions.create",
|
||||||
|
"session": "2bc6bb20-2f56-11ec-9543-test"
|
||||||
|
}
|
||||||
|
const response: Response = await request(app).post("/v1").send(payload);
|
||||||
|
expect(response.statusCode).toBe(200);
|
||||||
|
|
||||||
|
const apiResponse: V1ResponseSession = response.body;
|
||||||
|
expect(apiResponse.status).toBe("ok");
|
||||||
|
expect(apiResponse.message).toBe("Session created successfully.");
|
||||||
|
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
|
||||||
|
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
|
||||||
|
expect(apiResponse.version).toBe(version);
|
||||||
|
expect(apiResponse.session).toBe("2bc6bb20-2f56-11ec-9543-test");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Cmd 'sessions.list' should return OK", async () => {
|
||||||
|
// create one session for testing
|
||||||
|
const payload0 = {
|
||||||
|
"cmd": "sessions.create"
|
||||||
|
}
|
||||||
|
const response0: Response = await request(app).post("/v1").send(payload0);
|
||||||
|
expect(response0.statusCode).toBe(200);
|
||||||
|
|
||||||
|
const payload = {
|
||||||
|
"cmd": "sessions.list"
|
||||||
|
}
|
||||||
|
const response: Response = await request(app).post("/v1").send(payload);
|
||||||
|
expect(response.statusCode).toBe(200);
|
||||||
|
|
||||||
|
const apiResponse: V1ResponseSessions = response.body;
|
||||||
|
expect(apiResponse.status).toBe("ok");
|
||||||
|
expect(apiResponse.message).toBe("");
|
||||||
|
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
|
||||||
|
expect(apiResponse.endTimestamp).toBeGreaterThanOrEqual(apiResponse.startTimestamp);
|
||||||
|
expect(apiResponse.version).toBe(version);
|
||||||
|
expect(apiResponse.sessions.length).toBeGreaterThan(0)
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Cmd 'sessions.destroy' should return OK", async () => {
|
||||||
|
// create one session for testing
|
||||||
|
const payload0 = {
|
||||||
|
"cmd": "sessions.create"
|
||||||
|
}
|
||||||
|
const response0: Response = await request(app).post("/v1").send(payload0);
|
||||||
|
expect(response0.statusCode).toBe(200);
|
||||||
|
const apiResponse0: V1ResponseSession = response0.body;
|
||||||
|
const sessionId0 = apiResponse0.session
|
||||||
|
|
||||||
|
const payload = {
|
||||||
|
"cmd": "sessions.destroy",
|
||||||
|
"session": sessionId0
|
||||||
|
}
|
||||||
|
const response: Response = await request(app).post("/v1").send(payload);
|
||||||
|
expect(response.statusCode).toBe(200);
|
||||||
|
|
||||||
|
const apiResponse: V1ResponseBase = response.body;
|
||||||
|
expect(apiResponse.status).toBe("ok");
|
||||||
|
expect(apiResponse.message).toBe("The session has been removed.");
|
||||||
|
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
|
||||||
|
expect(apiResponse.endTimestamp).toBeGreaterThanOrEqual(apiResponse.startTimestamp);
|
||||||
|
expect(apiResponse.version).toBe(version);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Cmd 'sessions.destroy' should fail", async () => {
|
||||||
|
const payload = {
|
||||||
|
"cmd": "sessions.destroy",
|
||||||
|
"session": "bad-session"
|
||||||
|
}
|
||||||
|
const response: Response = await request(app).post("/v1").send(payload);
|
||||||
|
expect(response.statusCode).toBe(500);
|
||||||
|
|
||||||
|
const apiResponse: V1ResponseBase = response.body;
|
||||||
|
expect(apiResponse.status).toBe("error");
|
||||||
|
expect(apiResponse.message).toBe("Error: This session does not exist.");
|
||||||
|
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
|
||||||
|
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
|
||||||
|
expect(apiResponse.version).toBe(version);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("Cmd 'request.get' should use session", async () => {
|
||||||
|
// create one session for testing
|
||||||
|
const payload0 = {
|
||||||
|
"cmd": "sessions.create"
|
||||||
|
}
|
||||||
|
const response0: Response = await request(app).post("/v1").send(payload0);
|
||||||
|
expect(response0.statusCode).toBe(200);
|
||||||
|
const apiResponse0: V1ResponseSession = response0.body;
|
||||||
|
const sessionId0 = apiResponse0.session
|
||||||
|
|
||||||
|
// first request should solve the challenge
|
||||||
|
const payload = {
|
||||||
|
"cmd": "request.get",
|
||||||
|
"url": cfUrl,
|
||||||
|
"session": sessionId0
|
||||||
|
}
|
||||||
|
const response: Response = await request(app).post("/v1").send(payload);
|
||||||
|
expect(response.statusCode).toBe(200);
|
||||||
|
|
||||||
|
const apiResponse: V1ResponseSolution = response.body;
|
||||||
|
expect(apiResponse.status).toBe("ok");
|
||||||
|
const cfCookie: string = (apiResponse.solution.cookies as any[]).filter(function(cookie) {
|
||||||
|
return cookie.name == "cf_clearance";
|
||||||
|
})[0].value
|
||||||
|
expect(cfCookie.length).toBeGreaterThan(30)
|
||||||
|
|
||||||
|
// second request should have the same cookie
|
||||||
|
const response2: Response = await request(app).post("/v1").send(payload);
|
||||||
|
expect(response2.statusCode).toBe(200);
|
||||||
|
|
||||||
|
const apiResponse2: V1ResponseSolution = response2.body;
|
||||||
|
expect(apiResponse2.status).toBe("ok");
|
||||||
|
const cfCookie2: string = (apiResponse2.solution.cookies as any[]).filter(function(cookie) {
|
||||||
|
return cookie.name == "cf_clearance";
|
||||||
|
})[0].value
|
||||||
|
expect(cfCookie2.length).toBeGreaterThan(30)
|
||||||
|
expect(cfCookie2).toBe(cfCookie)
|
||||||
|
});
|
||||||
|
|
||||||
|
});
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
import { IncomingMessage, ServerResponse } from 'http';
|
|
||||||
|
|
||||||
export interface RequestContext {
|
|
||||||
req: IncomingMessage
|
|
||||||
res: ServerResponse
|
|
||||||
startTimestamp: number
|
|
||||||
errorResponse: (msg: string) => void,
|
|
||||||
successResponse: (msg: string, extendedProperties?: object) => void
|
|
||||||
}
|
|
||||||
31
src/utils.ts
31
src/utils.ts
@@ -1,31 +0,0 @@
|
|||||||
import * as fs from 'fs'
|
|
||||||
import * as Path from 'path'
|
|
||||||
import { promisify } from 'util'
|
|
||||||
|
|
||||||
export const sleep = promisify(setTimeout)
|
|
||||||
|
|
||||||
// recursive fs.rmdir needs node version 12:
|
|
||||||
// https://github.com/ngosang/FlareSolverr/issues/5#issuecomment-655572712
|
|
||||||
export function deleteFolderRecursive(path: string) {
|
|
||||||
if (fs.existsSync(path)) {
|
|
||||||
fs.readdirSync(path).forEach((file) => {
|
|
||||||
const curPath = Path.join(path, file)
|
|
||||||
if (fs.lstatSync(curPath).isDirectory()) { // recurse
|
|
||||||
deleteFolderRecursive(curPath)
|
|
||||||
} else { // delete file
|
|
||||||
fs.unlinkSync(curPath)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
fs.rmdirSync(path)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export const removeEmptyFields = (o: Record<string, any>): typeof o => {
|
|
||||||
const r: typeof o = {}
|
|
||||||
for (const k in o) {
|
|
||||||
if (o[k] !== undefined) {
|
|
||||||
r[k] = o[k]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return r
|
|
||||||
}
|
|
||||||
Reference in New Issue
Block a user