Bump version 2.2.7

credit: @realivanjx
https://github.com/FlareSolverr/FlareSolverr/issues/481#issuecomment-1242951929

.dockerignore

@@ -1,6 +1,7 @@
-node_modules
+.git/
-npm-debug.log
+.github/
-Dockerfile
+.idea/
-.dockerignore
+bin/
-.git
+dist/
-.gitignore
+node_modules/
+resources/

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,32 @@
+**Please use the search bar** at the top of the page and make sure you are not creating an already submitted issue.
+Check closed issues as well, because your issue may have already been fixed.
+
+### How to enable debug and html traces
+
+[Follow the instructions from this wiki page](https://github.com/FlareSolverr/FlareSolverr/wiki/How-to-enable-debug-and-html-trace)
+
+### Environment
+
+* **FlareSolverr version**:
+* **Last working FlareSolverr version**:
+* **Operating system**:
+* **Are you using Docker**: [yes/no]
+* **FlareSolverr User-Agent (see log traces or / endpoint)**:
+* **Are you using a proxy or VPN?** [yes/no]
+* **Are you using Captcha Solver:** [yes/no]
+* **If using captcha solver, which one:**
+* **URL to test this issue:**
+
+### Description
+
+[List steps to reproduce the error and details on what happens and what you expected to happen]
+
+### Logged Error Messages
+
+[Place any relevant error messages you noticed from the logs here.]
+
+[Make sure you attach the full logs with your personal information removed in case we need more information]
+
+### Screenshots
+
+[Place any screenshots of the issue here if needed]

.github/workflows/release-docker.yml

@@ -1,4 +1,4 @@
-name: publish
+name: release-docker
 
 on:
   push:
@@ -24,19 +24,19 @@ jobs:
           tag-sha: false
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v1.0.1
       -
         name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
       -
         name: Login to DockerHub
-        uses: docker/login-action@v1 
+        uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       -
         name: Login to GitHub Container Registry
-        uses: docker/login-action@v1 
+        uses: docker/login-action@v1
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}

.github/workflows/release.yml

@@ -0,0 +1,55 @@
+name: release
+
+on:
+  push:
+    tags:
+      - 'v*.*.*'
+
+jobs:
+  build:
+    name: Create release
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # get all commits, branches and tags (required for the changelog)
+
+      - name: Setup Node
+        uses: actions/setup-node@v2
+        with:
+          node-version: '16'
+
+      - name: Build artifacts
+        run: |
+          npm install
+          npm run build
+          npm run package
+
+      - name: Build changelog
+        id: github_changelog
+        run: |
+          changelog=$(git log $(git tag | tail -2 | head -1)..HEAD --no-merges --oneline)
+          changelog="${changelog//'%'/'%25'}"
+          changelog="${changelog//$'\n'/'%0A'}"
+          changelog="${changelog//$'\r'/'%0D'}"
+          echo "##[set-output name=changelog;]${changelog}"
+
+      - name: Create release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_PAT }}
+        with:
+          tag_name: ${{ github.ref }}
+          release_name: ${{ github.ref }}
+          body: ${{ steps.github_changelog.outputs.changelog }}
+          draft: false
+          prerelease: false
+
+      - name: Upload release artifacts
+        uses: alexellis/upload-assets@0.2.2
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_PAT }}
+        with:
+          asset_paths: '["./bin/*.zip"]'

.gitignore

@@ -121,3 +121,6 @@ dist
 
 # Project Development
 testing/
+
+# Binaries
+bin/

Dockerfile

@@ -1,30 +1,29 @@
-FROM --platform=${TARGETPLATFORM:-linux/amd64} node:15.2.1-alpine3.11
+FROM node:16-alpine3.15
 
-# Print build information
+# Install the web browser (package firefox-esr is available too)
-ARG TARGETPLATFORM
+RUN apk update && \
-ARG BUILDPLATFORM
+    apk add --no-cache firefox dumb-init && \
-RUN printf "I am running on ${BUILDPLATFORM:-linux/amd64}, building for ${TARGETPLATFORM:-linux/amd64}\n$(uname -a)\n"
+    rm -Rf /var/cache
-
-# Install Chromium, dumb-init and remove all locales but en-US
-RUN apk add --no-cache chromium dumb-init && \
-    find /usr/lib/chromium/locales -type f ! -name 'en-US.*' -delete
 
 # Copy FlareSolverr code
 USER node
 RUN mkdir -p /home/node/flaresolverr
 WORKDIR /home/node/flaresolverr
-COPY --chown=node:node package.json package-lock.json tsconfig.json ./
+COPY --chown=node:node package.json package-lock.json tsconfig.json install.js ./
 COPY --chown=node:node src ./src/
 
-# Install package. Skip installing Chrome, we will use the installed package.
+# Install package. Skip installing the browser, we will use the installed package.
-ENV PUPPETEER_PRODUCT=chrome \
+ENV PUPPETEER_PRODUCT=firefox \
     PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
-    PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
+    PUPPETEER_EXECUTABLE_PATH=/usr/bin/firefox
 RUN npm install && \
     npm run build && \
-    rm -rf src tsconfig.json && \
+    npm prune --production && \
-    npm prune --production
+    rm -rf /home/node/.npm
 
 EXPOSE 8191
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]
-CMD ["npm", "start"]
+CMD ["node", "./dist/server.js"]
+
+# docker build -t flaresolverr:custom .
+# docker run -p 8191:8191 -e LOG_LEVEL=debug flaresolverr:custom

README.md

@@ -1,33 +1,89 @@
 # FlareSolverr
 
-Proxy server to bypass Cloudflare protection
+[![Latest release](https://img.shields.io/github/v/release/FlareSolverr/FlareSolverr)](https://github.com/FlareSolverr/FlareSolverr/releases)
+[![Docker Pulls](https://img.shields.io/docker/pulls/flaresolverr/flaresolverr)](https://hub.docker.com/r/flaresolverr/flaresolverr/)
+[![GitHub issues](https://img.shields.io/github/issues/FlareSolverr/FlareSolverr)](https://github.com/FlareSolverr/FlareSolverr/issues)
+[![GitHub pull requests](https://img.shields.io/github/issues-pr/FlareSolverr/FlareSolverr)](https://github.com/FlareSolverr/FlareSolverr/pulls)
+[![Donate PayPal](https://img.shields.io/badge/Donate-PayPal-yellow.svg)](https://www.paypal.com/paypalme/diegoheras0xff)
+[![Donate Bitcoin](https://img.shields.io/badge/Donate-Bitcoin-f7931a.svg)](https://www.blockchain.com/btc/address/13Hcv77AdnFWEUZ9qUpoPBttQsUT7q9TTh)
+[![Donate Ethereum](https://img.shields.io/badge/Donate-Ethereum-8c8c8c.svg)](https://www.blockchain.com/eth/address/0x0D1549BbB00926BF3D92c1A8A58695e982f1BE2E)
 
-:warning: This project is in beta state. Some things may not work and the API can change at any time.
+FlareSolverr is a proxy server to bypass Cloudflare and DDoS-GUARD protection.
-See the known issues section.
 
 ## How it works
 
-FlareSolverr starts a proxy server and it waits for user requests in an idle state using few resources.
+FlareSolverr starts a proxy server, and it waits for user requests in an idle state using few resources.
 When some request arrives, it uses [puppeteer](https://github.com/puppeteer/puppeteer) with the
 [stealth plugin](https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin-stealth)
-to create a headless browser (Chrome). It opens the URL with user parameters and waits until the
+to create a headless browser (Firefox). It opens the URL with user parameters and waits until the Cloudflare challenge
-Cloudflare challenge is solved (or timeout). The HTML code and the cookies are sent back to the
+is solved (or timeout). The HTML code and the cookies are sent back to the user, and those cookies can be used to
-user and those cookies can be used to bypass Cloudflare using other HTTP clients.
+bypass Cloudflare using other HTTP clients.
 
-**NOTE**: Web browsers consume a lot of memory. If you are running FlareSolverr on a machine with few RAM,
+**NOTE**: Web browsers consume a lot of memory. If you are running FlareSolverr on a machine with few RAM, do not make
-do not make many requests at once. With each request a new browser is launched.
+many requests at once. With each request a new browser is launched.
-(It is possible to use a permanent session. However, if you use sessions, you should make sure to close them as soon as you are done using them.)
+
+It is also possible to use a permanent session. However, if you use sessions, you should make sure to close them as
+soon as you are done using them.
 
 ## Installation
 
-It requires NodeJS.
+### Docker
 
-Run `PUPPETEER_PRODUCT=chrome npm install` to install FlareSolverr dependencies.
+It is recommended to install using a Docker container because the project depends on an external browser that is
+already included within the image.
+
+Docker images are available in:
+* GitHub Registry => https://github.com/orgs/FlareSolverr/packages/container/package/flaresolverr
+* DockerHub => https://hub.docker.com/r/flaresolverr/flaresolverr
+
+Supported architectures are:
+| Architecture | Tag |
+| :----: | --- |
+| x86-64 | linux/amd64 |
+| ARM64 | linux/arm64 |
+| ARM32 | linux/arm/v7 |
+
+We provide a `docker-compose.yml` configuration file. Clone this repository and execute `docker-compose up -d` to start
+the container.
+
+If you prefer the `docker cli` execute the following command.
+```bash
+docker run -d \
+  --name=flaresolverr \
+  -p 8191:8191 \
+  -e LOG_LEVEL=info \
+  --restart unless-stopped \
+  ghcr.io/flaresolverr/flaresolverr:latest
+```
+
+If your host OS is Debian, make sure `libseccomp2` version is 2.5.x. You can check the version with `sudo apt-cache policy libseccomp2` 
+and update the package with `sudo apt install libseccomp2=2.5.1-1~bpo10+1` or `sudo apt install libseccomp2=2.5.1-1+deb11u1`.
+Remember to restart the Docker daemon and the container after the update.
+
+### Precompiled binaries
+
+This is the recommended way for Windows users.
+* Download the [FlareSolverr zip](https://github.com/FlareSolverr/FlareSolverr/releases) from the release's assets. It is available for Windows and Linux.
+* Extract the zip file. FlareSolverr executable and firefox folder must be in the same directory.
+* Execute FlareSolverr binary. In the environment variables section you can find how to change the configuration.
+
+### From source code
+
+This is the recommended way for macOS users and for developers.
+* Install [NodeJS](https://nodejs.org/) 16.
+* Clone this repository and open a shell in that path.
+* Run `export PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true` (Linux/macOS) or `set PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true` (Windows).
+* Run `npm install` command to install FlareSolverr dependencies.
+* Run `npm start` command to compile TypeScript code and start FlareSolverr.
+
+If you get errors related to firefox not installed try running `node install.js` to install Firefox.
+
+### Systemd service
+
+We provide an example Systemd unit file `flaresolverr.service` as reference. You have to modify the file to suit your needs: paths, user and environment variables.
 
 ## Usage
 
-First run `npm run build`. Once the TypeScript is compiled, you can use `npm start` to start FlareSolverr.
-
 Example request:
 ```bash
 curl -L -X POST 'http://localhost:8191/v1' \
@@ -35,11 +91,7 @@ curl -L -X POST 'http://localhost:8191/v1' \
 --data-raw '{
   "cmd": "request.get",
   "url":"http://www.google.com/",
-  "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36",
+  "maxTimeout": 60000
-  "maxTimeout": 60000,
-  "headers": {
-    "X-Test": "Testing 123..."
-  }
 }'
 ```
 
@@ -47,23 +99,22 @@ curl -L -X POST 'http://localhost:8191/v1' \
 
 #### + `sessions.create`
 
-This will launch a new browser instance which will retain cookies until you destroy it
+This will launch a new browser instance which will retain cookies until you destroy it with `sessions.destroy`.
-with `sessions.destroy`. This comes in handy so you don't have to keep solving challenges
+This comes in handy, so you don't have to keep solving challenges over and over and you won't need to keep sending
-over and over and you won't need to keep sending cookies for the browser to use.
+cookies for the browser to use.
 
-This also speeds up the requests since it won't have to launch a new browser instance for
+This also speeds up the requests since it won't have to launch a new browser instance for every request.
-every request.
 
 Parameter | Notes
 |--|--|
-session | Optional. The session ID that you want to be assinged to the instance. If one isn't set a random UUID will be assigned.
+session | Optional. The session ID that you want to be assigned to the instance. If isn't set a random UUID will be assigned.
-userAgent | Optional. Will be used by the headless browser.
+proxy | Optional, default disabled. Eg: `"proxy": {"url": "http://127.0.0.1:8888"}`. You must include the proxy schema in the URL: `http://`, `socks4://` or `socks5://`. Authorization (username/password) is not supported.
 
 #### + `sessions.list`
 
-Returns a list of all the active sessions. More for debuging if you are curious to see
+Returns a list of all the active sessions. More for debugging if you are curious to see how many sessions are running.
-how many sessions are running. You should always make sure to properly close each
+You should always make sure to properly close each session when you are done using them as too many may slow your
-session when you are done using them as too many may slow your computer down.
+computer down.
 
 Example response:
 
@@ -79,9 +130,8 @@ Example response:
 
 #### + `sessions.destroy`
 
-This will properly shutdown a browser instance and remove all files associaded with it
+This will properly shutdown a browser instance and remove all files associated with it to free up resources for a new
-to free up resources for a new session. Whenever you no longer need to use a session you
+session. When you no longer need to use a session you should make sure to close it.
-should make sure to close it.
 
 Parameter | Notes
 |--|--|
@@ -93,9 +143,12 @@ Parameter | Notes
 |--|--|
 url | Mandatory
 session | Optional. Will send the request from and existing browser instance. If one is not sent it will create a temporary instance that will be destroyed immediately after the request is completed.
-headers | Optional. To specify user headers.
+maxTimeout | Optional, default value 60000. Max timeout to solve the challenge in milliseconds.
-maxTimeout | Optional. Max timeout to solve the challenge
+cookies | Optional. Will be used by the headless browser. Follow [this](https://github.com/puppeteer/puppeteer/blob/v3.3.0/docs/api.md#pagesetcookiecookies) format.
-cookies | Optional. Will be used by the headless browser. Follow [this](https://github.com/puppeteer/puppeteer/blob/v3.3.0/docs/api.md#pagesetcookiecookies) format
+returnOnlyCookies | Optional, default false. Only returns the cookies. Response data, headers and other parts of the response are removed.
+proxy | Optional, default disabled. Eg: `"proxy": {"url": "http://127.0.0.1:8888"}`. You must include the proxy schema in the URL: `http://`, `socks4://` or `socks5://`. Authorization (username/password) is not supported. (When the `session` parameter is set, the proxy is ignored; a session specific proxy can be set in `sessions.create`.)
+
+:warning: If you want to use Cloudflare clearance cookie in your scripts, make sure you use the FlareSolverr User-Agent too. If they don't match you will see the challenge.
 
 Example response from running the `curl` above:
 
@@ -117,14 +170,13 @@ Example response from running the `curl` above:
             "content-length": "61587",
             "x-xss-protection": "0",
             "x-frame-options": "SAMEORIGIN",
-            "set-cookie": "1P_JAR=2020-07-16-04; expires=Sat, 15-Aug-2020 04:15:49 GMT; path=/; domain=.google.com; Secure; SameSite=none\nNID=204=QE3Ocq15XalczqjuDy52HeseG3zAZuJzID3R57g_oeQHyoV5DuvDhpWc4r9IcPoeIYmkr_ZTX_MNOU8IAbtXmVO7Bmq0adb-hpIHaTBIdBk3Ofifp4gO6vZleVuFYfj7ePkHeHdzGoX-en0FvKtd9iofX4O6RiAdEIAnpL7Wge4; expires=Fri, 15-Jan-2021 04:15:49 GMT; path=/; domain=.google.com; Secure; HttpOnly; SameSite=none",
+            "set-cookie": "1P_JAR=2020-07-16-04; expires=Sat..."
-            "alt-svc": "h3-29=\":443\"; ma=2592000,h3-27=\":443\"; ma=2592000,h3-25=\":443\"; ma=2592000,h3-T050=\":443\"; ma=2592000,h3-Q050=\":443\"; ma=2592000,h3-Q046=\":443\"; ma=2592000,h3-Q043=\":443\"; ma=2592000,quic=\":443\"; ma=2592000; v=\"46,43\""
         },
         "response":"<!DOCTYPE html>...",
         "cookies": [
             {
                 "name": "NID",
-                "value": "204=QE3Ocq15XalczqjuDy52HeseG3zAZuJzID3R57g_oeQHyoV5DuvDhpWc4r9IcPoeIYmkr_ZTX_MNOU8IAbtXmVO7Bmq0adb-hpIHaTBIdBk3Ofifp4gO6vZleVuFYfj7ePkHeHdzGoX-en0FvKtd9iofX4O6RiAdEIAnpL7Wge4",
+                "value": "204=QE3Ocq15XalczqjuDy52HeseG3zAZuJzID3R57...",
                 "domain": ".google.com",
                 "path": "/",
                 "expires": 1610684149.307722,
@@ -147,7 +199,7 @@ Example response from running the `curl` above:
                 "sameSite": "None"
             }
         ],
-        "userAgent": "Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36"
+        "userAgent": "Windows NT 10.0; Win64; x64) AppleWebKit/5..."
     },
     "status": "ok",
     "message": "",
@@ -163,92 +215,38 @@ This is the same as `request.get` but it takes one more param:
 
 Parameter | Notes
 |--|--|
-postData | Must be a string. If you want to POST a form, don't forget to set the `Content-Type` header to `application/x-www-form-urlencoded` or the server might not understand your request.
+postData | Must be a string with `application/x-www-form-urlencoded`. Eg: `a=b&c=d`
-
-## Downloading Images and PDFs (small files)
-
-If you need to access an image/pdf or small file, you should pass the `download` parameter to
-`request.get` setting it to `true`. Rather than access the html and return text it will
-return a the buffer **base64** encoded which you will be able to decode and save the image/pdf.
-
-This method isn't recommended for videos or anything larger. As that should be streamed back to
-the client and at the moment there is nothing setup to do so. If this is something you need feel
-free to create an issue and/or submit a PR.
 
 ## Environment variables
 
-To set the environment vars in Linux run `export LOG_LEVEL=debug` and then start FlareSolverr in the same shell.
-
 Name | Default | Notes
 |--|--|--|
-LOG_LEVEL | info | Used to change the verbosity of the logging.
+LOG_LEVEL | info | Verbosity of the logging. Use `LOG_LEVEL=debug` for more information.
-LOG_HTML | false | Used for debugging. If `true` all html that passes through the proxy will be logged to the console.
+LOG_HTML | false | Only for debugging. If `true` all HTML that passes through the proxy will be logged to the console in `debug` level.
-PORT | 8191 | Change this if you already have a process running on port `8191`.
+CAPTCHA_SOLVER | none | Captcha solving method. It is used when a captcha is encountered. See the Captcha Solvers section.
-HOST | 0.0.0.0 | This shouldn't need to be messed with but if you insist, it's here!
+TZ | UTC | Timezone used in the logs and the web browser. Example: `TZ=Europe/London`.
-CAPTCHA_SOLVER | None | This is used to select which captcha solving method it used when a captcha is encounted.
+HEADLESS | true | Only for debugging. To run the web browser in headless mode or visible.
-HEADLESS | true | This is used to debug the browser by not running it in headless mode.
+BROWSER_TIMEOUT | 40000 | If you are experiencing errors/timeouts because your system is slow, you can try to increase this value. Remember to increase the `maxTimeout` parameter too.
+TEST_URL | https://www.google.com | FlareSolverr makes a request on start to make sure the web browser is working. You can change that URL if it is blocked in your country.
+PORT | 8191 | Listening port. You don't need to change this if you are running on Docker.
+HOST | 0.0.0.0 | Listening interface. You don't need to change this if you are running on Docker.
+
+Environment variables are set differently depending on the operating system. Some examples:
+* Docker: Take a look at the Docker section in this document. Environment variables can be set in the `docker-compose.yml` file or in the Docker CLI command.
+* Linux: Run `export LOG_LEVEL=debug` and then start FlareSolverr in the same shell.
+* Windows: Open `cmd.exe`, run `set LOG_LEVEL=debug` and then start FlareSolverr in the same shell.
 
 ## Captcha Solvers
 
-Sometimes CF not only gives mathmatical computations and browser tests, sometimes they also require
+:warning: At this time none of the captcha solvers work. You can check the status in the open issues. Any help is welcome.
-the user to solve a captcha. If this is the case, FlareSolverr will return the captcha page. But that's
-not very helpful to you is it?
 
-FlareSolverr can be customized to solve the captcha's automatically by setting the environment variable
+Sometimes CloudFlare not only gives mathematical computations and browser tests, sometimes they also require the user to
-`CAPTCHA_SOLVER` to the file name of one of the adapters inside the [/captcha](src/captcha) directory.
+solve a captcha.
+If this is the case, FlareSolverr will return the error `Captcha detected but no automatic solver is configured.`
 
-### [CaptchaHarvester](https://github.com/NoahCardoza/CaptchaHarvester)
+FlareSolverr can be customized to solve the captchas automatically by setting the environment variable `CAPTCHA_SOLVER`
+to the file name of one of the adapters inside the [/captcha](src/captcha) directory.
 
-This method makes use of the [CaptchaHarvester](https://github.com/NoahCardoza/CaptchaHarvester) project which allows users to collect thier own tokens from ReCaptcha V2/V3 and hCaptcha for free.
+## Related projects
 
-To use this method you must set these ENV variables:
+* C# implementation => https://github.com/FlareSolverr/FlareSolverrSharp
-
-```bash
-CAPTCHA_SOLVER=harvester
-HARVESTER_ENDPOINT=https://127.0.0.1:5000/token
-```
-
-**Note**: above I set `HARVESTER_ENDPOINT` to the default configureation
-of the captcha harvester's server, but that could change if
-you customize the command line flags. Simply put, `HARVESTER_ENDPOINT`
-should be set to the URI of the route that returns a token in plain text when called.
-
-### [hcaptcha-solver](https://github.com/JimmyLaurent/hcaptcha-solver)
-
-This method makes use of the [hcaptcha-solver](https://github.com/JimmyLaurent/hcaptcha-solver) project which attempts to solve hcaptcha by randomly selecting images.
-
-To use this solver you must first install it and then set it as the `CAPTCHA_SOLVER`.
-
-```bash
-npm i hcaptcha-solver
-CAPTCHA_SOLVER=hcaptcha-solver
-```
-
-## Docker
-
-You can edit environment variables in `./Dockerfile` and build your own image.
-
-```bash
-docker build -t flaresolverr:latest .
-docker run --restart=always --name flaresolverr -p 8191:8191 -d flaresolverr:latest
-```
-
-## TypeScript
-
-I'm quite new to TypeScript. If you spot any funny business or anything that is or isn't being
-used properly feel free to submit a PR or open an issue.
-
-## Known issues / Roadmap
-
-The current implementation seems to be working on the sites I have been testing them on. However, if you find it unable to access a site, open an issue and I'd be happy to investigate.
-
-That being said, the project uses the [puppeteer stealth plugin](https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin-stealth). If Cloudflare is able to detect the headless browser, it's more that projects domain to fix.
-
-TODO:
-
-* Fix remaining issues in the code (see TODOs in code)
-* Make the maxTimeout more accurate (count the time to open the first page / maybe count the captcha solve time?)
-* Hide sensitive information in logs
-* Reduce Docker image size
-* Docker image for ARM architecture
-* Install instructions for Windows

build-binaries.js

@@ -0,0 +1,90 @@
+const fs = require('fs')
+const path = require('path')
+const { execSync } = require('child_process')
+const archiver = require('archiver')
+const https = require('https')
+const puppeteer = require('puppeteer')
+const version = 'v' + require('./package.json').version;
+
+(async () => {
+  const builds = [
+    {
+      platform: 'linux',
+      firefoxFolder: 'firefox',
+      fsExec: 'flaresolverr-linux',
+      fsZipExec: 'flaresolverr',
+      fsZipName: 'linux-x64',
+      fsLicenseName: 'LICENSE'
+    },
+    {
+      platform: 'win64',
+      firefoxFolder: 'firefox',
+      fsExec: 'flaresolverr-win.exe',
+      fsZipExec: 'flaresolverr.exe',
+      fsZipName: 'windows-x64',
+      fsLicenseName: 'LICENSE.txt'
+    }
+    // todo: this has to be build in macOS (hdiutil is required). changes required in sessions.ts too
+    // {
+    //   platform: 'mac',
+    //   firefoxFolder: 'firefox',
+    //   fsExec: 'flaresolverr-macos',
+    //   fsZipExec: 'flaresolverr',
+    //   fsZipName: 'macos',
+    //   fsLicenseName: 'LICENSE'
+    // }
+  ]
+
+  // generate executables
+  console.log('Generating executables...')
+  if (fs.existsSync('bin')) {
+    fs.rmSync('bin', { recursive: true })
+  }
+  execSync('./node_modules/.bin/pkg -t node16-win-x64,node16-linux-x64 --out-path bin .')
+  // execSync('./node_modules/.bin/pkg -t node16-win-x64,node16-mac-x64,node16-linux-x64 --out-path bin .')
+
+  // Puppeteer does not allow to download Firefox revisions, just the last Nightly
+  // We this script we can download any version
+  const revision = '94.0a1';
+  const downloadHost = 'https://archive.mozilla.org/pub/firefox/nightly/2021/10/2021-10-01-09-33-23-mozilla-central';
+
+  // download firefox and zip together
+  for (const os of builds) {
+    console.log('Building ' + os.fsZipName + ' artifact')
+
+    // download firefox
+    console.log(`Downloading firefox ${revision} for ${os.platform} ...`)
+    const f = puppeteer.createBrowserFetcher({
+      product: 'firefox',
+      platform: os.platform,
+      host: downloadHost,
+      path: path.join(__dirname, 'bin', 'puppeteer')
+    })
+    await f.download(revision)
+
+    // compress in zip
+    console.log('Compressing zip file...')
+    const zipName = 'bin/flaresolverr-' + version + '-' + os.fsZipName + '.zip'
+    const output = fs.createWriteStream(zipName)
+    const archive = archiver('zip')
+
+    output.on('close', function () {
+      console.log('File ' + zipName + ' created. Size: ' + archive.pointer() + ' bytes')
+    })
+
+    archive.on('error', function (err) {
+      throw err
+    })
+
+    archive.pipe(output)
+
+    archive.file('LICENSE', { name: 'flaresolverr/' + os.fsLicenseName })
+    archive.file('bin/' + os.fsExec, { name: 'flaresolverr/' + os.fsZipExec })
+    archive.directory('bin/puppeteer/' + os.platform + '-' + revision + '/' + os.firefoxFolder, 'flaresolverr/firefox')
+    if (os.platform === 'linux') {
+      archive.file('flaresolverr.service', { name: 'flaresolverr/flaresolverr.service' })
+    }
+
+    await archive.finalize()
+  }
+})()

docker-compose.yml

@@ -0,0 +1,15 @@
+---
+version: "2.1"
+services:
+  flaresolverr:
+    # DockerHub mirror flaresolverr/flaresolverr:latest
+    image: ghcr.io/flaresolverr/flaresolverr:latest
+    container_name: flaresolverr
+    environment:
+      - LOG_LEVEL=${LOG_LEVEL:-info}
+      - LOG_HTML=${LOG_HTML:-false}
+      - CAPTCHA_SOLVER=${CAPTCHA_SOLVER:-none}
+      - TZ=Europe/London
+    ports:
+      - "${PORT:-8191}:8191"
+    restart: unless-stopped

flaresolverr.service

@@ -0,0 +1,19 @@
+[Unit]
+Description=FlareSolverr
+After=network.target
+
+[Service]
+SyslogIdentifier=flaresolverr
+Restart=always
+RestartSec=5
+Type=simple
+User=flaresolverr
+Group=flaresolverr
+Environment="LOG_LEVEL=info"
+Environment="CAPTCHA_SOLVER=none"
+WorkingDirectory=/opt/flaresolverr
+ExecStart=/opt/flaresolverr/flaresolverr
+TimeoutStopSec=30
+
+[Install]
+WantedBy=multi-user.target

html_samples/cloudflare_captcha_hcaptcha_v1.html

@@ -0,0 +1,219 @@
+<!DOCTYPE html>
+<html lang="en-US">
+
+<head>
+    <title>Just a moment...</title>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=Edge">
+    <meta name="robots" content="noindex,nofollow">
+    <meta name="viewport" content="width=device-width,initial-scale=1">
+    <link href="Just%20a%20moment_files/cf-errors.css" rel="stylesheet">
+
+    <script>
+        (function () {
+            window._cf_chl_opt = {
+                cvId: '2',
+                cType: 'managed',
+                cNounce: '67839',
+                cRay: '732fbc436ab471ed',
+                cHash: 'dce5bd920f3aa51',
+                cUPMDTk: "\/search?q=2022&__cf_chl_tk=lkycIb1jDXlmFqiB7AXTwy38_EzYPvu79CCQyU9lhUE-1659201316-0-gaNycGzNCf0",
+                cFPWv: 'g',
+                cTTimeMs: '1000',
+                cTplV: 2,
+                cRq: {
+                    ru: 'aHR0cHM6Ly8wbWFnbmV0LmNvbS9zZWFyY2g/cT0yMDIy',
+                    ra: 'TW96aWxsYS81LjAgKFgxMTsgTGludXggeDg2XzY0OyBydjoxMDUuMCkgR2Vja28vMjAxMDAxMDEgRmlyZWZveC8xMDUuMA==',
+                    rm: 'R0VU',
+                    d: 'MqxNbbGfWazPaVMZ7GQRz02TV/pSUL9POWx0y4e7HFRwP1RTAxLc1RZRuHg+N/bGMuPj08kSx0UpcjEjMkSOqiU6I/64IDYbCJvey5rY07fkkljpZaYGTDZIoWdOWlgP3ky15ybZ42xMK4tfI1yJ+iFZCVgR6VBjJzi5I56j9Ijog2AvsoQW2TrguGpgKaT1LkhxWNElzBbvXWt1uyRgE19UQ9J/5vtxEwoh5wodHh7WE297n8uI1hpDgge2bDYQvwe+RDq3QAyhQOmymg+IIlt1y115v9R8k5ehT9TFY3vYvYnoJu9cOyHYprf9Z0jTNGxSTvLHYJbfq30Samu5fKfE0oZREZizvPUgUsJm2rRKkCY9VCdBkpO8vaUgIwIYkeWavtqdudjb3zEDBCD4cAH/xv3Bl1VRy2Qf7XlcbpElCOq06TDTQ1uGjyCqbVbvjesrOy0Dp2nXTjdfbkWvnN7mWpFlPUD7/41MUo9lc6V1Aj1Kjg6AKfVV4DUHpq6ZVnMHzrcPQLy4qD7CptcMpQKArZtJCRsUpgq8GWKJcU4dU8ZmyROAA+l+JEVnGbh2bsRdif4azh57OdjZfEKSa5c+AL3i66vyWAZCw9Wl6CAQdFTA+ixkbl8zKbCm8ulv',
+                    t: 'MTY1OTIwMTMxNi4zOTIwMDA=',
+                    m: '3l81qRkXiMTbjTzBtc0v1XwSheF46UfagbXVhYgbAVw=',
+                    i1: 'Iu5a1gH3p9igzqBwncow9g==',
+                    i2: 'PmNXozjc73unhnp/X0+kUQ==',
+                    zh: 'qP4bnGc6j96JlnjNSE7HmQci3S9L50bHFtm4bQRjjKU=',
+                    uh: 'SK3PXNkeRzZtkRARhJpbmZpCIiWQw6+5gpOE7vojWx4=',
+                    hh: 'azXzJl8Ou22g0nN/9idVUoB9EqZ7fLmkSdDRHM3Lkmw=',
+                }
+            }
+            window._cf_chl_enter = function () { window._cf_chl_opt.p = 1 };
+        })();
+    </script>
+
+    <script src="Just%20a%20moment_files/v1.js"></script>
+    <script type="text/javascript" src="Just%20a%20moment_files/api.js"></script>
+</head>
+
+<body class="no-js">
+
+    <div class="privacy-pass">
+        <a rel="noopener noreferrer" href="https://addons.mozilla.org/en-US/firefox/addon/privacy-pass/"
+            target="_blank">
+            Privacy Pass
+            <span class="privacy-pass-icon-wrapper">
+                <div class="privacy-pass-icon"></div>
+            </span>
+        </a>
+    </div>
+
+    <div class="main-wrapper" role="main">
+        <div class="main-content">
+            <h1 class="zone-name-title h1">
+                <img class="heading-favicon" src="Just%20a%20moment_files/favicon.ico"
+                    onerror="this.onerror=null;this.parentNode.removeChild(this)">
+                0MAGNET.COM
+            </h1>
+            <h2 class="h2" id="cf-challenge-running">
+                Checking if the site connection is secure
+            </h2>
+            <div id="cf-challenge-stage" style="display: block;">
+                <div id="cf-challenge-hcaptcha-wrapper" class="captcha-prompt spacer">
+                    <div style="display: none;" class="hcaptcha-box"><iframe src="Just%20a%20moment_files/hcaptcha.html"
+                            title="widget containing checkbox for hCaptcha security challenge" tabindex="0"
+                            scrolling="no" data-hcaptcha-widget-id="0tiueg8lyuj" data-hcaptcha-response=""
+                            style="width: 303px; height: 78px; overflow: hidden;" frameborder="0"></iframe><textarea
+                            id="h-captcha-response-0tiueg8lyuj" name="h-captcha-response"
+                            style="display: none;"></textarea></div>
+                    <div class="hcaptcha-box"><iframe src="Just%20a%20moment_files/hcaptcha_002.html"
+                            title="widget containing checkbox for hCaptcha security challenge" tabindex="0"
+                            scrolling="no" data-hcaptcha-widget-id="10tlmhzz0qyq" data-hcaptcha-response=""
+                            style="width: 303px; height: 78px; overflow: hidden;" frameborder="0"></iframe><textarea
+                            id="h-captcha-response-10tlmhzz0qyq" name="h-captcha-response"
+                            style="display: none;"></textarea></div>
+                </div>
+            </div>
+            <div id="cf-challenge-spinner" class="spacer loading-spinner" style="display: none; visibility: hidden;">
+                <div class="lds-ring">
+                    <div></div>
+                    <div></div>
+                    <div></div>
+                    <div></div>
+                </div>
+            </div>
+            <noscript>
+                <div id="cf-challenge-error-title">
+                    <div class="h2">
+                        <span class="icon-wrapper">
+                            <div class="heading-icon warning-icon"></div>
+                        </span>
+                        <span id="cf-challenge-error-text">
+                            Enable JavaScript and cookies to continue
+                        </span>
+                    </div>
+                </div>
+            </noscript>
+            <div
+                style="display:none;background-image:url('/cdn-cgi/images/trace/captcha/nojs/transparent.gif?ray=732fbc436ab471ed')">
+            </div>
+            <div id="cf-challenge-body-text" class="core-msg spacer">
+                0magnet.com needs to review the security of your connection before
+                proceeding.
+            </div>
+            <div id="cf-challenge-fact-wrapper" style="display: block; visibility: visible;" class="fact spacer hidden">
+                <span class="fact-title">Did you know</span> <span id="cf-challenge-fact" class="body-text">the first
+                    botnet in 2003 took over 500-1000 devices? Today, botnets take over millions of devices at
+                    once.</span>
+            </div>
+            <div id="cf-challenge-explainer-expandable" class="hidden expandable body-text spacer"
+                style="display: block; visibility: visible;">
+                <div class="expandable-title" id="cf-challenge-explainer-summary"><button class="expandable-summary-btn"
+                        id="cf-challenge-explainer-btn" type="button"> Why am I seeing this page? <span
+                            class="caret-icon-wrapper">
+                            <div class="caret-icon"></div>
+                        </span> </button> </div>
+                <div class="expandable-details" id="cf-challenge-explainer-details">
+                    Requests from malicious bots can pose as legitimate traffic.
+                    Occasionally, you may see this page while the site ensures that the
+                    connection is secure.</div>
+            </div>
+            <div id="cf-challenge-success" style="display: none;">
+                <div class="h2"><span class="icon-wrapper"><img class="heading-icon" alt="Success icon"
+                            src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADQAAAA0CAMAAADypuvZAAAANlBMVEUAAAAxMTEwMDAxMTExMTEwMDAwMDAwMDAxMTExMTExMTEwMDAwMDAxMTExMTEwMDAwMDAxMTHB9N+uAAAAEXRSTlMA3zDvfyBAEJC/n3BQz69gX7VMkcMAAAGySURBVEjHnZZbFoMgDEQJiDzVuv/NtgbtFGuQ4/zUKpeMIQbUhXSKE5l1XSn4pFWHRm/WShT1HRLWC01LGxFEVkCc30eYkLJ1Sjk9pvkw690VY6k8DWP9OM9yMG0Koi+mi8XA36NXmW0UXra4eJ3iwHfrfXVlgL0NqqGBHdqfeQhMmyJ48WDuKP81h3+SMPeRKkJcSXiLUK4XTHCjESOnz1VUXQoc6lgi2x4cI5aTQ201Mt8wHysI5fc05M5c81uZEtHcMKhxZ7iYEty1GfhLvGKpm+EYkdGxm1F5axmcB93DoORIbXfdN7f+hlFuyxtDP+sxtBnF43cIYwaZAWRgzxIoiXEMESoPlMhwLRDXeK772CAzXEdBRV7cmnoVBp0OSlyGidEzJTFq5hhcsA5388oSGM6b5p+qjpZrBlMS9xj4AwXmz108ukU1IomM3ceiW0CDwHCqp1NjAqXlFrbga+xuloQJ+tuyfbIBPNpqnmxqT7dPaOnZqBfhSBCteJAxWj58zLk2xgg+SPGYM6dRO6WczSnIxxwEExRaO+UyCUhbOp7CGQ+kxSUfNtLQFC+Po29vvy7jj4y0yAAAAABJRU5ErkJggg=="></span>Connection
+                    is secure</div>
+                <div class="core-msg spacer">Proceeding...</div>
+            </div>
+            <form id="challenge-form"
+                action="/search?q=2022&amp;__cf_chl_f_tk=lkycIb1jDXlmFqiB7AXTwy38_EzYPvu79CCQyU9lhUE-1659201316-0-gaNycGzNCf0"
+                method="POST" enctype="application/x-www-form-urlencoded">
+                <input type="hidden" name="md"
+                    value="P4fDbSohR3e3VZmGdBSN0Gd8t8ueht.ZVgSdQYwa45Y-1659201316-0-AesEKnKN8eJLiLESJle3R0T3fwKbVMlX09CR0sIU1LruDXen0nSlT2a5OpMUFYR7HQMGcF9Ja227n2p2D2ffUlWHPVeFX-YSNiewLZA3XuAQmOn-1DyWKA-SaMH_MW2vOSC7PCHAdJDhoRWjM_o3MyKziopj3WmDcaCI_ikk68bJTIValZ_e9tO7hmHC8zjsxDC8kXmI0tbrhyW5nyS2hRlx_ZVRcRHbHsVRN0-FGtEbCoaHmnp-q0N4AYhCJXofYRunPcSG_Y1iWMk-7ofOXON_gO7oGG_8-WWD5EG1jaz2ldpNO1RTkS7dQvTiC1Io1qAsVnQtokEaDR2zoWK_MF-hz6tOmuJIDgnAoH6vPFAa9EyJOUiG2RV-3q1CKTUgr82XRJw5CaXpN0QeBq0xHxFl5mzkFO8xqQsRnPkGUKtxBQ58syPIhR4AvNp8HA028gUNmaztJZ9i2UcWydut4VghHsoJjS5DEKTamjJhNrrkargjXUekXTfKXMVKCXxo0NFObTmKwzsNB5hrk3M43KzZCOOgTnqsrVUk54bAeDsr4qmTVW2wVk-0u78QpV2JFFOIJxRLikPmqo9CUokgUJ_IPsEjA5Q3kjrf9yq2OHU0MkwzLFNOAyc5N3A4WSYp91kESwxM98qFetpAZ0R3LID2c2-MraHnpOI2Xn4bxbDIdUPmjy6VB8Huuuf6M-o3Tw">
+                <input type="hidden" name="r"
+                    value="bdZ7.nm8dGOZxq3EDOv_Kx7nKVv68q7b0RARXAlR9kQ-1659201316-0-AawyK3x4GgWasA2OtBBEp9Ea52qs8zEWwnQJxLWUnC+1jqlxaKHTIHeVQjvrTl/ccu6QA41yrSTKvazKiv6zQEiDj/6ziYkhldx+oJ7SqgMzPozzza1jofsGpCCPAzIlDicF+7sh4WKOxUJOeHgHCgfEZF/MPNsaahvbQ10U8Ei9tmvj8c2tkoybya75Bj5XHPPu0S9hnOH7S24ltm9vmyHlttI7uuI962FzPCTGjuAl4R/5+06WVAzBCJrS4biDNIuyYe22PtLl4b3Yf55eW7AFgyzKgddsohZJuNNliKyD6cusHDhm7MYpnXc5zwTdCbt6KGK/tBaylNyYwH/WBAUhyRYN5EVt9/iIKHrb+P6Z0RL4nO3BtQE/Zwx1VC3g1Wy4PPQJjqLixQptzl5eIzu43JIO/LBvT/mWuheH4eoPlghvyMYwfHcs7B4d7FCv1Tj9Skp9Fcj6HBAZlq/ss/eIwk7oOcTviQs+EUF9/yYatgtpXX9RCyvhMU6/ghOLfXRmOpAzsmoGnVqEpc2IMlZegYtieLveXU35cGJMI6wCR2ciCJIX995vLuL/4BdCAMEhyMAUWxtaCD2ZfRHyOWKNuf80w9k6/Ofhu7RevCr2mjQJAVTyE2OWWgOUuYJ4pZim93J7slMXieL3S5/JM08Q8g179Of7dzpN/oG7s80ljxAiCprpUAwpEmNiqNJN//v0e9KxknhCHeAWSAe8IeXbp5PSEQHXTmsqOFRkpud1pTsETcNbdonk8XMyv8mZRcFPVWRRWUb8hupn/d+x9r6mOdKdJkH8ZZ0R30LG0SLPYEvsVr2yU9o+uCZrRWkuE3SP3Lq3BIx+0vtm0DOvj6cODxy5/4Zm4x7LIpSa9wr69Rs2x+t+U5ydUupZ7oiAbWfYZSXHpmB0zJYOLMPJZcut50J/IgWuTMda8QBcTG3jRr4BTwpcmBZRmddfOJYgD7EMpOi1HgwLnS7l5QELafaMn0Hl6G774GVy4lEK2jURG9IEE3PV1m5Y903pqldFkJQsMxdisJWOzVjbtf41fxxnt4cQgiDQhktqCwg8xP6ijzPeWgvQHL5fMq61cQ5/4HB+yt9wKWMlBfUJR+ocI2MYx2nUWz+0BwCnTU29D9bx1xkir9bsnUnfOlfRDO2OEvI0iTe82666rVQO9XqTEz3POxrJYzLcSC9fTHpHfmCVwT2zWGGLi6pW5kqZh/uzSQ12MSvF5+dwvhe7yRks5gwMhnDHMQFyxKw3Xxm+dq2Ix/1uUucOhCu2L72j/NIwkF3Z7O7afY9nIu+NqMe8PbPJjq5ovEluosQfAMzWJH5Va8iur8o6K1y6hm7XFdNYAR+uCtxMw6WzF58QWVXXrDvfPeBMaNz+VVCnGP9elAwv62tc4Uh2SCbKbWdZchtLHgrJgYgQtCMhBDh0AXzE6ubbtfm9jE2vWcPj5jbo8U72i1pL2j8Xfr562Xc2WrQ7tKvSFQepGxfu2XgF7q55XKVqrnrBeXxZViUkB/gyXxI26CfrVfPLW+sYUo3JS+eCjyn2K7phv+630ixdpKrRJCTmkP3G8tcoLTJCB67/pbz+dXiNSB4JlHf4i3FVRkr8TAWS2zuMjJhB+ZyxnrGq/m7KSwpEEqgSCpOrQ5nkeoKIOyITfe9EPBSy9QtYDK+SAhUiLnICVURK7kGgrhZuKyK5/nyK9l7ffg16aaChJBisPBeiYsTDHlAeq0GbW7VR/jQDAVtVldeyD/dM5rJ4X+wl3A+faYD1OUxYT3n8dMs+E/1jLnYixXJpo7iXCqlTV3phOatg4XDQ5Bj6EYQIljVI4x2e8XHspcETIa0WepLsZF7WUtY7KbN8ZyDBFXgTMb4lzPmWyY8hZ05uX8EBKqUJhWh91AUob/OpJdf+u3axDDeRgsjl8K6CFM/5uQKo93co3KPqGZiqx0JoVj1t6KGxkrzYsgwlrTyeL44cEgr0zRQz5oFExuwHGYyogbHZ8EvU1eoiJ3IuQFxUH/1ULidfGtB371RYfz9gqONi1KiVzJ+zLjw+4HgMKXOV+ra09Dyg+eyUNfHillLXkhKWVoDpUhc+r46W6vXFp3oMKUWTRM0dE7iHofo+0tHb73d3ID2blRXgUeoMCQwOptoAYlFBIUYjggrIhd1AMC8TiZmiNULyP5imDePwcfq+ZjGH3o8VKRI2FcoFmChQegGco6pEbB5DxCguDuJbFRwGH4t9T0y74ZhlZiTNKA4xXsQnfIBEC5qz3mkcDAWoe73zqFAjp35JRVBjo3UDvehJppxzuoCXt9UbeuNEGll5/YJR4lfbUsEai0U6TFVleTTY53ofYCWEM6EnNDIToTFbm514YFTUSc4h8Qlq2fPeqC3IcCmirNT4Kf0FCO7MQrtGNFPJme2cpb/pZguS3pxxkKb4lOS+eGiUBGcSs1v3zHroJ+hum4wTJFRG0Yb99aCVQU44wgV3nKW7FZkXzwO3QY7nnkFI2kaAXerCPF4+Ho463g==">
+
+                <span style="display: none;"><span class="text-gray-600" data-translate="error">error code:
+                        1020</span></span>
+            </form>
+        </div>
+    </div>
+    <script>
+        (function () {
+            var trkjs = document.createElement('img');
+            trkjs.setAttribute('src', '/cdn-cgi/images/trace/captcha/js/transparent.gif?ray=732fbc436ab471ed');
+            trkjs.setAttribute('style', 'display: none');
+            document.body.appendChild(trkjs);
+            var cpo = document.createElement('script');
+            cpo.src = '/cdn-cgi/challenge-platform/h/g/orchestrate/managed/v1?ray=732fbc436ab471ed';
+            window._cf_chl_opt.cOgUHash = location.hash === '' && location.href.indexOf('#') !== -1 ? '#' : location.hash;
+            window._cf_chl_opt.cOgUQuery = location.search === '' && location.href.slice(0, -window._cf_chl_opt.cOgUHash.length).indexOf('?') !== -1 ? '?' : location.search;
+            if (window.history && window.history.replaceState) {
+                var ogU = location.pathname + window._cf_chl_opt.cOgUQuery + window._cf_chl_opt.cOgUHash;
+                history.replaceState(null, null, "\/search?q=2022&__cf_chl_rt_tk=lkycIb1jDXlmFqiB7AXTwy38_EzYPvu79CCQyU9lhUE-1659201316-0-gaNycGzNCf0" + window._cf_chl_opt.cOgUHash);
+                cpo.onload = function () {
+                    history.replaceState(null, null, ogU);
+                };
+            }
+            document.getElementsByTagName('head')[0].appendChild(cpo);
+        }());
+    </script><img src="Just%20a%20moment_files/transparent.gif" style="display: none">
+
+    <div class="footer" role="contentinfo">
+        <div class="footer-inner">
+            <div class="clearfix diagnostic-wrapper">
+                <div class="ray-id">Ray ID: <code>732fbc436ab471ed</code></div>
+            </div>
+            <div class="text-center">
+                Performance &amp; security by
+                <a rel="noopener noreferrer" href="https://www.cloudflare.com/" target="_blank">Cloudflare</a>
+            </div>
+        </div>
+    </div>
+
+
+    <div style="background-color: rgb(255, 255, 255); border: 1px solid rgb(215, 215, 215); box-shadow: rgba(0, 0, 0, 0.1) 0px 0px 4px; border-radius: 4px; left: -10000px; top: -10000px; z-index: -2147483648; position: absolute; transition: opacity 0.15s ease-out 0s; opacity: 0; visibility: hidden;"
+        aria-hidden="true">
+        <div style="position: relative; z-index: 1;"><iframe src="Just%20a%20moment_files/hcaptcha_003.html"
+                title="Main content of the hCaptcha challenge" scrolling="no"
+                style="border: 0px none; z-index: 2000000000; position: relative;" frameborder="0"></iframe></div>
+        <div
+            style="width: 100%; height: 100%; position: fixed; pointer-events: none; top: 0px; left: 0px; z-index: 0; background-color: rgb(255, 255, 255); opacity: 0.05;">
+        </div>
+        <div
+            style="border-width: 11px; position: absolute; pointer-events: none; margin-top: -11px; z-index: 1; right: 100%;">
+            <div
+                style="border-width: 10px; border-style: solid; border-color: transparent rgb(255, 255, 255) transparent transparent; position: relative; top: 10px; z-index: 1;">
+            </div>
+            <div
+                style="border-width: 11px; border-style: solid; border-color: transparent rgb(215, 215, 215) transparent transparent; position: relative; top: -11px; z-index: 0;">
+            </div>
+        </div>
+    </div>
+    <div style="background-color: rgb(255, 255, 255); border: 1px solid rgb(215, 215, 215); box-shadow: rgba(0, 0, 0, 0.1) 0px 0px 4px; border-radius: 4px; left: -10000px; top: -10000px; z-index: -2147483648; position: absolute; transition: opacity 0.15s ease-out 0s; opacity: 0; visibility: hidden;"
+        aria-hidden="true">
+        <div style="position: relative; z-index: 1;"><iframe src="Just%20a%20moment_files/hcaptcha_004.html"
+                title="Main content of the hCaptcha challenge" scrolling="no"
+                style="border: 0px none; z-index: 2000000000; position: relative;" frameborder="0"></iframe></div>
+        <div
+            style="width: 100%; height: 100%; position: fixed; pointer-events: none; top: 0px; left: 0px; z-index: 0; background-color: rgb(255, 255, 255); opacity: 0.05;">
+        </div>
+        <div
+            style="border-width: 11px; position: absolute; pointer-events: none; margin-top: -11px; z-index: 1; right: 100%;">
+            <div
+                style="border-width: 10px; border-style: solid; border-color: transparent rgb(255, 255, 255) transparent transparent; position: relative; top: 10px; z-index: 1;">
+            </div>
+            <div
+                style="border-width: 11px; border-style: solid; border-color: transparent rgb(215, 215, 215) transparent transparent; position: relative; top: -11px; z-index: 0;">
+            </div>
+        </div>
+    </div>
+</body>
+
+</html>

html_samples/cloudflare_captcha_norobot_v1.html

@@ -0,0 +1,170 @@
+<!DOCTYPE html>
+<html lang="en-US">
+
+<head>
+    <title>Just a moment...</title>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=Edge">
+    <meta name="robots" content="noindex,nofollow">
+    <meta name="viewport" content="width=device-width,initial-scale=1">
+    <link href="Just%20a%20moment2_files/cf-errors.css" rel="stylesheet">
+
+    <script>
+        (function () {
+            window._cf_chl_opt = {
+                cvId: '2',
+                cType: 'managed',
+                cNounce: '94250',
+                cRay: '732fc1c74f757330',
+                cHash: '8c4978fa93c1751',
+                cUPMDTk: "\/search?q=2022&__cf_chl_tk=6E3KpS5eCzuCMJG64ch2shvOMHdwQ8ioliqACpoQqM8-1659201542-0-gaNycGzNCeU",
+                cFPWv: 'g',
+                cTTimeMs: '1000',
+                cTplV: 2,
+                cRq: {
+                    ru: 'aHR0cHM6Ly8wbWFnbmV0LmNvbS9zZWFyY2g/cT0yMDIy',
+                    ra: 'TW96aWxsYS81LjAgKFgxMTsgTGludXggeDg2XzY0OyBydjoxMDUuMCkgR2Vja28vMjAxMDAxMDEgRmlyZWZveC8xMDUuMA==',
+                    rm: 'R0VU',
+                    d: 'C4CtJo9JDMtUWZ0r+/s2CwjYdSTdqGYK3qFo1OXpvSc9v7/3d5QuMwmvG3e5oV1BpjlQb8eJJ23gVRxavjw/gpPp1brmKoHuvcJEmAP3Sof38vqcpF91/9NHe3JbmCM2xshiGvJdbpJXb5wXdYKYPMqy7NUHL1VU4hupa3Da3tBq9zyuMa1NcZaiyeE6piSl7n96m+VziRdwyG+SBUldIG/Fsv9J1yl+Gj19wbX1XEneMXChcClGgRrSe1MTd9thLkq2NGFqROnsUmpA8b+2Eqi+IPYQfkPcydWkHmJqQixN9ZFTIBChIC60hGHOQ7O354ju65tVGAhB/nBRREpdqvwoYzgufgg83+dbPHVdQasiuLRHvftOtHhS5/iaBOVoEBH+rElTSk/OYjU2Yh6gkQj0FjkbebEBptFeVAxgqoYZljOrhamWYYZ14tOKeonzc1rz/FXNTM5qVtrWCwAlt9SsXDjM/GYXZMTbOdNLnLZGlLNQCx+l6hMC0OQC45sWFzZECljbjXwiYfodKobeqe11lUXnskj8AN5Qc7O8OqtALsxoNCLZ7ou+ORY0lauremeuu3U3WqadgSGFGA+TZZw2VcCA3BIUKCGlsNLBlJ8wQS2UAGJfGLOVuhErmtsM',
+                    t: 'MTY1OTIwMTU0Mi4yOTUwMDA=',
+                    m: 'eWHHJ28v6yOyvSePVqcdyHxAYkkc3xq3VJ8YiDCk5nk=',
+                    i1: 'M3dMvem+HcwSbNQrJbaYdQ==',
+                    i2: 'ebY327qYCu6NZKHSQXkbaQ==',
+                    zh: 'qP4bnGc6j96JlnjNSE7HmQci3S9L50bHFtm4bQRjjKU=',
+                    uh: 'SK3PXNkeRzZtkRARhJpbmZpCIiWQw6+5gpOE7vojWx4=',
+                    hh: 'azXzJl8Ou22g0nN/9idVUoB9EqZ7fLmkSdDRHM3Lkmw=',
+                }
+            }
+            window._cf_chl_enter = function () { window._cf_chl_opt.p = 1 };
+        })();
+    </script>
+
+    <script src="Just%20a%20moment2_files/v1.js"></script>
+    <script type="text/javascript" src="Just%20a%20moment2_files/api.js"></script>
+</head>
+
+<body class="no-js">
+
+    <div class="privacy-pass">
+        <a rel="noopener noreferrer" href="https://addons.mozilla.org/en-US/firefox/addon/privacy-pass/"
+            target="_blank">
+            Privacy Pass
+            <span class="privacy-pass-icon-wrapper">
+                <div class="privacy-pass-icon"></div>
+            </span>
+        </a>
+    </div>
+
+    <div class="main-wrapper" role="main">
+        <div class="main-content">
+            <h1 class="zone-name-title h1">
+                <img class="heading-favicon" src="Just%20a%20moment2_files/favicon.ico"
+                    onerror="this.onerror=null;this.parentNode.removeChild(this)">
+                0MAGNET.COM
+            </h1>
+            <h2 class="h2" id="cf-challenge-running">
+                Checking if the site connection is secure
+            </h2>
+            <div id="cf-challenge-stage" style="display: block;">
+                <div id="cf-norobot-container" style="display: flex;"><input type="button" value="Verify you are human"
+                        class="big-button pow-button" style="cursor: pointer;"></div>
+            </div>
+            <div id="cf-challenge-spinner" class="spacer loading-spinner" style="display: none; visibility: hidden;">
+                <div class="lds-ring">
+                    <div></div>
+                    <div></div>
+                    <div></div>
+                    <div></div>
+                </div>
+            </div>
+            <noscript>
+                <div id="cf-challenge-error-title">
+                    <div class="h2">
+                        <span class="icon-wrapper">
+                            <div class="heading-icon warning-icon"></div>
+                        </span>
+                        <span id="cf-challenge-error-text">
+                            Enable JavaScript and cookies to continue
+                        </span>
+                    </div>
+                </div>
+            </noscript>
+            <div
+                style="display:none;background-image:url('/cdn-cgi/images/trace/captcha/nojs/transparent.gif?ray=732fc1c74f757330')">
+            </div>
+            <div id="cf-challenge-body-text" class="core-msg spacer">
+                0magnet.com needs to review the security of your connection before
+                proceeding.
+            </div>
+            <div id="cf-challenge-fact-wrapper" style="display: block; visibility: visible;" class="fact spacer hidden">
+                <span class="fact-title">Did you know</span> <span id="cf-challenge-fact" class="body-text">botnets can
+                    be used to shutdown popular websites?</span>
+            </div>
+            <div id="cf-challenge-explainer-expandable" class="hidden expandable body-text spacer"
+                style="display: block; visibility: visible;">
+                <div class="expandable-title" id="cf-challenge-explainer-summary"><button class="expandable-summary-btn"
+                        id="cf-challenge-explainer-btn" type="button"> Why am I seeing this page? <span
+                            class="caret-icon-wrapper">
+                            <div class="caret-icon"></div>
+                        </span> </button> </div>
+                <div class="expandable-details" id="cf-challenge-explainer-details">
+                    Requests from malicious bots can pose as legitimate traffic.
+                    Occasionally, you may see this page while the site ensures that the
+                    connection is secure.</div>
+            </div>
+            <div id="cf-challenge-success" style="display: none;">
+                <div class="h2"><span class="icon-wrapper"><img class="heading-icon" alt="Success icon"
+                            src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADQAAAA0CAMAAADypuvZAAAANlBMVEUAAAAxMTEwMDAxMTExMTEwMDAwMDAwMDAxMTExMTExMTEwMDAwMDAxMTExMTEwMDAwMDAxMTHB9N+uAAAAEXRSTlMA3zDvfyBAEJC/n3BQz69gX7VMkcMAAAGySURBVEjHnZZbFoMgDEQJiDzVuv/NtgbtFGuQ4/zUKpeMIQbUhXSKE5l1XSn4pFWHRm/WShT1HRLWC01LGxFEVkCc30eYkLJ1Sjk9pvkw690VY6k8DWP9OM9yMG0Koi+mi8XA36NXmW0UXra4eJ3iwHfrfXVlgL0NqqGBHdqfeQhMmyJ48WDuKP81h3+SMPeRKkJcSXiLUK4XTHCjESOnz1VUXQoc6lgi2x4cI5aTQ201Mt8wHysI5fc05M5c81uZEtHcMKhxZ7iYEty1GfhLvGKpm+EYkdGxm1F5axmcB93DoORIbXfdN7f+hlFuyxtDP+sxtBnF43cIYwaZAWRgzxIoiXEMESoPlMhwLRDXeK772CAzXEdBRV7cmnoVBp0OSlyGidEzJTFq5hhcsA5388oSGM6b5p+qjpZrBlMS9xj4AwXmz108ukU1IomM3ceiW0CDwHCqp1NjAqXlFrbga+xuloQJ+tuyfbIBPNpqnmxqT7dPaOnZqBfhSBCteJAxWj58zLk2xgg+SPGYM6dRO6WczSnIxxwEExRaO+UyCUhbOp7CGQ+kxSUfNtLQFC+Po29vvy7jj4y0yAAAAABJRU5ErkJggg=="></span>Connection
+                    is secure</div>
+                <div class="core-msg spacer">Proceeding...</div>
+            </div>
+            <form id="challenge-form"
+                action="/search?q=2022&amp;__cf_chl_f_tk=6E3KpS5eCzuCMJG64ch2shvOMHdwQ8ioliqACpoQqM8-1659201542-0-gaNycGzNCeU"
+                method="POST" enctype="application/x-www-form-urlencoded">
+                <input type="hidden" name="md"
+                    value="UPeuijc1TS5ZQ21GIY6wjg6HHN_jWKH9sqolcSJABwg-1659201542-0-AR_ZxgiwVB4GwEgAjllIrmnGAumHNwuvfpFBddySYLh6CWexrUnxVYlX_wlB19Yndm45fs-KngMxbYB4dEOuf4MOJ_yL_BsNG3_cIPybV0bNn9WQXecJg3FfFrIBuMFIappZOX4hdDjLtRo9f4JsVsU6FzD9sUoKJRd4BTkjTAm25yFbqmPgV15XZhnJ5HRux044u0IIOVZCwTTzgRLCqToVb-OfiuUcHBzt4W7_wNlF1ObUi2oEr00DA1zZvzzY2KnXdZVN8m2OaNY_f2zkk9uDlLQRob_Ti6MHPNDr4eRkyMqZMZ1XDCxe-9lBkcEfpqtg6_4yac9ZiIEoNdJnJVE6cuNzb59DcBooXAq3IWp6fK4y4UIBStjqOXk4bxQb5yt1COfdPuQ9iLE_7yYOPG_t7n5I-4mjwvG7_U337A17oeEemXHfJkGC88Vm3SQdEHiW96VJuOA_X-rb7p3iOMlLYB5DKJ5DaBoPnP86uAWhoHWE6nrVzeAxeQ1y0uBHYPioJba5Kn9d-e2HsTMuAi7ZgSKuk90ApclIiW3owI4bLc4wxO5cu3ZIz7sZfbdvIKDhf9ESZhpQrITU_4Hgqjz0s3lt-MVeNP_0bz31XSeA--pdiulzUpQWLx1jhC4s7Av6STUb9bmbHpE41283KbbpuzBbmHN1UczNiaaquYZiEXRHKYyEMhKD782nWTJwQA">
+                <input type="hidden" name="r"
+                    value="i1ShtnCs9Zs8QexeFnp6EFtrWs3WbGEVQGXbVfYwpRI-1659201542-0-AbDM6G9qkbgoH+BqDdr1tzCDHr/DU9Sdxelapvp2/FZN6VqYfpDkJGv+HxhBQng6aVktcEobxp2ouOxJZxPQrR6tVFIhOW6uPOAdy5kh2BBJWUHfER13aq8LQ86fvDyRh3AThEHj6bgs2udacfvOrDrHT2j/KHBPePlGKbh8rzDTJBKw0ejUleHk8eKX/BQ1bVULgxT+ZZY721lyn2wrjsde1j1OAsiiCDkVvQ4Rs+Bas7UApD5HeWzyrCu2VFk/Qf+Rk+6spM+StYenQUAKXXrekJoIeNxPf/W9ZRsJfwUoY0JUK2thOWiwQOtw21nVDpiCFB9nhhOsmBzBoRQGjckZyu/O5U7jMIFdS9ThCFC0Kffg0MEr5xTkmgw+CNSwN7AlI9v3GS2XdTFOPXe29b68fZXzYfbm2CjqYhmxomZjCGTAmkzXWaVnMOs9Vl/8VurCUEu8SAt5k9Za/vFrEurX1edXNCviVuTOBSLHjqBiLui9FbufzGLq6BaHYi3WIFA1nMkoxduxbErP+Eqyi8UNvzvmEqUbj2COalXcQzkbHkyyLo33MNHZEi1zhhHjwCm1lp6mm4BRe60kRgTHb8X7oxBpY4vEcMz4jQQdsW15xBPAjsH8m9cj1H2ujpd7kfo8JGTyZ7FcoxOzGOuZr8XRpGkH72HaWYz7M+GIb3BBZ1v2Za7sSrzinNLFjHCCVXq68MqOmZ6RhgeexGoKJzcMHsHvgGXB8CisyyNTtA3OQOujybNUnNzlW7vJ/wDreTHkko6jQ/Lm/X2GnLg85BIg6IeROzt3eInAYsCaNKpST/h5bvSGCyzRoOW46oO8ZzZrV2FI2rEr0xLTIVWzQ//K2iGOCz58RisCfxWiF2n+fzj/5nE/0cjTPzYP68TM5BxB058EO7ZEFbgqhUji8IR9V2ahy7kI9dUhwd2S4IyjL+O6hCNPwjpRohkt93wXUCZDMgNoxi1BIylqqtAxYBfodyjFz8mB8GgcqBaCHN3tI0BINENVfvSJwKniYxL73frTX5KEqniT9GdT15o4F7QLf4S1atwYzF6ezJTYgLf6fOWUZKpaFMSRzEsxmZDmOFZeiss8lj7bKS6drOpkaOYzZiSgp5t5VwLKT0yQ+PDWQmqkpZ5WOa9/ayXLyOCunzk1IUO6VkvgFe0P2LZC9XEZUfwAFakYemej8/SZx0EknoPob1il3MMsbfHNAvcvUJK9xDbdAQ7rz34r4D5zO2aPnmYw1yv9K36z78I2dZpjVT9kpiKFwaOTkuSDUDtcmnhKM1XE+goG/C66G6PsChpGKLCeaDw4Rp7BxlumiSGB4Mp/bs8pTz3gez7pSu1oNodr7Tr1wJvCK8T5nVJ5GRO/tQ+Ff2K2s67udoV0CFtKufJyRsGCEv/0u5sArg3uwtwIz1W0JtAVjhe+J2nUihLa0Gqm7AwcCwfhsLHOhMG28V2NAw19iVq8RuMN7A2kGg5PH6bUeilWUxxZvWyDfyRSJZYMQytwAJdt4gQ++Qnl3mcaSk1N3pSiltVUDpfLcYb5gd35m+mKQWtPnIDlJMAtGoBeqROQPLNDg+LYdI/dnJzIOHjI3J+pTWhbAlF7B7NtccZOHmI9Cl3vS6Fpqs5aSPEDoDENTap6JN1kgm5NszMay9tAm66AcKF95W6QhwgQsyRrwScgRaPUtCx9ZJcbav6T/CAulcBB85MjwAd8+HF1g+UZT9VvChZoxh7NzfMoR53pVbxvW6acO8oVN5ITTP8mNAIisRvWi2KVdi4KqaLjYtLFNN8AMzjAC0vBIaFyGZlIbFsB44MRiMufD64b/66dqeC0l0WrUlUG/DgrnSQr6lgK2gONJKPQZGXoaK0Ga8O8xMOkaFLNaqH5UH5KpHvIQ8nwhuXk/MS/7Gdp1W02OEB4l0hhKFytgWdo9QmCquSatvOjuFyRPa6tV8ceGmuDnQw22bJM9BwzdKlHn/2/mHjCz7gcEA3Hb/CbeP8V8mF1mc5R8HEEdz/rx+BTESmGiRivv+WQYpRKNh77iqbYvCvkduK4b3UErNbvcS10aTt8zDFF/oIwjDpsniJsrIUcC0FdQRs2dqPIfkoSHvs7YGmOjx9QThCAiTkPKUE9C5C4YPY4CWRV3nYAFJrTq0F047PkzDYm0AJMCahWK7Vq/Ra3l3nRHu9yI+P0HiruUbkzLgiEJYnAuUtxvpC/Vj0uhr+A0R9Obs1MHkwtDuMs/ETh3ZymeFtWLj70StkslJxTzKGimZSsqQXRFYGHY6CHqHwIXGrArYNjTty48VIfbfaEu58KQp6roOdFmx90AcK2lV0V5UdyuzDJeH/V5ERAmxWLrXQKWgiDrY4ZqecnRk5XEAVMq/ChPts9gR7xsQK5WsHtQNKLfltkL8YvAoS+jZvxzfUUBg99YSC4J/HzQS+FQAnkDxCgeroahXysNN1bgDASXOrn3NsC3LYpUiZ2AVTLPkj1roR9r65O">
+
+                <span style="display: none;"><span style="display: none;" class="text-gray-600"
+                        data-translate="error">error code: 1020</span></span>
+            </form>
+        </div>
+    </div>
+    <script>
+        (function () {
+            var trkjs = document.createElement('img');
+            trkjs.setAttribute('src', '/cdn-cgi/images/trace/captcha/js/transparent.gif?ray=732fc1c74f757330');
+            trkjs.setAttribute('style', 'display: none');
+            document.body.appendChild(trkjs);
+            var cpo = document.createElement('script');
+            cpo.src = '/cdn-cgi/challenge-platform/h/g/orchestrate/managed/v1?ray=732fc1c74f757330';
+            window._cf_chl_opt.cOgUHash = location.hash === '' && location.href.indexOf('#') !== -1 ? '#' : location.hash;
+            window._cf_chl_opt.cOgUQuery = location.search === '' && location.href.slice(0, -window._cf_chl_opt.cOgUHash.length).indexOf('?') !== -1 ? '?' : location.search;
+            if (window.history && window.history.replaceState) {
+                var ogU = location.pathname + window._cf_chl_opt.cOgUQuery + window._cf_chl_opt.cOgUHash;
+                history.replaceState(null, null, "\/search?q=2022&__cf_chl_rt_tk=6E3KpS5eCzuCMJG64ch2shvOMHdwQ8ioliqACpoQqM8-1659201542-0-gaNycGzNCeU" + window._cf_chl_opt.cOgUHash);
+                cpo.onload = function () {
+                    history.replaceState(null, null, ogU);
+                };
+            }
+            document.getElementsByTagName('head')[0].appendChild(cpo);
+        }());
+    </script><img src="Just%20a%20moment2_files/transparent.gif" style="display: none">
+
+    <div class="footer" role="contentinfo">
+        <div class="footer-inner">
+            <div class="clearfix diagnostic-wrapper">
+                <div class="ray-id">Ray ID: <code>732fc1c74f757330</code></div>
+            </div>
+            <div class="text-center">
+                Performance &amp; security by
+                <a rel="noopener noreferrer" href="https://www.cloudflare.com/" target="_blank">Cloudflare</a>
+            </div>
+        </div>
+    </div>
+
+
+</body>
+
+</html>

html_samples/cloudflare_init_v1.html

@@ -0,0 +1,120 @@
+<!DOCTYPE html>
+<html lang="en-US">
+
+<head>
+    <title>Just a moment...</title>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+    <meta http-equiv="X-UA-Compatible" content="IE=Edge" />
+    <meta name="robots" content="noindex,nofollow" />
+    <meta name="viewport" content="width=device-width,initial-scale=1" />
+    <link href="/cdn-cgi/styles/cf-errors.css" rel="stylesheet" />
+
+    <script>
+        (function () {
+            window._cf_chl_opt = {
+                cvId: '2',
+                cType: 'managed',
+                cNounce: '46449',
+                cRay: '732fd3bc9c1d72de',
+                cHash: '8838fcad2a7f56c',
+                cUPMDTk: "\/search?q=2022&__cf_chl_tk=y4XnN88eYeUiXmFkQeqEipve1VuK0jJA.G4Hz6xztsM-1659202277-0-gaNycGzNBz0",
+                cFPWv: 'g',
+                cTTimeMs: '1000',
+                cTplV: 2,
+                cRq: {
+                    ru: 'aHR0cHM6Ly8wbWFnbmV0LmNvbS9zZWFyY2g/cT0yMDIy',
+                    ra: 'Y3VybC83Ljg0LjA=',
+                    rm: 'R0VU',
+                    d: '+SdFLvm4kJf8Z9BVci1ZbUOY6ab/Dm5Zzyb0IvscIzmY9PnAAcvPfJ/3TD9YJViBxB/ArnbCQrOUfbSkq4odyaZmW19gm+exRuL8Z3POm1ABs7y6jwMshM19q4Gr3eFY/MUO/IYWuyA2F9q94hRCI6ZNb7dLEh9yh6hORbKRd62pdn59h1xCx8tNdKDtP7VXPXo85nYmJJPLOdXTnII+YxZ03a4isAmBHbi+lGoQN/bCV0K006VmpfPElAfAO9jm45o7pc1NgPQhZSKWpTyI/nHMueH6wacPREzN5RtREoQfKuwYpV++Gq56qr5bAe/SKeF+rI0x7OSqC4HQvrNwbA+kHZzaxgOKeiMFjDxmro/GyC/+sxeZmrxnSIAh4BScjPxEl1FLLkg/6D0JH6HmxoT8N/Jgpi9447Am4WeX+WQxJ9+uDs5WrFIahx7pWrgcZUTRPh+UCu3allJ2Q3cAfwK6BclhES/HhBBbJv0pnR1R2RfKDM/gr1MpLuhaK4mFEO/kSyNUjOnCjOfd+5d7Qb0DZn7sHpF2SVc+zNv5OWSvCRDUcNHjIOV6fq0datVyVWmxD6unPS0MMUFO+ZZNiB4ionrhVCiLrb2FjPQ8tzyCqXg+tnV7WtZ0h4+JuK3rxcaQ8PQy60/As8dKHqVTnw==',
+                    t: 'MTY1OTIwMjI3Ny44NjMwMDA=',
+                    m: 'zvAOPvfoONkW1BzH+jMnKOPtDpPpZijRP52DVDWH+i8=',
+                    i1: 'dDlQDNhOEuHzFEPo/etoAA==',
+                    i2: '+LTK9hchBRjTTQk1WQU1Vw==',
+                    zh: 'qP4bnGc6j96JlnjNSE7HmQci3S9L50bHFtm4bQRjjKU=',
+                    uh: 'IdIU2i4FhVxxcYhzSFWdjoBuQm7qnyVK65JGofJuWV4=',
+                    hh: 'azXzJl8Ou22g0nN/9idVUoB9EqZ7fLmkSdDRHM3Lkmw=',
+                }
+            }
+            window._cf_chl_enter = function () { window._cf_chl_opt.p = 1 };
+        })();
+    </script>
+
+</head>
+
+<body class="no-js">
+
+    <div class="main-wrapper" role="main">
+        <div class="main-content">
+            <h1 class="zone-name-title h1">
+                <img class="heading-favicon" src="/favicon.ico"
+                    onerror="this.onerror=null;this.parentNode.removeChild(this)" />
+                0MAGNET.COM
+            </h1>
+            <h2 class="h2" id="cf-challenge-running">
+                Checking if the site connection is secure
+            </h2>
+            <noscript>
+                <div id="cf-challenge-error-title">
+                    <div class="h2">
+                        <span class="icon-wrapper">
+                            <div class="heading-icon warning-icon"></div>
+                        </span>
+                        <span id="cf-challenge-error-text">
+                            Enable JavaScript and cookies to continue
+                        </span>
+                    </div>
+                </div>
+            </noscript>
+            <div
+                style="display:none;background-image:url('/cdn-cgi/images/trace/captcha/nojs/transparent.gif?ray=732fd3bc9c1d72de')">
+            </div>
+            <div id="cf-challenge-body-text" class="core-msg spacer">
+                0magnet.com needs to review the security of your connection before
+                proceeding.
+            </div>
+            <form id="challenge-form"
+                action="/search?q=2022&amp;__cf_chl_f_tk=y4XnN88eYeUiXmFkQeqEipve1VuK0jJA.G4Hz6xztsM-1659202277-0-gaNycGzNBz0"
+                method="POST" enctype="application/x-www-form-urlencoded">
+                <input type="hidden" name="md"
+                    value="DpGhFnuVRfDhqsQNASrgdT4WiiJ8m6lqTIs03.l6RLc-1659202277-0-AfUEAk9DsJ4rmpVI_Al7-eogy2CmM3YgWe4-31iw0oG2CcDIbYvauEW2IvK9m27_gq1FvdH-UPaGHR0q6Q2haXlX4pgmQK5rlQUSEd5HquGdtWMasHWqL_Q_TZGdOKz30bE2FEk8wLHRErHJRJloDRj0tiG8MreT2La_GLvovNK1XbMXDxFZT2Cc-DThBvxbgbDffw3okYfdl1ECXhLw9G6L4o8xgLsz3QZQG3dNZNhm5n4mf55-BBsFDzDTEN1_1BgORVw3mtbsodedktcACsVBCRupyBpTev9MML1jHzk06ZT9dhcCP4zXvsMS4-gG212LFu79Cpl0MHifKvPk0DTJQja1ulaT4gVuIvmLPihPh1IYMGbEcdX4MFH0Wu_RL6UPINE6esf-oAx8-imKhKITB_R4974rpq9XJk65Kf9R6AJhu072CyOqW1YcmYMkUCqFjdZnRyNgHRT2Q5bMEJ8fv0DwfFV6ynG7n6JGMd_pEnZp0nEvjWXpK6Ft8ZZGOXtFMfmW4vNgFhs6xJ1wnaJWuLXae3V6gTZYxMkeIsyMzlvRSzYBz_rgRBNkvvAwbNvOZ369tKbaElS39hOI1WTaoOsnY2d0Z4mDe4AVbSs3fVJGikzZSa3Ctr1RnqqOztVIRYL1Q7IYRJ02P6egL7sn7RniJ6znNAoPhaWJLYzynWXeQF5YO5U0Zf779qkm3A" />
+                <input type="hidden" name="r"
+                    value="QJznOl.RWpNvkdG1Pf6TAzaNhRIFpH8DJ0w1yAuwRLw-1659202277-0-AcOBapBisncM3qf1RYdkTNlIXCth/TmoAMnk3vJozFlG8/vYeLPpjG389mhQu01aSlpJqFWn0VQf9c/7w3yh85jHmrpaxJtpxTiSL9k+AWm61kE6DkHgJBl5jUc7gu4W3oHdmP4FyOUzhbBpIkOAntSkVJJmgu6SaIE3I9fRAFu7bPxBveT8zZGyVUJSPKpwx/w4rNPzs2VnCeEVL1eOdbLInHYR1kqC8M4JyBynwdVXxIX+j5o/rTrNK8E/W4UZMhuWqIaOnX7FzmceglyBSjDqJFLCt0TOhc66m82Y25Obi8Gvsqn34bjwPA2G8qOvgrHA2RFH6lEQFSdGMzLrF4qU5P9j9FzU1CPSTfGtkKbsMnGcMrtzmyQ7LdMIfghYvnCBXTi82iIzaSwzY3sEnW9KZs24Akxu/AV1E03sqW1CAA1UCRURpX4GKXvD6UYpSgc6++q8naLdRozkLP81T/CvHyIRdQx8vylmVN9u/rPvMbW1jWtniDmuAjBQDUd058YH+IRmm4lREG5JN2yeX083h/BG6tssEQVdTcIgwZRNDB+kK8vtOmywmo5qTAX1VE/sgfPCw5+3Xxu+hhZON3C7VGfrCQI5ZSb6+YvBLXmO26Nlp3fSOyeBwZy3pVuGwv/TrEo+e8USIlIs1T6MQJYQeX/4vOdy89npo6KBqY23giTFDh8EMZo//93hfBsRUbHrY/It6kp42qzsnWTjbkyiqd1zBSpQhuMyuMPeKpQ63oVI2tlGyioLg3HcfhbHQcdpUAWDn8lZ4+GTFVMix+20fGbErkVeBs7WvFSLlZ1YtYpCXrgVaomj7WCr8Icb7ASXKfvEuqC1ZnZgn6Lb6x3dUBGiDtnSFnixHFElIF6nPedVIV0+TxccjlV/LJeyNM58GHtRo4NcmIo1a6kN3vzPAjTUhgDJe4aYP6oVKCRNDcrHlGlLubu6XuIvBFM5Sq401xxahOe3VP2u7JovkzXwfl+yUxQOYaoq1LR+wnDhXgVbBNbM2QfIhez578zu2TN5bu5H14UXZ1E78KA6Op9b/PUgA1AsgTJVRk4M6OQSpa5wRIKkXzxpGIRz6+YBxSjIaX2I220GH4s6Te4CBpq77g6V4CVIkEvqZwbN9hIoAoljWVbEEdb3WmYZqoPxN/8ZIjU7uwQUyDgnCOlc7Z52TgG6nVvj7RVyxv5ugskW+fcOI12o35iYNNpXTh1boHyn7nlPG7wtSsl9UlTss27nd04AIzbH0qyX3kn77yPsobMDYUJ3IGhOujV8Cg08XHFIlYSYGPbqqpog+CuuWtzvwyk5mmHXkJNPyFEZL/irApJbatpGNqgGNnQL+5KYp+/U8/kROLTOWa8tG5609MF+wdrScsfPT9eE+HYh7tEFURnwm8kJtdAadcxYjzO60PFcUI1R5SMGHRflAnpY2gvAzbsSssk1WIF+6eHSe6FLHMXCHMp0w1XkNKpny5Ce3YTKhJ4TRg7HfN1pvet2Duj4G04A328uYUppPlU7Spz0fj5N/FHJf3sPaqJC8jn74L0mT92ecGaxS3ZGvytw51ulA00wgzfZDWL4pirzgYVjUQTqVl9FzWYua4Vk4l3BX0opWKA4FloLTP3ekrvmO/zkztMBV4fvK+F8JIOzLOs4AuoCv8uXl7Ny9wLQI3a0hJAdbXJpI3WV/iuV7da4fQao2Z2HiatQh3ZtdLqWmGtqQlcVtsBrac82eo7mKAfwltTfLlX9Drtp4ohwoFe0Upm+YsfY6DK7zHrk3k9GN7gm6cMi1neNFaqWZR9s8ABDBg==" />
+
+            </form>
+        </div>
+    </div>
+    <script>
+        (function () {
+            var trkjs = document.createElement('img');
+            trkjs.setAttribute('src', '/cdn-cgi/images/trace/captcha/js/transparent.gif?ray=732fd3bc9c1d72de');
+            trkjs.setAttribute('style', 'display: none');
+            document.body.appendChild(trkjs);
+            var cpo = document.createElement('script');
+            cpo.src = '/cdn-cgi/challenge-platform/h/g/orchestrate/managed/v1?ray=732fd3bc9c1d72de';
+            window._cf_chl_opt.cOgUHash = location.hash === '' && location.href.indexOf('#') !== -1 ? '#' : location.hash;
+            window._cf_chl_opt.cOgUQuery = location.search === '' && location.href.slice(0, -window._cf_chl_opt.cOgUHash.length).indexOf('?') !== -1 ? '?' : location.search;
+            if (window.history && window.history.replaceState) {
+                var ogU = location.pathname + window._cf_chl_opt.cOgUQuery + window._cf_chl_opt.cOgUHash;
+                history.replaceState(null, null, "\/search?q=2022&__cf_chl_rt_tk=y4XnN88eYeUiXmFkQeqEipve1VuK0jJA.G4Hz6xztsM-1659202277-0-gaNycGzNBz0" + window._cf_chl_opt.cOgUHash);
+                cpo.onload = function () {
+                    history.replaceState(null, null, ogU);
+                };
+            }
+            document.getElementsByTagName('head')[0].appendChild(cpo);
+        }());
+    </script>
+
+    <div class="footer" role="contentinfo">
+        <div class="footer-inner">
+            <div class="clearfix diagnostic-wrapper">
+                <div class="ray-id">Ray ID: <code>732fd3bc9c1d72de</code></div>
+            </div>
+            <div class="text-center">
+                Performance &amp; security by
+                <a rel="noopener noreferrer" href="https://www.cloudflare.com" target="_blank">Cloudflare</a>
+            </div>
+        </div>
+    </div>
+</body>
+
+</html>

html_samples/cloudflare_spinner_v1.html

@@ -0,0 +1,167 @@
+<html lang="en-US">
+
+<head>
+    <title>Just a moment...</title>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=Edge">
+    <meta name="robots" content="noindex,nofollow">
+    <meta name="viewport" content="width=device-width,initial-scale=1">
+    <link href="/cdn-cgi/styles/cf-errors.css" rel="stylesheet">
+
+    <script>
+        (function () {
+            window._cf_chl_opt = {
+                cvId: '2',
+                cType: 'managed',
+                cNounce: '52875',
+                cRay: '732fa2449b567521',
+                cHash: '79cce74ebb92671',
+                cUPMDTk: "\/search?q=2022&__cf_chl_tk=1qWQAgl8.irfEoDb73Rb0pUm1SXbis3ZamDAIoTcPks-1659200251-0-gaNycGzNCFE",
+                cFPWv: 'g',
+                cTTimeMs: '1000',
+                cTplV: 2,
+                cRq: {
+                    ru: 'aHR0cHM6Ly8wbWFnbmV0LmNvbS9zZWFyY2g/cT0yMDIy',
+                    ra: 'TW96aWxsYS81LjAgKFgxMTsgTGludXggeDg2XzY0KSBBcHBsZVdlYktpdC81MzcuMzYgKEtIVE1MLCBsaWtlIEdlY2tvKSBDaHJvbWUvMTAzLjAuNTA2MC4xMzQgU2FmYXJpLzUzNy4zNg==',
+                    rm: 'R0VU',
+                    d: 'UfK0k9mFeKGEdqoWAUIbk3OXbXe9DOHoYXdKLPyxbICSIQBS4GSNYar0DtbPI7+UQ7UeBZ2XCdQinvgH0pgzJCF1qB0nkXtu0qlLk6EwkrGAKD/pMGFFQF2EaCw3m00/xoRCDgLZRl/wUkRGz3HUOkTuPeKgZjsFyPoPv7MbYSMUtH7QU6ruIh+O3hvDOT2oA/BOKbRMSTnFedTIXADXL6GE8ZyNZ33wJlef5KzT0MHlN+3eZTAt6urCvJaY3MdTXKVye6fwyjqGEksaJ6B85vwrifLTYEU4/bORwXx8mTQTqjo3kh1rATlmthQwBpcQtWXmgDUcJ5gPrOk1fzhqrhO4b++HiIx3P5YZ9Ko2D0NNWeg1AYIwDjh9rZg5m0MmCXh1VqXDbnpseQW1vPkkZAADxyvLf/eEc1o2EpYGpK+qSpMZ4RcngnU0o8A2nS+j/CNsid0315OrYVyOIZcw6L3ovu6yfAAAALyOmg5ctXCqjzRthoibUb58u+myxOtfX1ew9IzNq8Z6t6RlomjR7Iy/7BJiJQNCF98dllNbODHz//TymlI1m8D9w+CYlZFIpiWJVH1M4h+tabH5YrqDVbkJgY6yVAfnr/NI6d6NHrhN+eSW30jkvAmZ6JRMhVWW',
+                    t: 'MTY1OTIwMDI1MS42MjAwMDA=',
+                    m: '/e8nTBb03IHZzN/DSkoHPRu0Ndm3ynYs8g6ZC+VxHcc=',
+                    i1: 'tx+ntPfeE2Gv81s52vIOlA==',
+                    i2: 'fpw8a/EO+Fo2t/ZiNKxEcg==',
+                    zh: 'qP4bnGc6j96JlnjNSE7HmQci3S9L50bHFtm4bQRjjKU=',
+                    uh: 'Eex9UQDjphKtV6LyVQ95F/MC5kBA3Rj4lC6CudiU3Vs=',
+                    hh: 'azXzJl8Ou22g0nN/9idVUoB9EqZ7fLmkSdDRHM3Lkmw=',
+                }
+            }
+            window._cf_chl_enter = function () { window._cf_chl_opt.p = 1 };
+        })();
+    </script>
+
+    <script src="/cdn-cgi/challenge-platform/h/g/orchestrate/managed/v1?ray=732fa2449b567521"></script>
+    <script type="text/javascript"
+        src="https://cloudflare.hcaptcha.com/1/api.js?endpoint=https%3A%2F%2Fcloudflare.hcaptcha.com&amp;assethost=https%3A%2F%2Fcf-assets.hcaptcha.com&amp;imghost=https%3A%2F%2Fcf-imgs.hcaptcha.com&amp;render=explicit&amp;recaptchacompat=off&amp;onload=_cf_chl_hload"></script>
+</head>
+
+<body class="no-js">
+
+    <div class="privacy-pass">
+        <a rel="noopener noreferrer"
+            href="https://chrome.google.com/webstore/detail/privacy-pass/ajhmfdgkijocedmfjonnpjfojldioehi"
+            target="_blank">
+            Privacy Pass
+            <span class="privacy-pass-icon-wrapper">
+                <div class="privacy-pass-icon"></div>
+            </span>
+        </a>
+    </div>
+
+    <div class="main-wrapper" role="main">
+        <div class="main-content">
+            <h1 class="zone-name-title h1">
+                <img class="heading-favicon" src="/favicon.ico"
+                    onerror="this.onerror=null;this.parentNode.removeChild(this)">
+                0MAGNET.COM
+            </h1>
+            <h2 class="h2" id="cf-challenge-running">
+                Checking if the site connection is secure
+            </h2>
+            <div id="cf-challenge-stage" style="display: none;"></div>
+            <div id="cf-challenge-spinner" class="spacer loading-spinner" style="display: block; visibility: visible;">
+                <div class="lds-ring">
+                    <div></div>
+                    <div></div>
+                    <div></div>
+                    <div></div>
+                </div>
+            </div>
+            <noscript>
+                <div id="cf-challenge-error-title">
+                    <div class="h2">
+                        <span class="icon-wrapper">
+                            <div class="heading-icon warning-icon"></div>
+                        </span>
+                        <span id="cf-challenge-error-text">
+                            Enable JavaScript and cookies to continue
+                        </span>
+                    </div>
+                </div>
+            </noscript>
+            <div
+                style="display:none;background-image:url('/cdn-cgi/images/trace/captcha/nojs/transparent.gif?ray=732fa2449b567521')">
+            </div>
+            <div id="cf-challenge-body-text" class="core-msg spacer">
+                0magnet.com needs to review the security of your connection before
+                proceeding.
+            </div>
+            <div id="cf-challenge-fact-wrapper" class="fact spacer hidden" style="display: block; visibility: visible;">
+                <span class="fact-title">Did you know</span> <span id="cf-challenge-fact" class="body-text">bots
+                    historically made up nearly 40% of all internet traffic?</span>
+            </div>
+            <div id="cf-challenge-explainer-expandable" class="hidden expandable body-text spacer"
+                style="display: none;">
+                <div class="expandable-title" id="cf-challenge-explainer-summary"><button class="expandable-summary-btn"
+                        id="cf-challenge-explainer-btn" type="button"> Why am I seeing this page? <span
+                            class="caret-icon-wrapper">
+                            <div class="caret-icon"></div>
+                        </span> </button> </div>
+                <div class="expandable-details" id="cf-challenge-explainer-details"> Requests from malicious bots can
+                    pose as legitimate traffic. Occasionally, you may see this page while the site ensures that the
+                    connection is secure.</div>
+            </div>
+            <div id="cf-challenge-success" style="display: none;">
+                <div class="h2"><span class="icon-wrapper"><img class="heading-icon" alt="Success icon"
+                            src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADQAAAA0CAMAAADypuvZAAAANlBMVEUAAAAxMTEwMDAxMTExMTEwMDAwMDAwMDAxMTExMTExMTEwMDAwMDAxMTExMTEwMDAwMDAxMTHB9N+uAAAAEXRSTlMA3zDvfyBAEJC/n3BQz69gX7VMkcMAAAGySURBVEjHnZZbFoMgDEQJiDzVuv/NtgbtFGuQ4/zUKpeMIQbUhXSKE5l1XSn4pFWHRm/WShT1HRLWC01LGxFEVkCc30eYkLJ1Sjk9pvkw690VY6k8DWP9OM9yMG0Koi+mi8XA36NXmW0UXra4eJ3iwHfrfXVlgL0NqqGBHdqfeQhMmyJ48WDuKP81h3+SMPeRKkJcSXiLUK4XTHCjESOnz1VUXQoc6lgi2x4cI5aTQ201Mt8wHysI5fc05M5c81uZEtHcMKhxZ7iYEty1GfhLvGKpm+EYkdGxm1F5axmcB93DoORIbXfdN7f+hlFuyxtDP+sxtBnF43cIYwaZAWRgzxIoiXEMESoPlMhwLRDXeK772CAzXEdBRV7cmnoVBp0OSlyGidEzJTFq5hhcsA5388oSGM6b5p+qjpZrBlMS9xj4AwXmz108ukU1IomM3ceiW0CDwHCqp1NjAqXlFrbga+xuloQJ+tuyfbIBPNpqnmxqT7dPaOnZqBfhSBCteJAxWj58zLk2xgg+SPGYM6dRO6WczSnIxxwEExRaO+UyCUhbOp7CGQ+kxSUfNtLQFC+Po29vvy7jj4y0yAAAAABJRU5ErkJggg=="></span>Connection
+                    is secure</div>
+                <div class="core-msg spacer">Proceeding...</div>
+            </div>
+            <form id="challenge-form"
+                action="/search?q=2022&amp;__cf_chl_f_tk=1qWQAgl8.irfEoDb73Rb0pUm1SXbis3ZamDAIoTcPks-1659200251-0-gaNycGzNCFE"
+                method="POST" enctype="application/x-www-form-urlencoded">
+                <input type="hidden" name="md"
+                    value="OghUU_ltYW6I0fpWl7rE4yHBGBPHfpZQIKZRSEpJKjE-1659200251-0-AWB-KR-MabhObmvYa3mR5-xDk3qZVV73547wjnl-QtfPoTxe017AXt4WUskEcVzEIUKC7dsJoiy8ec1NA0fxdnI8X9OfPhtynl00ReWBVZc_3Gba_wigWMmM_9e8PX9vpVDcXpCRbz1BJ5_YLsba9TJM1sp14U9RtIce-tRBB53qoxLxJRz9QFmckEVBvsba4RfoycOvYPMMsfAqSkq13qtsA3Kd6RDB5Rb5-qF8674DsB4AMvd9xu_fBplQqKjOpEtrThCUtw8M2DHY8FUr_owUo1NIS1s6fSBEyHh6ehz9CidJ7zpRwYZFwgz_Pq9i8LmQG_AajozOJJhLp-tox0dptbUZnRNGt3hGQgrNu3jlCfwPC2XVp7xgLvmZoPYrzzrZoi_wErnIvVgyGCw9-sDPblPdvLBUz6uXNreWwThEW6PeRtMXnePO9UwcZmj_2awhwcVSHSLz1t1z22LtVsQ8xNpMbiE7xDvI2D5LNHAPIUC7Wp4AcehWD-fEm0w5jnVTWOmFlVRxtcnYZSMfDSaRUxsZ3hg5B1-ghVMEX6M-r_hAd6pLKNmjIfdl_Nvdm6veQvV-gTFaULbfuhmQQjYEb9G2IptDiNTZs5S7FtmjqVBAA7PmvwBTQwxw86J0cV3v_4pT1Oj8tigwiPny35HMTrKRmRZWaAZudCmWxDZkJIW8Eir7KQ57ba-u9cHh0A">
+                <input type="hidden" name="r"
+                    value="q.UUtPBFcFi4IkcVw3l4U_xJJKDIbHJj7xmuB43IIAI-1659200251-0-Aa4lU5RipD+d4of3hcdQ0rVmZ4ulb3siZYKwhm1jGNiA+/9b8IW1HL8k1GrsYEVexDW7ycP5UINQZ1sYJvZBTCQe3lhyGLHdLZ7KdI9RXKEbPx1NUOR/HthCD0Wbo7H41jbAf7l+HhH0zTLjm77/6NpJZHcgfsbBwwubl4R3oLarzPSByV2PVBnkuMyKCYgibriuMUt2iJHoMLx7Cr+Bmjx1KEFCrPYP0t7vgQs2APTylhL7ebP77XB9ndxU6Of3r4eHnTwLIcomFJ3+jqL6pzFaNoXdUBHrv9oZs/33KZjf2NB8cu5KUpAdM2lp3t5oTSQE19fJVroxmf91hcTdele3F2DAeawFGDwncm/Jo725SlyNk4TqsmR+il7DLkS/FTcCNzQe4cQM6DRWdmF9I1OohAl1/uGXYqUJSK1F45n3gec/pyPTQyZI0OLc7sCGYXfn3VPFsGATkg5mxE9rgZIB2b6ID9JggzIlDdYxlQRWecpruu07KOgk3m7g95lyHNZTohqemo4T8Z2MOZECjmXGMAuwvvk4d5sakVHr39kmAY6aSfXrRB+iONCOKkahbumrVmjLsnMvrpTb0DFE5pRAxwANPZKzb6Ikmlvxh7oJIPOB0mG9hDeoc/AJVlvZJV4CrpDLulNjHetAWXMwMptZuYJGEhcDXxmYj0ybntTCU4Y3JJQc5K+7ehSdnluTvMueWfs628854r4PcOONZzsO337j+3lUxrP5vDUCzYD25FNxvs8jGfqRivqHMOq2z9iOs0sHQTlHroLLSt2G7M50yRJBGTfxrIsvLq+ML3e/mRIkYIQxOcp8ugoPoT4c9gex3OyY0cnnA2/9OibQs9kevwf9DSnutMRRcbIXZI0XO6FY07+MykWqUcXygwMHs1vQxhaQ26NFYwolEWfOL7EQpp4GKyN30nL4nPNil/7GsXIr5SC+o55KI0l3AOEYE1jirVx2G0U7Br7SW80Ih4Fn5U/+4qFfW57GAJrpuk9qjFfJehe7wFBu5bHghEGRhKAu0wvpY7UTc9AiacMfP7ujVWi4DIbTCfOzOgVT8E0T6KaUurBppPJflLQE41c8n29ULyKmki9t8lIKvxYmv/3/AauhXFAExh+JrdnaeSDxhJYjWEUDJiaNvnkDCHMxPs/bePhSg4DYRMh4ngcOHCRkkRlDjipUUgeCrwNBY0qu2DIqLZXI1ZMwU+R0nuWnwc5xJuMHtrLkWbziP0FQcGaF0B6SaFIcOLnWG7YjJZFzxjFpvLb8GnZxk7i2YHCDTn0Stq3JDZHCkjJQjaPMmuK+5KYzfaSHcKOcaQbkbyDjn3t/XQX3a7lknngVchIJVsVn8osqgKvOx3aAdCicYKR6QaukrXHhR9uIEbPdoBYqZPKFz0uvVOShsUx2f65CaI8wWMjOBRWxTK1xUPNsetOiyYSvNwjeULaCXPKLi2qv/cZRRbsr3g5ghdHvNTpD/O0/xUgiziev3/9CpNopyr6VzLar9dJ/s++imXY1w1TCRJ2uCI2H70XGBGWxSZdbnfxU+j3zNCL0dBuabwhDd4ZnOZmlFZjGBiOUpsWdRrHd3c+QpwXdxB3QurRwX6J+LhmkqcWsPhP7LlMnN7dr2HUFZ5FS4LASl5AOf8hjCnO06FT8fWLl1eKVVjCugx9w54qjGqOV8A0v/PdWr7Ic0WfriyYbmwn/XnH8t0ri3bqDZsDkfhQMMF9JSWHEdoGD60a7McGDxr4g9s3LZhq5KozgSvyG+RUBPla8g2zB253hR7amWE5WO4IChl7AXmRB89F9u2+AoDIbefseb3pwG7GfkYpSBwmgJ4Ju4LAWoSfBhZSPMQadHZOCg36R11KesUy+NAy9bvD1bE3UMx9e2NbFohu6sXlilpnxINHp0sFEeulreEjWSQreri1eZeKxV2QfKIzWUiMoNdyT0JzM+/brYzddBpO2DrlnK5bEPWgtu0D7d4Kfm+0T7S//Fq+hxf40lSMPP8cBlan6sEd2iWmZ6gW3z43wNbJaPQIUDgb58ELxaEKQN4tOOy75/XXfISNnhG0K8M79a175WUb8v0A=">
+
+                <span style="display: none;"><span class="text-gray-600" data-translate="error">error code:
+                        1020</span></span>
+            </form>
+        </div>
+    </div>
+    <script>
+        (function () {
+            var trkjs = document.createElement('img');
+            trkjs.setAttribute('src', '/cdn-cgi/images/trace/captcha/js/transparent.gif?ray=732fa2449b567521');
+            trkjs.setAttribute('style', 'display: none');
+            document.body.appendChild(trkjs);
+            var cpo = document.createElement('script');
+            cpo.src = '/cdn-cgi/challenge-platform/h/g/orchestrate/managed/v1?ray=732fa2449b567521';
+            window._cf_chl_opt.cOgUHash = location.hash === '' && location.href.indexOf('#') !== -1 ? '#' : location.hash;
+            window._cf_chl_opt.cOgUQuery = location.search === '' && location.href.slice(0, -window._cf_chl_opt.cOgUHash.length).indexOf('?') !== -1 ? '?' : location.search;
+            if (window.history && window.history.replaceState) {
+                var ogU = location.pathname + window._cf_chl_opt.cOgUQuery + window._cf_chl_opt.cOgUHash;
+                history.replaceState(null, null, "\/search?q=2022&__cf_chl_rt_tk=1qWQAgl8.irfEoDb73Rb0pUm1SXbis3ZamDAIoTcPks-1659200251-0-gaNycGzNCFE" + window._cf_chl_opt.cOgUHash);
+                cpo.onload = function () {
+                    history.replaceState(null, null, ogU);
+                };
+            }
+            document.getElementsByTagName('head')[0].appendChild(cpo);
+        }());
+    </script><img src="/cdn-cgi/images/trace/captcha/js/transparent.gif?ray=732fa2449b567521" style="display: none">
+
+    <div class="footer" role="contentinfo">
+        <div class="footer-inner">
+            <div class="clearfix diagnostic-wrapper">
+                <div class="ray-id">Ray ID: <code>732fa2449b567521</code></div>
+            </div>
+            <div class="text-center">
+                Performance &amp; security by
+                <a rel="noopener noreferrer" href="https://www.cloudflare.com" target="_blank">Cloudflare</a>
+            </div>
+        </div>
+    </div>
+
+
+</body>
+
+</html>

install.js

@@ -0,0 +1,40 @@
+const fs = require('fs');
+const puppeteer = require('puppeteer');
+
+(async () => {
+
+  // Puppeteer does not allow to download Firefox revisions, just the last Nightly
+  // We this script we can download any version
+  const revision = '94.0a1';
+  const downloadHost = 'https://archive.mozilla.org/pub/firefox/nightly/2021/10/2021-10-01-09-33-23-mozilla-central';
+
+  // skip installation (for Dockerfile)
+  if (process.env.PUPPETEER_EXECUTABLE_PATH) {
+    console.log('Skipping Firefox installation because the environment variable "PUPPETEER_EXECUTABLE_PATH" is set.');
+    return;
+  }
+
+  // check if Firefox is already installed
+  const f = puppeteer.createBrowserFetcher({
+    product: 'firefox',
+    host: downloadHost
+  })
+  if (fs.existsSync(f._getFolderPath(revision))) {
+    console.log(`Firefox ${revision} already installed...`)
+    return;
+  }
+
+  console.log(`Installing firefox ${revision} ...`)
+  const downloadPath = f._downloadsFolder;
+  console.log(`Download path: ${downloadPath}`)
+  if (fs.existsSync(downloadPath)) {
+    console.log(`Removing previous downloads...`)
+    fs.rmSync(downloadPath, { recursive: true })
+  }
+
+  console.log(`Downloading firefox ${revision} ...`)
+  await f.download(revision)
+
+  console.log('Installation complete...')
+
+})()

jest.config.js

@@ -0,0 +1,12 @@
+module.exports = {
+  // A list of paths to directories that Jest should use to search for files in
+  roots: [
+    "./src/"
+  ],
+  // Compile Typescript
+  transform: {
+    '^.+\\.(ts|tsx)$': 'ts-jest'
+  },
+  // Default value for FlareSolverr maxTimeout is 60000
+  testTimeout: 70000
+}

package.json

@@ -1,46 +1,46 @@
 {
   "name": "flaresolverr",
-  "version": "1.2.0",
+  "version": "2.2.7",
   "description": "Proxy server to bypass Cloudflare protection.",
   "scripts": {
-    "start": "node ./dist/index.js",
+    "install": "node install.js",
+    "start": "tsc && node ./dist/server.js",
     "build": "tsc",
-    "dev": "nodemon -e ts --exec ts-node src/index.ts"
+    "dev": "nodemon -e ts --exec ts-node src/server.ts",
+    "package": "tsc && node build-binaries.js",
+    "test": "jest --runInBand"
   },
   "author": "Diego Heras (ngosang)",
-  "contributors": [
-    {
-      "name": "Noah Cardoza",
-      "url": "https://github.com/NoahCardoza/CloudProxy.git"
-    }
-  ],
   "license": "MIT",
   "repository": {
     "type": "git",
     "url": "https://github.com/ngosang/FlareSolverr"
   },
+  "bin": {
+    "flaresolverr": "dist/server.js"
+  },
   "dependencies": {
+    "await-timeout": "^1.1.1",
+    "body-parser": "^1.20.0",
     "console-log-level": "^1.4.1",
-    "got": "^11.5.1",
+    "express": "^4.18.1",
-    "hcaptcha-solver": "^1.0.2",
+    "puppeteer": "^13.7.0",
-    "puppeteer": "^3.3.0",
+    "uuid": "^8.3.2"
-    "puppeteer-extra": "^3.1.15",
-    "puppeteer-extra-plugin-stealth": "^2.6.5",
-    "uuid": "^8.2.0"
   },
   "devDependencies": {
-    "@types/node": "^14.0.23",
+    "@types/await-timeout": "^0.3.1",
-    "@types/puppeteer": "^3.0.1",
+    "@types/body-parser": "^1.19.2",
-    "@types/uuid": "^8.0.0",
+    "@types/express": "^4.17.13",
-    "eslint": "^7.5.0",
+    "@types/jest": "^28.1.6",
-    "eslint-config-airbnb-base": "^14.2.0",
+    "@types/node": "^18.6.2",
-    "eslint-config-standard": "^14.1.1",
+    "@types/supertest": "^2.0.12",
-    "eslint-plugin-import": "^2.22.0",
+    "@types/uuid": "^8.3.4",
-    "eslint-plugin-node": "^11.1.0",
+    "archiver": "^5.3.1",
-    "eslint-plugin-promise": "^4.2.1",
+    "nodemon": "^2.0.19",
-    "eslint-plugin-standard": "^4.0.1",
+    "pkg": "^5.8.0",
-    "nodemon": "^2.0.4",
+    "supertest": "^6.2.4",
-    "ts-node": "^8.10.2",
+    "ts-jest": "^28.0.7",
-    "typescript": "^3.9.7"
+    "ts-node": "^10.9.1",
+    "typescript": "^4.7.4"
   }
 }

resources/flaresolverr_logo.svg

@@ -0,0 +1,180 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="256"
+   height="256"
+   viewBox="0 0 256 256"
+   id="svg2"
+   version="1.1"
+   inkscape:version="0.91 r13725"
+   sodipodi:docname="flaresolverr_logo.svg"
+   inkscape:export-filename="C:\Users\Diego\Desktop\flaresolverr_logo.png"
+   inkscape:export-xdpi="90"
+   inkscape:export-ydpi="90">
+  <defs
+     id="defs4" />
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="0.64"
+     inkscape:cx="-88.263072"
+     inkscape:cy="-93.571587"
+     inkscape:document-units="px"
+     inkscape:current-layer="layer1"
+     showgrid="false"
+     inkscape:window-width="2560"
+     inkscape:window-height="1377"
+     inkscape:window-x="-8"
+     inkscape:window-y="-8"
+     inkscape:window-maximized="1"
+     units="px"
+     showborder="true" />
+  <metadata
+     id="metadata7">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Capa 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(0,-796.36219)">
+    <g
+       id="g4177"
+       transform="matrix(0.51436047,0,0,0.59495735,-334.60687,650.43877)">
+      <g
+         id="g4141" />
+      <g
+         id="g4143" />
+      <g
+         id="g4145" />
+      <g
+         id="g4147" />
+      <g
+         id="g4149" />
+      <g
+         id="g4151" />
+      <g
+         id="g4153" />
+      <g
+         id="g4155" />
+      <g
+         id="g4157" />
+      <g
+         id="g4159" />
+      <g
+         id="g4161" />
+      <g
+         id="g4163" />
+      <g
+         id="g4165" />
+      <g
+         id="g4167" />
+      <g
+         id="g4169" />
+      <g
+         id="g4263"
+         transform="matrix(0.94954959,0,0,0.94954959,-111.49858,393.65111)">
+        <g
+           id="g4269" />
+        <g
+           id="g4342"
+           transform="translate(736.24631,-345.97247)">
+          <path
+             style="fill:#9dc6fb"
+             inkscape:connector-curvature="0"
+             d="m 584.32729,454.42324 c -0.995,-51.995 -44.49,-93.257 -96.488,-91.376 -7.616,0.273 -14.792,-3.862 -18.446,-10.55 -22.605,-41.376 -66.519,-69.441 -116.989,-69.441 -51.757,0 -96.596,29.528 -118.647,72.648 -6.423,12.56 -19.224,10.9 -24.689,10.9 -40.126,0 -74.199,25.852 -86.512,61.804 -2.046,5.973 -6.938,10.463 -12.894,12.556 -22.389998,7.87 -38.250998,29.605 -37.275998,54.902 1.163,30.174 26.849,53.631 57.044998,53.631 l 359.817,0 c 52.291,0 96.08,-42.793 95.079,-95.074 z"
+             id="path4285" />
+          <path
+             style="fill:#80b4fb"
+             inkscape:connector-curvature="0"
+             d="m 190.59629,495.86724 c -0.975,-25.298 14.885,-47.033 37.276,-54.902 5.956,-2.094 10.848,-6.584 12.894,-12.556 12.313,-35.952 46.385,-61.804 86.512,-61.804 5.465,0 18.265,1.66 24.688,-10.9 13.005,-25.43 33.94,-46.125 59.541,-58.832 -17.812,-8.834 -37.873,-13.816 -59.103,-13.816 -51.757,0 -96.596,29.528 -118.647,72.648 -6.423,12.56 -19.224,10.9 -24.689,10.9 -40.126,0 -74.199,25.852 -86.512,61.804 -2.046,5.973 -6.938,10.463 -12.894,12.556 -22.389998,7.87 -38.250998,29.605 -37.275998,54.902 1.163,30.174 26.849,53.63 57.044998,53.63 l 118.21,0 c -30.196,0 -55.881,-23.457 -57.045,-53.63 z"
+             id="path4287" />
+        </g>
+      </g>
+    </g>
+    <g
+       id="g4241"
+       transform="matrix(0.1453379,0,0,0.1453379,47.012211,854.83732)"
+       style="fill:#4d4d4d">
+      <g
+         id="g4197"
+         style="fill:#4d4d4d;fill-opacity:1">
+        <path
+           id="path4201"
+           d="m 867.699,356.238 -31.5,-26.6 c -9.699,-8.2 -24,-7.8 -33.199,0.9 l -17.4,16.3 c -14.699,-7.1 -30.299,-12.1 -46.4,-15 l -4.898,-24 c -2.5,-12.4 -14,-21 -26.602,-20 l -41.1,3.5 c -12.6,1.1 -22.5,11.4 -22.9,24.1 l -0.799,24.4 c -15.801,5.7 -30.701,13.5 -44.301,23.3 l -20.799,-13.8 c -10.602,-7 -24.701,-5 -32.9,4.7 l -26.6,31.7 c -8.201,9.7 -7.801,24 0.898,33.2 l 18.201,19.399 c -6.301,14.2 -10.801,29.101 -13.4,44.4 l -26,5.3 c -12.4,2.5 -21,14 -20,26.601 l 3.5,41.1 c 1.1,12.6 11.4,22.5 24.1,22.9 l 28.1,0.899 c 5.102,13.4 11.801,26.101 19.9,38 l -15.699,23.7 c -7,10.6 -5,24.7 4.699,32.9 l 31.5,26.6 c 9.701,8.2 24,7.8 33.201,-0.9 l 20.6,-19.3 c 13.5,6.3 27.699,11 42.299,13.8 l 5.701,28.2 c 2.5,12.4 14,21 26.6,20 l 41.1,-3.5 c 12.6,-1.1 22.5,-11.399 22.9,-24.1 l 0.9,-27.601 c 15,-5.3 29.199,-12.5 42.299,-21.399 l 22.701,15 c 10.6,7 24.699,5 32.9,-4.7 l 26.6,-31.5 c 8.199,-9.7 7.799,-24 -0.9,-33.2 L 872.7,592.138 c 6.701,-14.2 11.602,-29.2 14.4,-44.601 l 25,-5.1 c 12.4,-2.5 21,-14 20,-26.601 l -3.5,-41.1 c -1.1,-12.6 -11.4,-22.5 -24.1,-22.9 l -25.1,-0.8 c -5.201,-14.6 -12.201,-28.399 -20.9,-41.2 l 13.699,-20.6 c 7.201,-10.598 5.201,-24.798 -4.5,-32.998 z M 712.801,593.837 c -44.4,3.801 -83.602,-29.3 -87.301,-73.699 -3.801,-44.4 29.301,-83.601 73.699,-87.301 44.4,-3.8 83.602,29.301 87.301,73.7 3.801,44.401 -29.301,83.601 -73.699,87.3 z"
+           inkscape:connector-curvature="0"
+           style="fill:#4d4d4d;fill-opacity:1" />
+        <path
+           id="path4203"
+           d="m 205,704.438 c -12.6,1.3 -22.3,11.899 -22.4,24.6 l -0.3,25.3 c -0.2,12.7 9.2,23.5 21.8,25.101 l 18.6,2.399 c 3.1,11.301 7.5,22.101 13.2,32.301 l -12,14.8 c -8,9.899 -7.4,24.1 1.5,33.2 l 17.7,18.1 c 8.9,9.1 23.1,10.1 33.2,2.3 l 14.899,-11.5 c 10.5,6.2 21.601,11.101 33.2,14.5 l 2,19.2 c 1.3,12.6 11.9,22.3 24.6,22.4 l 25.301,0.3 c 12.699,0.2 23.5,-9.2 25.1,-21.8 l 2.3,-18.2 c 12.601,-3.101 24.601,-7.8 36,-14 l 14,11.3 c 9.9,8 24.101,7.4 33.201,-1.5 l 18.1,-17.7 c 9.1,-8.899 10.1,-23.1 2.301,-33.2 L 496.6,818.438 c 6.6,-11 11.701,-22.7 15.201,-35 l 16.6,-1.7 c 12.6,-1.3 22.299,-11.9 22.4,-24.6 l 0.299,-25.301 c 0.201,-12.699 -9.199,-23.5 -21.799,-25.1 l -16.201,-2.1 c -3.1,-12.2 -7.699,-24 -13.699,-35 l 10.1,-12.4 c 8,-9.9 7.4,-24.1 -1.5,-33.2 l -17.699,-18.1 c -8.9,-9.101 -23.102,-10.101 -33.201,-2.3 l -12.101,9.3 c -11.399,-6.9 -23.6,-12.2 -36.399,-15.8 L 407,581.437 c -1.3,-12.601 -11.899,-22.3 -24.6,-22.4 l -25.3,-0.3 c -12.7,-0.2 -23.5,9.2 -25.101,21.8 l -2,15.601 c -13.199,3.399 -25.899,8.6 -37.699,15.399 l -12.5,-10.2 c -9.9,-8 -24.101,-7.399 -33.201,1.5 l -18.2,17.801 c -9.1,8.899 -10.1,23.1 -2.3,33.199 l 10.7,13.801 c -6.2,11 -11.1,22.699 -14.3,35 l -17.499,1.8 z m 163.3,-28.601 c 36.3,0.4 65.399,30.301 65,66.601 -0.4,36.3 -30.301,65.399 -66.601,65 -36.3,-0.4 -65.399,-30.3 -65,-66.601 0.401,-36.299 30.301,-65.399 66.601,-65 z"
+           inkscape:connector-curvature="0"
+           style="fill:#4d4d4d;fill-opacity:1" />
+      </g>
+      <g
+         id="g4205"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4207"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4209"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4211"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4213"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4215"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4217"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4219"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4221"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4223"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4225"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4227"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4229"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4231"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4233"
+         style="fill:#4d4d4d" />
+    </g>
+  </g>
+</svg>

src/app.ts

@@ -0,0 +1,83 @@
+import log from './services/log'
+import {NextFunction, Request, Response} from 'express';
+import {getUserAgent} from "./services/sessions";
+import {controllerV1} from "./controllers/v1";
+
+const express = require('express');
+const app = express();
+const bodyParser = require('body-parser');
+const version: string = 'v' + require('../package.json').version
+
+// Convert request objects to JSON
+app.use(bodyParser.json({
+    limit: '50mb',
+    verify(req: Request, res: Response, buf: any) {
+        req.body = buf;
+    }
+}));
+
+// Access log
+app.use(function(req: Request, res: Response, next: NextFunction) {
+    if (req.url != '/health') {
+        // count the request for the log prefix
+        log.incRequests()
+        // build access message
+        let body = "";
+        if (req.method == 'POST' && req.body) {
+            body += " body: "
+            try {
+                body += JSON.stringify(req.body)
+            } catch(e) {
+                body += req.body
+            }
+        }
+        log.info(`Incoming request => ${req.method} ${req.url}${body}`);
+    }
+    next();
+});
+
+// *********************************************************************************************************************
+// Routes
+
+// Show welcome message
+app.get("/", ( req: Request, res: Response ) => {
+    res.send({
+        "msg": "FlareSolverr is ready!",
+        "version": version,
+        "userAgent": getUserAgent()
+    });
+});
+
+// Health endpoint. this endpoint is special because it doesn't print traces
+app.get("/health", ( req: Request, res: Response ) => {
+    res.send({
+        "status": "ok"
+    });
+});
+
+// Controller v1
+app.post("/v1", async( req: Request, res: Response ) => {
+    await controllerV1(req, res);
+});
+
+// *********************************************************************************************************************
+
+// Unknown paths or verbs
+app.use(function (req : Request, res : Response) {
+    res.status(404)
+        .send({"error": "Unknown resource or HTTP verb"})
+})
+
+// Errors
+app.use(function (err: any, req: Request, res: Response, next: NextFunction) {
+    if (err) {
+        let msg = 'Invalid request: ' + err;
+        msg = msg.replace("\n", "").replace("\r", "")
+        log.error(msg)
+        res.send({"error": msg})
+    } else {
+        next()
+    }
+})
+
+module.exports = app;

src/captcha/harvester.ts

@@ -1,31 +0,0 @@
-import got from 'got'
-import { sleep } from '../utils'
-
-/*
-    This method uses the captcha-harvester project:
-        https://github.com/NoahCardoza/CaptchaHarvester
-
-    While the function must take url/sitekey/type args,
-    they aren't used because the harvester server must
-    be preconfigured.
-
-    ENV:
-        HARVESTER_ENDPOINT: This must be the full path
-        to the /token endpoint of the harvester.
-        E.G. "https://127.0.0.1:5000/token"
-*/
-
-export default async function solve(): Promise<string> {
-  const endpoint = process.env.HARVESTER_ENDPOINT
-  if (!endpoint) { throw Error('ENV variable `HARVESTER_ENDPOINT` must be set.') }
-  while (true) {
-    try {
-      return (await got.get(process.env.HARVESTER_ENDPOINT, {
-        https: { rejectUnauthorized: false }
-      })).body
-    } catch (e) {
-      if (e.response.statusCode !== 418) { throw e }
-    }
-    await sleep(3000)
-  }
-}

src/captcha/hcaptcha-solver.ts

@@ -1,21 +0,0 @@
-const solveCaptcha = require('hcaptcha-solver');
-import { SolverOptions } from '.'
-/*
-    This method uses the hcaptcha-solver project:
-        https://github.com/JimmyLaurent/hcaptcha-solver
-
-    TODO: allow user pass custom options to the solver.
-
-    ENV:
-        There are no other variables that must be set to get this to work
-*/
-
-export default async function solve({ url }: SolverOptions): Promise<string> {
-  try {
-    const token = await solveCaptcha(url)
-    return token
-  } catch (e) {
-    console.error(e)
-    return null
-  }
-}

src/captcha/index.ts

@@ -1,3 +1,5 @@
+import log from "../services/log";
+
 export enum CaptchaType {
   re = 'reCaptcha',
   h = 'hCaptcha'
@@ -16,7 +18,9 @@ const captchaSolvers: { [key: string]: Solver } = {}
 export default (): Solver => {
   const method = process.env.CAPTCHA_SOLVER
 
-  if (!method) { return null }
+  if (!method || method.toLowerCase() == 'none') {
+    return null;
+  }
 
   if (!(method in captchaSolvers)) {
     try {
@@ -26,10 +30,12 @@ export default (): Solver => {
         throw Error(`The solver '${method}' is not a valid captcha solving method.`)
       } else {
         console.error(e)
-        throw Error(`An error occured loading the solver '${method}'.`)
+        throw Error(`An error occurred loading the solver '${method}'.`)
       }
     }
   }
 
+  log.info(`Using '${method}' to solve the captcha.`);
+
   return captchaSolvers[method]
-}
+}

src/controllers/v1.ts

@@ -0,0 +1,178 @@
+import {Request, Response} from 'express';
+import {Protocol} from "devtools-protocol";
+
+import log from '../services/log'
+import {browserRequest, ChallengeResolutionResultT, ChallengeResolutionT} from "../services/solver";
+import {SessionCreateOptions} from "../services/sessions";
+const sessions = require('../services/sessions')
+const version: string = 'v' + require('../../package.json').version
+
+interface V1Routes {
+  [key: string]: (params: V1RequestBase, response: V1ResponseBase) => Promise<void>
+}
+
+export interface Proxy {
+  url?: string
+  username?: string
+  password?: string
+}
+
+export interface V1RequestBase {
+  cmd: string
+  cookies?: Protocol.Network.CookieParam[],
+  maxTimeout?: number
+  proxy?: Proxy
+  session: string
+  headers?: Record<string, string> // deprecated v2, not used
+  userAgent?: string // deprecated v2, not used
+}
+
+interface V1RequestSession extends V1RequestBase {
+}
+
+export interface V1Request extends V1RequestBase {
+  url: string
+  method?: string
+  postData?: string
+  returnOnlyCookies?: boolean
+  download?: boolean // deprecated v2, not used
+  returnRawHtml?: boolean // deprecated v2, not used
+}
+
+export interface V1ResponseBase {
+  status: string
+  message: string
+  startTimestamp: number
+  endTimestamp: number
+  version: string
+}
+
+export interface V1ResponseSolution extends V1ResponseBase {
+  solution: ChallengeResolutionResultT
+}
+
+export interface V1ResponseSession extends V1ResponseBase {
+  session: string
+}
+
+export interface V1ResponseSessions extends V1ResponseBase {
+  sessions: string[]
+}
+
+export const routes: V1Routes = {
+  'sessions.create': async (params: V1RequestSession, response: V1ResponseSession): Promise<void> => {
+    const options: SessionCreateOptions = {
+      oneTimeSession: false,
+      cookies: params.cookies,
+      maxTimeout: params.maxTimeout,
+      proxy: params.proxy
+    }
+    const { sessionId, browser } = await sessions.create(params.session, options)
+    if (browser) {
+      response.status = "ok";
+      response.message = "Session created successfully.";
+      response.session = sessionId
+    } else {
+      throw Error('Error creating session.')
+    }
+  },
+  'sessions.list': async (params: V1RequestSession, response: V1ResponseSessions): Promise<void> => {
+    response.status = "ok";
+    response.message = "";
+    response.sessions = sessions.list();
+  },
+  'sessions.destroy': async (params: V1RequestSession, response: V1ResponseBase): Promise<void> => {
+    if (await sessions.destroy(params.session)) {
+      response.status = "ok";
+      response.message = "The session has been removed.";
+    } else {
+      throw Error('This session does not exist.')
+    }
+  },
+  'request.get': async (params: V1Request, response: V1ResponseSolution): Promise<void> => {
+    params.method = 'GET'
+    if (params.postData) {
+      throw Error('Cannot use "postBody" when sending a GET request.')
+    }
+    if (params.returnRawHtml) {
+      log.warn("Request parameter 'returnRawHtml' was removed in FlareSolverr v2.")
+    }
+    if (params.download) {
+      log.warn("Request parameter 'download' was removed in FlareSolverr v2.")
+    }
+    const result: ChallengeResolutionT = await browserRequest(params)
+
+    response.status = result.status;
+    response.message = result.message;
+    response.solution = result.result;
+    if (response.message) {
+      log.info(response.message)
+    }
+  },
+  'request.post': async (params: V1Request, response: V1ResponseSolution): Promise<void> => {
+    params.method = 'POST'
+    if (!params.postData) {
+      throw Error('Must send param "postBody" when sending a POST request.')
+    }
+    if (params.returnRawHtml) {
+      log.warn("Request parameter 'returnRawHtml' was removed in FlareSolverr v2.")
+    }
+    if (params.download) {
+      log.warn("Request parameter 'download' was removed in FlareSolverr v2.")
+    }
+    const result: ChallengeResolutionT = await browserRequest(params)
+
+    response.status = result.status;
+    response.message = result.message;
+    response.solution = result.result;
+    if (response.message) {
+      log.info(response.message)
+    }
+  },
+}
+
+export async function controllerV1(req: Request, res: Response): Promise<void> {
+  const response: V1ResponseBase = {
+    status: null,
+    message: null,
+    startTimestamp: Date.now(),
+    endTimestamp: 0,
+    version: version
+  }
+
+  try {
+    const params: V1RequestBase = req.body
+    // do some validations
+    if (!params.cmd) {
+      throw Error("Request parameter 'cmd' is mandatory.")
+    }
+    if (params.headers) {
+      log.warn("Request parameter 'headers' was removed in FlareSolverr v2.")
+    }
+    if (params.userAgent) {
+      log.warn("Request parameter 'userAgent' was removed in FlareSolverr v2.")
+    }
+
+    // set default values
+    if (!params.maxTimeout || params.maxTimeout < 1) {
+      params.maxTimeout = 60000;
+    }
+
+    // execute the command
+    const route = routes[params.cmd]
+    if (route) {
+      await route(params, response)
+    } else {
+      throw Error(`The command '${params.cmd}' is invalid.`)
+    }
+  } catch (e) {
+    res.status(500)
+    response.status = "error";
+    response.message = e.toString();
+    log.error(response.message)
+  }
+
+  response.endTimestamp = Date.now()
+  log.info(`Response in ${(response.endTimestamp - response.startTimestamp) / 1000} s`)
+  res.send(response)
+}

src/index.ts

@@ -1,106 +0,0 @@
-import log from './log'
-import { createServer, IncomingMessage, ServerResponse } from 'http';
-import { RequestContext } from './types'
-import Router, { BaseAPICall } from './routes'
-
-const version: string = require('../package.json').version
-const serverPort: number = Number(process.env.PORT) || 8191
-const serverHost: string = process.env.HOST || '0.0.0.0'
-
-
-function errorResponse(errorMsg: string, res: ServerResponse, startTimestamp: number) {
-  log.error(errorMsg)
-  const response = {
-    status: 'error',
-    message: errorMsg,
-    startTimestamp,
-    endTimestamp: Date.now(),
-    version
-  }
-  res.writeHead(500, {
-    'Content-Type': 'application/json'
-  })
-  res.write(JSON.stringify(response))
-  res.end()
-}
-
-function successResponse(successMsg: string, extendedProperties: object, res: ServerResponse, startTimestamp: number) {
-  const endTimestamp = Date.now()
-  log.info(`Successful response in ${(endTimestamp - startTimestamp) / 1000} s`)
-  if (successMsg) { log.info(successMsg) }
-
-  const response = Object.assign({
-    status: 'ok',
-    message: successMsg || '',
-    startTimestamp,
-    endTimestamp,
-    version
-  }, extendedProperties || {})
-  res.writeHead(200, {
-    'Content-Type': 'application/json'
-  })
-  res.write(JSON.stringify(response))
-  res.end()
-}
-
-function validateIncomingRequest(ctx: RequestContext, params: BaseAPICall) {
-  log.info(`Params: ${JSON.stringify(params)}`)
-
-  if (ctx.req.method !== 'POST') {
-    ctx.errorResponse('Only the POST method is allowed')
-    return false
-  }
-
-  if (ctx.req.url !== '/v1') {
-    ctx.errorResponse('Only /v1 endpoint is allowed')
-    return false
-  }
-
-  if (!params.cmd) {
-    ctx.errorResponse("Parameter 'cmd' is mandatory")
-    return false
-  }
-
-  return true
-}
-
-createServer((req: IncomingMessage, res: ServerResponse) => {
-  // count the request for the log prefix
-  log.incRequests()
-
-  const startTimestamp = Date.now()
-  log.info(`Incoming request: ${req.method} ${req.url}`)
-  const bodyParts: any[] = []
-  req.on('data', chunk => {
-    bodyParts.push(chunk)
-  }).on('end', () => {
-    // parse params
-    const body = Buffer.concat(bodyParts).toString()
-    let params: BaseAPICall = null
-    try {
-      params = JSON.parse(body)
-    } catch (err) {
-      errorResponse('Body must be in JSON format', res, startTimestamp)
-      return
-    }
-
-    const ctx: RequestContext = {
-      req,
-      res,
-      startTimestamp,
-      errorResponse: (msg) => errorResponse(msg, res, startTimestamp),
-      successResponse: (msg, extendedProperties) => successResponse(msg, extendedProperties, res, startTimestamp)
-    }
-
-    // validate params
-    if (!validateIncomingRequest(ctx, params)) { return }
-
-    // process request
-    Router(ctx, params).catch(e => {
-      console.error(e)
-      ctx.errorResponse(e.message)
-    })
-  })
-}).listen(serverPort, serverHost, () => {
-  log.info(`FlareSolverr v${version} listening on http://${serverHost}:${serverPort}`)
-})

src/log.ts

@@ -1,19 +0,0 @@
-let requests = 0
-
-const LOG_HTML: boolean = Boolean(process.env.LOG_HTML) || false
-
-export default {
-  incRequests: () => { requests++ },
-  html(html: string) {
-    if (LOG_HTML)
-      this.debug(html)
-  },
-  ...require('console-log-level')(
-    {
-      level: process.env.LOG_LEVEL || 'debug',
-      prefix(level: string) {
-        return `${new Date().toISOString()} ${level.toUpperCase()} REQ-${requests}`
-      }
-    }
-  )
-}

src/providers/cloudflare.ts

@@ -0,0 +1,163 @@
+import {Page, HTTPResponse} from 'puppeteer'
+
+import log from "../services/log";
+
+/**
+ *  This class contains the logic to solve protections provided by CloudFlare
+ **/
+
+const BAN_SELECTORS: string[] = [];
+const CHALLENGE_SELECTORS: string[] = [
+    // todo: deprecate  '#trk_jschal_js', '#cf-please-wait'
+    '#cf-challenge-running', '#trk_jschal_js', '#cf-please-wait', // CloudFlare
+    '#link-ddg', // DDoS-GUARD
+    'td.info #js_info' // Custom CloudFlare for EbookParadijs, Film-Paleis, MuziekFabriek and Puur-Hollands
+];
+const CAPTCHA_SELECTORS: string[] = [
+    // todo: deprecate 'input[name="cf_captcha_kind"]'
+    '#cf-challenge-hcaptcha-wrapper', '#cf-norobot-container', 'input[name="cf_captcha_kind"]'
+];
+
+const delay = (ms: number) => new Promise(resolve => setTimeout(resolve, ms))
+
+export default async function resolveChallenge(url: string, page: Page, response: HTTPResponse): Promise<HTTPResponse> {
+
+  // look for challenge and return fast if not detected
+  let cfDetected = response.headers().server &&
+      (response.headers().server.startsWith('cloudflare') || response.headers().server.startsWith('ddos-guard'));
+  if (cfDetected) {
+    if (response.status() == 403 || response.status() == 503) {
+      cfDetected = true; // Defected CloudFlare and DDoS-GUARD
+    } else if (response.headers().vary && response.headers().vary.trim() == 'Accept-Encoding,User-Agent' &&
+        response.headers()['content-encoding'] && response.headers()['content-encoding'].trim() == 'br') {
+      cfDetected = true; // Detected Custom CloudFlare for EbookParadijs, Film-Paleis, MuziekFabriek and Puur-Hollands
+    } else {
+      cfDetected = false;
+    }
+  }
+
+  if (cfDetected) {
+    log.info('Cloudflare detected');
+  } else {
+    log.info('Cloudflare not detected');
+    return response;
+  }
+
+  if (await findAnySelector(page, BAN_SELECTORS)) {
+    throw new Error('Cloudflare has blocked this request. Probably your IP is banned for this site, check in your web browser.');
+  }
+
+  // find Cloudflare selectors
+  let selectorFound = false;
+  let selector: string = await findAnySelector(page, CHALLENGE_SELECTORS)
+  if (selector) {
+    selectorFound = true;
+    log.debug(`Javascript challenge element '${selector}' detected.`)
+    log.debug('Waiting for Cloudflare challenge...')
+
+    while (true) {
+      try {
+
+        selector = await findAnySelector(page, CHALLENGE_SELECTORS)
+        if (!selector) {
+          // solved!
+          log.debug('Challenge element not found')
+          break
+
+        } else {
+          log.debug(`Javascript challenge element '${selector}' detected.`)
+
+          // check for CAPTCHA challenge
+          if (await findAnySelector(page, CAPTCHA_SELECTORS)) {
+            // captcha detected
+            break
+          }
+
+          // new Cloudflare Challenge #cf-please-wait
+          const displayStyle = await page.evaluate((selector) => {
+            return getComputedStyle(document.querySelector(selector)).getPropertyValue("display");
+          }, selector);
+          if (displayStyle == "none") {
+            // spinner is hidden, could be a captcha or not
+            log.debug('Challenge element is hidden')
+            log.debug("Waiting for 30 secs");
+            await delay(30 * 1000);
+            // wait until redirecting disappears
+            while (true) {
+              try {
+                await page.waitForTimeout(1000)
+                const displayStyle2 = await page.evaluate(() => {
+                  return getComputedStyle(document.querySelector('#cf-spinner-redirecting')).getPropertyValue("display");
+                });
+                if (displayStyle2 == "none") {
+                  break // hCaptcha detected
+                }
+              } catch (error) {
+                break // redirection completed
+              }
+            }
+            break
+          } else {
+            log.debug('Challenge element is visible')
+          }
+        }
+        log.debug('Found challenge element again')
+
+      } catch (error)
+      {
+        log.debug("Unexpected error: " + error);
+        if (!error.toString().includes("Execution context was destroyed")) {
+          break
+        }
+      }
+
+      log.debug('Waiting for Cloudflare challenge...')
+      await page.waitForTimeout(1000)
+    }
+
+    log.debug('Validating HTML code...')
+  } else {
+    log.debug(`No challenge element detected.`)
+  }
+
+  // check for CAPTCHA challenge
+  if (await findAnySelector(page, CAPTCHA_SELECTORS)) {
+    log.info('CAPTCHA challenge detected');
+    throw new Error('FlareSolverr can not resolve CAPTCHA challenges. Since the captcha doesn\'t always appear, you may have better luck with the next request.');
+
+    // const captchaSolver = getCaptchaSolver()
+    // if (captchaSolver) {
+    //     // to-do: get the params
+    //     log.info('Waiting to receive captcha token to bypass challenge...')
+    //     const token = await captchaSolver({
+    //       url,
+    //       sitekey,
+    //       type: captchaType
+    //     })
+    //     log.debug(`Token received: ${token}`);
+    //     // to-do: send the token
+    //   }
+    // } else {
+    //   throw new Error('Captcha detected but no automatic solver is configured.');
+    // }
+  } else {
+    if (!selectorFound)
+    {
+      throw new Error('No challenge selectors found, unable to proceed.')
+    } else {
+      log.info('Challenge solved');
+    }
+  }
+
+  return response;
+}
+
+async function findAnySelector(page: Page, selectors: string[]) {
+  for (const selector of selectors) {
+    const cfChallengeElem = await page.$(selector)
+    if (cfChallengeElem) {
+      return selector;
+    }
+  }
+  return null;
+}

src/routes.ts

@@ -1,471 +0,0 @@
-import { v1 as UUIDv1 } from 'uuid'
-import sessions, { SessionsCacheItem } from './session'
-import { RequestContext } from './types'
-import log from './log'
-import { SetCookie, Request, Headers, HttpMethod, Overrides, Cookie } from 'puppeteer'
-import { TimeoutError } from 'puppeteer/Errors'
-import getCaptchaSolver, { CaptchaType } from './captcha'
-import * as Puppeteer from "puppeteer-extra/dist/puppeteer";
-
-export interface BaseAPICall {
-  cmd: string
-}
-
-interface BaseSessionsAPICall extends BaseAPICall {
-  session?: string
-}
-
-interface SessionsCreateAPICall extends BaseSessionsAPICall {
-  userAgent?: string,
-  cookies?: SetCookie[],
-  headers?: Headers
-  maxTimeout?: number
-  proxy?: any
-}
-
-interface BaseRequestAPICall extends BaseAPICall {
-  url: string
-  method?: HttpMethod
-  postData?: string
-  session?: string
-  userAgent?: string
-  maxTimeout?: number
-  cookies?: SetCookie[],
-  headers?: Headers
-  proxy?: any, // TODO: use interface not any
-  download?: boolean
-  returnOnlyCookies?: boolean
-}
-
-
-interface Routes {
-  [key: string]: (ctx: RequestContext, params: BaseAPICall) => void | Promise<void>
-}
-
-interface ChallengeResolutionResultT {
-  url: string
-  status: number,
-  headers?: Headers,
-  response: string,
-  cookies: object[]
-  userAgent: string
-}
-
-interface ChallengeResolutionT {
-  status?: string
-  message: string
-  result: ChallengeResolutionResultT
-}
-
-interface OverrideResolvers {
-  method?: (request: Request) => HttpMethod,
-  postData?: (request: Request) => string,
-  headers?: (request: Request) => Headers
-}
-
-type OverridesProps =
-  'method' |
-  'postData' |
-  'headers'
-
-// We always set a Windows User-Agent because ARM builds are detected by CloudFlare
-const DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36"
-const CHALLENGE_SELECTORS = ['#trk_jschal_js', '.ray_id', '.attack-box']
-const TOKEN_INPUT_NAMES = ['g-recaptcha-response', 'h-captcha-response']
-
-async function interceptResponse(page: Puppeteer.Page, callback: (payload: ChallengeResolutionT) => any) {
-  const client = await page.target().createCDPSession();
-  await client.send('Fetch.enable', {
-    patterns: [
-      {
-        urlPattern: '*',
-        resourceType: 'Document',
-        requestStage: 'Response',
-      },
-    ],
-  });
-
-  client.on('Fetch.requestPaused', async (e) => {
-    log.debug('Fetch.requestPaused. Checking if the response has valid cookies')
-    let headers = e.responseHeaders || []
-
-    let cookies = await page.cookies();
-    log.debug(cookies)
-
-    if (cookies.filter((c: Cookie) => c.name === 'cf_clearance').length > 0) {
-      log.debug('Aborting request and return cookies. valid cookies found')
-      await client.send('Fetch.failRequest', {requestId: e.requestId, errorReason: 'Aborted'})
-
-      let status = 'ok'
-      let message = ''
-      const payload: ChallengeResolutionT = {
-        status,
-        message,
-        result: {
-          url: page.url(),
-          status: e.status,
-          headers: headers.reduce((a: any, x: { name: any; value: any }) => ({ ...a, [x.name]: x.value }), {}),
-          response: null,
-          cookies: cookies,
-          userAgent: ''
-        }
-      }
-
-      callback(payload);
-    } else {
-      log.debug('Continuing request. no valid cookies found')
-      await client.send('Fetch.continueRequest', {requestId: e.requestId})
-    }
-  });
-}
-
-async function resolveChallenge(ctx: RequestContext, { url, maxTimeout, proxy, download, returnOnlyCookies }: BaseRequestAPICall, page: Puppeteer.Page): Promise<ChallengeResolutionT | void> {
-
-  maxTimeout = maxTimeout || 60000
-  let status = 'ok'
-  let message = ''
-
-  if (proxy) {
-    log.debug("Apply proxy");
-    if (proxy.username)
-      await page.authenticate({ username: proxy.username, password: proxy.password });
-  }
-
-  log.debug(`Navigating to... ${url}`)
-  let response = await page.goto(url, { waitUntil: 'domcontentloaded' })
-
-  // look for challenge
-  if (response.headers().server.startsWith('cloudflare')) {
-    log.info('Cloudflare detected')
-
-    if (await page.$('.cf-error-code')) {
-      await page.close()
-      return ctx.errorResponse('Cloudflare has blocked this request (Code 1020 Detected).')
-    }
-
-    if (response.status() > 400) {
-      // detect cloudflare wait 5s
-      for (const selector of CHALLENGE_SELECTORS) {
-        const cfChallengeElem = await page.$(selector)
-        if (cfChallengeElem) {
-          log.html(await page.content())
-          log.debug('Waiting for Cloudflare challenge...')
-
-          let interceptingResult: ChallengeResolutionT;
-          if (returnOnlyCookies) { //If we just want to get the cookies, intercept the response before we get the content/body (just cookies and headers)
-            await interceptResponse(page, async function(payload){
-              interceptingResult = payload;
-            });
-          }
-
-          // TODO: find out why these pages hang sometimes
-          while (Date.now() - ctx.startTimestamp < maxTimeout) {
-            await page.waitFor(1000)
-            try {
-              // catch exception timeout in waitForNavigation
-              response = await page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 5000 })
-            } catch (error) { }
-
-            if (returnOnlyCookies && interceptingResult) {
-              await page.close();
-              return interceptingResult;
-            }
-
-            try {
-              // catch Execution context was destroyed
-              const cfChallengeElem = await page.$(selector)
-              if (!cfChallengeElem) { break }
-              log.debug('Found challenge element again...')
-            } catch (error)
-            { }
-
-            response = await page.reload({ waitUntil: 'domcontentloaded' })
-            log.debug('Reloaded page...')
-          }
-
-          if (Date.now() - ctx.startTimestamp >= maxTimeout) {
-            ctx.errorResponse(`Maximum timeout reached. maxTimeout=${maxTimeout} (ms)`)
-            return
-          }
-
-          log.debug('Validating HTML code...')
-          break
-        } else {
-          log.debug(`No '${selector}' challenge element detected.`)
-        }
-      }
-    }
-
-    // it seems some captcha pages return 200 sometimes
-    if (await page.$('input[name="cf_captcha_kind"]')) {
-      const captchaSolver = getCaptchaSolver()
-      if (captchaSolver) {
-        const captchaStartTimestamp = Date.now()
-        const challengeForm = await page.$('#challenge-form')
-        if (challengeForm) {
-          log.html(await page.content())
-          const captchaTypeElm = await page.$('input[name="cf_captcha_kind"]')
-          const cfCaptchaType: string = await captchaTypeElm.evaluate((e: any) => e.value)
-          const captchaType: CaptchaType = (CaptchaType as any)[cfCaptchaType]
-          if (!captchaType) { return ctx.errorResponse('Unknown captcha type!') }
-
-          let sitekey = null
-          if (captchaType != 'hCaptcha' && process.env.CAPTCHA_SOLVER != 'hcaptcha-solver') {
-            const sitekeyElem = await page.$('*[data-sitekey]')
-            if (!sitekeyElem) { return ctx.errorResponse('Could not find sitekey!') }
-            sitekey = await sitekeyElem.evaluate((e) => e.getAttribute('data-sitekey'))
-          }
-
-          log.info('Waiting to receive captcha token to bypass challenge...')
-          const token = await captchaSolver({
-            url,
-            sitekey,
-            type: captchaType
-          })
-
-          if (!token) {
-            await page.close()
-            return ctx.errorResponse('Token solver failed to return a token.')
-          }
-
-          for (const name of TOKEN_INPUT_NAMES) {
-            const input = await page.$(`textarea[name="${name}"]`)
-            if (input) { await input.evaluate((e: HTMLTextAreaElement, token) => { e.value = token }, token) }
-          }
-
-          // ignore preset event listeners on the form
-          await page.evaluate(() => {
-            window.addEventListener('submit', (e) => { event.stopPropagation() }, true)
-          })
-
-          // it seems some sites obfuscate their challenge forms
-          // TODO: look into how they do it and come up with a more solid solution
-          try {
-            // this element is added with js and we want to wait for all the js to load before submitting
-            await page.waitForSelector('#challenge-form [type=submit]', { timeout: 5000 })
-          } catch (err) {
-            if (err instanceof TimeoutError) {
-              log.debug(`No '#challenge-form [type=submit]' element detected.`)
-            }
-          }
-
-          // calculates the time it took to solve the captcha
-          const captchaSolveTotalTime = Date.now() - captchaStartTimestamp
-
-          // generates a random wait time
-          const randomWaitTime = (Math.floor(Math.random() * 20) + 10) * 1000
-
-          // waits, if any, time remaining to appear human but stay as fast as possible
-          const timeLeft = randomWaitTime - captchaSolveTotalTime
-          if (timeLeft > 0) { await page.waitFor(timeLeft) }
-
-          let interceptingResult: ChallengeResolutionT;
-          if (returnOnlyCookies) { //If we just want to get the cookies, intercept the response before we get the content/body (just cookies and headers)
-            await interceptResponse(page, async function(payload){
-              interceptingResult = payload;
-            });
-          }
-
-          // submit captcha response
-          challengeForm.evaluate((e: HTMLFormElement) => e.submit())
-          response = await page.waitForNavigation({ waitUntil: 'domcontentloaded' })
-
-          if (returnOnlyCookies && interceptingResult) {
-            await page.close();
-            return interceptingResult;
-          }
-        }
-      } else {
-        status = 'warning'
-        message = 'Captcha detected but no automatic solver is configured.'
-      }
-    }
-  }
-
-  const payload: ChallengeResolutionT = {
-    status,
-    message,
-    result: {
-      url: page.url(),
-      status: response.status(),
-      headers: response.headers(),
-      response: null,
-      cookies: await page.cookies(),
-      userAgent: await page.evaluate(() => navigator.userAgent)
-    }
-  }
-
-  if (download) {
-    // for some reason we get an error unless we reload the page
-    // has something to do with a stale buffer and this is the quickest
-    // fix since I am short on time
-    response = await page.goto(url, { waitUntil: 'domcontentloaded' })
-    payload.result.response = (await response.buffer()).toString('base64')
-  } else {
-    payload.result.response = await page.content()
-  }
-
-  // make sure the page is closed because if it isn't and error will be thrown
-  // when a user uses a temporary session, the browser make be quit before
-  // the page is properly closed.
-  await page.close()
-
-  return payload
-}
-
-function mergeSessionWithParams({ defaults }: SessionsCacheItem, params: BaseRequestAPICall): BaseRequestAPICall {
-  const copy = { ...defaults, ...params }
-
-  // custom merging logic
-  copy.headers = { ...defaults.headers || {}, ...params.headers || {} } || null
-
-  return copy
-}
-
-async function setupPage(ctx: RequestContext, params: BaseRequestAPICall, browser: Puppeteer.Browser): Promise<Puppeteer.Page> {
-  const page = await browser.newPage()
-
-  // merge session defaults with params
-  const { method, postData, userAgent, headers, cookies } = params
-
-  let overrideResolvers: OverrideResolvers = {}
-
-  if (method !== 'GET') {
-    log.debug(`Setting method to ${method}`)
-    overrideResolvers.method = request => method
-  }
-
-  if (postData) {
-    log.debug(`Setting body data to ${postData}`)
-    overrideResolvers.postData = request => postData
-  }
-
-  if (userAgent) {
-    log.debug(`Using custom UA: ${userAgent}`)
-    await page.setUserAgent(userAgent)
-  } else {
-    await page.setUserAgent(DEFAULT_USER_AGENT)
-  }
-
-  if (headers) {
-    log.debug(`Adding custom headers: ${JSON.stringify(headers, null, 2)}`,)
-    overrideResolvers.headers = request => Object.assign(request.headers(), headers)
-  }
-
-  if (cookies) {
-    log.debug(`Setting custom cookies: ${JSON.stringify(cookies, null, 2)}`,)
-    await page.setCookie(...cookies)
-  }
-
-  // if any keys have been set on the object
-  if (Object.keys(overrideResolvers).length > 0) {
-    log.debug(overrideResolvers)
-    let callbackRunOnce = false
-    const callback = (request: Request) => {
-
-      if (callbackRunOnce || !request.isNavigationRequest()) {
-        request.continue()
-        return
-      }
-
-      callbackRunOnce = true
-      const overrides: Overrides = {}
-
-      Object.keys(overrideResolvers).forEach((key: OverridesProps) => {
-        // @ts-ignore
-        overrides[key] = overrideResolvers[key](request)
-      });
-
-      log.debug(overrides)
-
-      request.continue(overrides)
-    }
-
-    await page.setRequestInterception(true)
-    page.on('request', callback)
-  }
-
-  return page
-}
-
-const browserRequest = async (ctx: RequestContext, params: BaseRequestAPICall) => {
-  const oneTimeSession = params.session === undefined
-  const sessionId = params.session || UUIDv1()
-  const session = oneTimeSession
-    ? await sessions.create(sessionId, {
-      userAgent: params.userAgent,
-      oneTimeSession
-    })
-    : sessions.get(sessionId)
-
-  if (session === false) {
-    return ctx.errorResponse('This session does not exist. Use \'list_sessions\' to see all the existing sessions.')
-  }
-
-  params = mergeSessionWithParams(session, params)
-
-  try {
-    const page = await setupPage(ctx, params, session.browser)
-    const data = await resolveChallenge(ctx, params, page)
-
-    if (data) {
-      const { status } = data
-      delete data.status
-      ctx.successResponse(data.message, {
-        ...(oneTimeSession ? {} : { session: sessionId }),
-        ...(status ? { status } : {}),
-        solution: data.result
-      })
-    }
-  } catch (error) {
-    log.error(error)
-    return ctx.errorResponse("Unable to process browser request. Error: " + error)
-  } finally {
-    if (oneTimeSession) { sessions.destroy(sessionId) }
-  }
-}
-
-export const routes: Routes = {
-  'sessions.create': async (ctx, { session, ...options }: SessionsCreateAPICall) => {
-    session = session || UUIDv1()
-    const { browser } = await sessions.create(session, options)
-    if (browser) { ctx.successResponse('Session created successfully.', { session }) }
-  },
-  'sessions.list': (ctx) => {
-    ctx.successResponse(null, { sessions: sessions.list() })
-  },
-  'sessions.destroy': async (ctx, { session }: BaseSessionsAPICall) => {
-    if (await sessions.destroy(session)) { return ctx.successResponse('The session has been removed.') }
-    ctx.errorResponse('This session does not exist.')
-  },
-  'request.get': async (ctx, params: BaseRequestAPICall) => {
-    params.method = 'GET'
-    if (params.postData) {
-      return ctx.errorResponse('Cannot use "postBody" when sending a GET request.')
-    }
-    await browserRequest(ctx, params)
-  },
-  'request.post': async (ctx, params: BaseRequestAPICall) => {
-    params.method = 'POST'
-
-    if (!params.postData) {
-      return ctx.errorResponse('Must send param "postBody" when sending a POST request.')
-    }
-
-    await browserRequest(ctx, params)
-  },
-  'request.cookies': async (ctx, params: BaseRequestAPICall) => {
-    params.returnOnlyCookies = true
-    params.method = 'GET'
-    if (params.postData) {
-      return ctx.errorResponse('Cannot use "postBody" when sending a GET request.')
-    }
-    await browserRequest(ctx, params)
-  },
-}
-
-export default async function Router(ctx: RequestContext, params: BaseAPICall): Promise<void> {
-  const route = routes[params.cmd]
-  if (route) { return await route(ctx, params) }
-  return ctx.errorResponse(`The command '${params.cmd}' is invalid.`)
-}

src/server.ts

@@ -0,0 +1,63 @@
+import log from './services/log'
+import {testWebBrowserInstallation} from "./services/sessions";
+
+const app = require("./app");
+const version: string = 'v' + require('../package.json').version
+const serverPort: number = Number(process.env.PORT) || 8191
+const serverHost: string = process.env.HOST || '0.0.0.0'
+
+function validateEnvironmentVariables() {
+  // ip and port variables are validated by nodejs
+  if (process.env.LOG_LEVEL && ['error', 'warn', 'info', 'verbose', 'debug'].indexOf(process.env.LOG_LEVEL) == -1) {
+    log.error(`The environment variable 'LOG_LEVEL' is wrong. Check the documentation.`);
+    process.exit(1);
+  }
+  if (process.env.LOG_HTML && ['true', 'false'].indexOf(process.env.LOG_HTML) == -1) {
+    log.error(`The environment variable 'LOG_HTML' is wrong. Check the documentation.`);
+    process.exit(1);
+  }
+  if (process.env.HEADLESS && ['true', 'false'].indexOf(process.env.HEADLESS) == -1) {
+    log.error(`The environment variable 'HEADLESS' is wrong. Check the documentation.`);
+    process.exit(1);
+  }
+  // todo: fix resolvers
+  // try {
+  //   getCaptchaSolver();
+  // } catch (e) {
+  //   log.error(`The environment variable 'CAPTCHA_SOLVER' is wrong. ${e.message}`);
+  //   process.exit(1);
+  // }
+}
+
+// Init
+log.info(`FlareSolverr ${version}`);
+log.debug('Debug log enabled');
+
+process.on('SIGTERM', () => {
+  // Capture signal on Docker Stop #158
+  log.info("Process interrupted")
+  process.exit(0)
+})
+
+process.on('uncaughtException', function(err) {
+  // Avoid crashing in NodeJS 17 due to UnhandledPromiseRejectionWarning: Unhandled promise rejection.
+  log.error(err)
+})
+
+validateEnvironmentVariables();
+
+testWebBrowserInstallation().then(() => {
+  // Start server
+  app.listen(serverPort, serverHost, () => {
+    log.info(`Listening on http://${serverHost}:${serverPort}`);
+  })
+}).catch(function(e) {
+  log.error(e);
+  const msg: string = "" + e;
+  if (msg.includes('while trying to connect to the browser!')) {
+    log.error(`It seems that the system is too slow to run FlareSolverr. 
+    If you are running with Docker, try to remove CPU limits in the container. 
+    If not, try setting the 'BROWSER_TIMEOUT' environment variable and the 'maxTimeout' parameter to higher values.`);
+  }
+  process.exit(1);
+})

src/services/log.ts

@@ -0,0 +1,41 @@
+let requests = 0
+
+const LOG_HTML: boolean = process.env.LOG_HTML == 'true';
+
+function toIsoString(date: Date) {
+  // this function fixes Date.toISOString() adding timezone
+  let tzo = -date.getTimezoneOffset(),
+      dif = tzo >= 0 ? '+' : '-',
+      pad = function(num: number) {
+        let norm = Math.floor(Math.abs(num));
+        return (norm < 10 ? '0' : '') + norm;
+      };
+
+  return date.getFullYear() +
+      '-' + pad(date.getMonth() + 1) +
+      '-' + pad(date.getDate()) +
+      'T' + pad(date.getHours()) +
+      ':' + pad(date.getMinutes()) +
+      ':' + pad(date.getSeconds()) +
+      dif + pad(tzo / 60) +
+      ':' + pad(tzo % 60);
+}
+
+export default {
+  incRequests: () => {
+      requests++
+  },
+  html(html: string) {
+    if (LOG_HTML) {
+        this.debug(html)
+    }
+  },
+  ...require('console-log-level')(
+    {level: process.env.LOG_LEVEL || 'info',
+      prefix(level: string) {
+        const req = (requests > 0) ? ` REQ-${requests}` : '';
+        return `${toIsoString(new Date())} ${level.toUpperCase()}${req}`
+      }
+    }
+  )
+}

src/services/sessions.ts

@@ -0,0 +1,194 @@
+import {v1 as UUIDv1} from 'uuid'
+import * as path from 'path'
+import {Browser} from 'puppeteer'
+import {Protocol} from "devtools-protocol";
+
+import log from './log'
+import {Proxy} from "../controllers/v1";
+
+const os = require('os');
+const fs = require('fs');
+const puppeteer = require('puppeteer');
+
+export interface SessionsCacheItem {
+  sessionId: string
+  browser: Browser
+}
+
+interface SessionsCache {
+  [key: string]: SessionsCacheItem
+}
+
+export interface SessionCreateOptions {
+  oneTimeSession: boolean
+  cookies?: Protocol.Network.CookieParam[],
+  maxTimeout?: number
+  proxy?: Proxy
+}
+
+const sessionCache: SessionsCache = {}
+let webBrowserUserAgent: string;
+
+function buildExtraPrefsFirefox(proxy: Proxy): object {
+  // Default configurations are defined here
+  // https://github.com/puppeteer/puppeteer/blob/v3.3.0/src/Launcher.ts#L481
+  const extraPrefsFirefox = {
+    // Disable newtabpage
+    "browser.newtabpage.enabled": false,
+    "browser.startup.homepage": "about:blank",
+
+    // Do not warn when closing all open tabs
+    "browser.tabs.warnOnClose": false,
+
+    // Disable telemetry
+    "toolkit.telemetry.reportingpolicy.firstRun": false,
+
+    // Disable first-run welcome page
+    "startup.homepage_welcome_url": "about:blank",
+    "startup.homepage_welcome_url.additional": "",
+
+    // Detected !
+    // // Disable images to speed up load
+    // "permissions.default.image": 2,
+
+    // Limit content processes to 1
+    "dom.ipc.processCount": 1
+  }
+
+  // proxy.url format => http://<host>:<port>
+  if (proxy && proxy.url) {
+    log.debug(`Using proxy: ${proxy.url}`)
+    const [host, portStr] = proxy.url.replace(/.+:\/\//g, '').split(':');
+    const port = parseInt(portStr);
+    if (!host || !portStr || !port) {
+      throw new Error("Proxy configuration is invalid! Use the format: protocol://ip:port")
+    }
+
+    const proxyPrefs = {
+      "network.proxy.type": 1,
+      "network.proxy.share_proxy_settings": true
+    }
+    if (proxy.url.indexOf("socks") != -1) {
+      // SOCKSv4 & SOCKSv5
+      Object.assign(proxyPrefs, {
+        "network.proxy.socks": host,
+        "network.proxy.socks_port": port,
+        "network.proxy.socks_remote_dns": true
+      });
+      if (proxy.url.indexOf("socks4") != -1) {
+        Object.assign(proxyPrefs, {
+          "network.proxy.socks_version": 4
+        });
+      } else {
+        Object.assign(proxyPrefs, {
+          "network.proxy.socks_version": 5
+        });
+      }
+    } else {
+      // HTTP
+      Object.assign(proxyPrefs, {
+        "network.proxy.ftp": host,
+        "network.proxy.ftp_port": port,
+        "network.proxy.http": host,
+        "network.proxy.http_port": port,
+        "network.proxy.ssl": host,
+        "network.proxy.ssl_port": port
+      });
+    }
+
+    // merge objects
+    Object.assign(extraPrefsFirefox, proxyPrefs);
+  }
+
+  return extraPrefsFirefox;
+}
+
+export function getUserAgent() {
+  return webBrowserUserAgent
+}
+
+export async function testWebBrowserInstallation(): Promise<void> {
+  log.info("Testing web browser installation...")
+
+  // check user home dir. this dir will be used by Firefox
+  const homeDir = os.homedir();
+  fs.accessSync(homeDir, fs.constants.F_OK | fs.constants.R_OK | fs.constants.W_OK | fs.constants.X_OK);
+  log.debug("FlareSolverr user home directory is OK: " + homeDir)
+
+  // test web browser
+  const testUrl = process.env.TEST_URL || "https://www.google.com";
+  log.debug("Test URL: " + testUrl)
+  const session = await create(null, {
+    oneTimeSession: true
+  })
+  const page = await session.browser.newPage()
+  const pageTimeout = Number(process.env.BROWSER_TIMEOUT) || 40000
+  await page.goto(testUrl, {waitUntil: 'domcontentloaded', timeout: pageTimeout})
+  webBrowserUserAgent = await page.evaluate(() => navigator.userAgent)
+
+  // replace Linux ARM user-agent because it's detected
+  if (["arm", "aarch64"].some(arch => webBrowserUserAgent.toLocaleLowerCase().includes('linux ' + arch))) {
+    webBrowserUserAgent = webBrowserUserAgent.replace(/linux \w+;/i, 'Linux x86_64;')
+  }
+
+  log.info("FlareSolverr User-Agent: " + webBrowserUserAgent)
+  await page.close()
+  await destroy(session.sessionId)
+
+  log.info("Test successful")
+}
+
+export async function create(session: string, options: SessionCreateOptions): Promise<SessionsCacheItem> {
+  log.debug('Creating new session...')
+
+  const sessionId = session || UUIDv1()
+
+  // NOTE: cookies can't be set in the session, you need to open the page first
+
+  const puppeteerOptions: any = {
+    product: 'firefox',
+    headless: process.env.HEADLESS !== 'false',
+    timeout: Number(process.env.BROWSER_TIMEOUT) || 40000
+  }
+
+  puppeteerOptions.extraPrefsFirefox = buildExtraPrefsFirefox(options.proxy)
+
+  // if we are running inside executable binary, change browser path
+  if (typeof (process as any).pkg !== 'undefined') {
+    const exe = process.platform === "win32" ? 'firefox.exe' : 'firefox';
+    puppeteerOptions.executablePath = path.join(path.dirname(process.execPath), 'firefox', exe)
+  }
+
+  log.debug('Launching web browser...')
+  let browser: Browser = await puppeteer.launch(puppeteerOptions)
+  if (!browser) {
+    throw Error(`Failed to launch web browser.`)
+  }
+
+  sessionCache[sessionId] = {
+    sessionId: sessionId,
+    browser: browser
+  }
+
+  return sessionCache[sessionId]
+}
+
+export function list(): string[] {
+  return Object.keys(sessionCache)
+}
+
+export async function destroy(id: string): Promise<boolean>{
+  if (id && sessionCache.hasOwnProperty(id)) {
+    const { browser } = sessionCache[id]
+    if (browser) {
+      await browser.close()
+      delete sessionCache[id]
+      return true
+    }
+  }
+  return false
+}
+
+export function get(id: string): SessionsCacheItem {
+  return sessionCache[id]
+}

src/services/solver.ts

@@ -0,0 +1,206 @@
+import {Page, HTTPResponse} from 'puppeteer'
+const Timeout = require('await-timeout');
+
+import log from './log'
+import {SessionCreateOptions, SessionsCacheItem} from "./sessions";
+import {V1Request} from "../controllers/v1";
+import cloudflareProvider from '../providers/cloudflare';
+
+const sessions = require('./sessions')
+
+export interface ChallengeResolutionResultT {
+    url: string
+    status: number,
+    headers?: Record<string, string>,
+    response: string,
+    cookies: object[]
+    userAgent: string
+}
+
+export interface ChallengeResolutionT {
+    status?: string
+    message: string
+    result: ChallengeResolutionResultT
+}
+
+async function resolveChallengeWithTimeout(params: V1Request, session: SessionsCacheItem) {
+    const timer = new Timeout();
+    try {
+        const promise = resolveChallenge(params, session);
+        return await Promise.race([
+            promise,
+            timer.set(params.maxTimeout, `Maximum timeout reached. maxTimeout=${params.maxTimeout} (ms)`)
+        ]);
+    } finally {
+        timer.clear();
+    }
+}
+
+async function resolveChallenge(params: V1Request, session: SessionsCacheItem): Promise<ChallengeResolutionT | void> {
+    try {
+        let status = 'ok'
+        let message = ''
+
+        const page: Page = await session.browser.newPage()
+
+        // the Puppeter timeout should be half the maxTimeout because we reload the page and wait for challenge
+        // the user can set a really high maxTimeout if he wants to
+        await page.setDefaultNavigationTimeout(params.maxTimeout / 2)
+
+        // the user-agent is changed just for linux arm build
+        await page.setUserAgent(sessions.getUserAgent())
+
+        // set the proxy
+        if (params.proxy) {
+            log.debug(`Using proxy: ${params.proxy.url}`);
+            // todo: credentials are not working
+            // if (params.proxy.username) {
+            //     await page.authenticate({
+            //         username: params.proxy.username,
+            //         password: params.proxy.password
+            //     });
+            // }
+        }
+
+        // go to the page
+        log.debug(`Navigating to... ${params.url}`)
+        let response: HTTPResponse = await gotoPage(params, page);
+
+        // set cookies
+        if (params.cookies) {
+            for (const cookie of params.cookies) {
+                // the other fields in the cookie can cause issues
+                await page.setCookie({
+                    "name": cookie.name,
+                    "value": cookie.value
+                })
+            }
+            // reload the page
+            response = await gotoPage(params, page);
+        }
+
+        // log html in debug mode
+        log.html(await page.content())
+
+        // detect protection services and solve challenges
+        try {
+            response = await cloudflareProvider(params.url, page, response);
+
+            // is response is ok
+            // reload the page to be sure we get the real page
+            log.debug("Reloading the page")
+            try {
+                response = await gotoPage(params, page, params.method);
+            } catch (e) {
+                log.warn("Page not reloaded (do not report!): Cause: " + e.toString())
+            }
+
+        } catch (e) {
+            status = "error";
+            message = "Cloudflare " + e.toString();
+        }
+
+        const payload: ChallengeResolutionT = {
+            status,
+            message,
+            result: {
+                url: page.url(),
+                status: response.status(),
+                headers: response.headers(),
+                response: null,
+                cookies: await page.cookies(),
+                userAgent: sessions.getUserAgent()
+            }
+        }
+
+        if (params.returnOnlyCookies) {
+            payload.result.headers = null;
+            payload.result.userAgent = null;
+        } else {
+            payload.result.response = await page.content()
+        }
+
+        // make sure the page is closed because if it isn't and error will be thrown
+        // when a user uses a temporary session, the browser make be quit before
+        // the page is properly closed.
+        await page.close()
+
+        return payload
+    } catch (e) {
+        log.error("Unexpected error: " + e);
+        throw e;
+    }
+}
+
+async function gotoPage(params: V1Request, page: Page, method: string = 'GET'): Promise<HTTPResponse> {
+    let pageTimeout = params.maxTimeout / 3;
+    let response: HTTPResponse
+
+    try {
+        response = await page.goto(params.url, {waitUntil: 'domcontentloaded', timeout: pageTimeout});
+    } catch (e) {
+        // retry
+        response = await page.goto(params.url, {waitUntil: 'domcontentloaded', timeout: pageTimeout});
+    }
+
+    if (method == 'POST') {
+        // post hack, it only works with utf-8 encoding
+
+        let postForm = `<form id="hackForm" action="${params.url}" method="POST">`;
+        let queryString = params.postData;
+        let pairs = (queryString[0] === '?' ? queryString.substr(1) : queryString).split('&');
+        for (let i = 0; i < pairs.length; i++) {
+            let pair = pairs[i].split('=');
+            let name; try { name = decodeURIComponent(pair[0]) } catch { name = pair[0] }
+            if (name == 'submit') continue;
+            let value; try { value = decodeURIComponent(pair[1] || '') } catch { value = pair[1] || '' }
+            postForm += `<input type="text" name="${name}" value="${value}"><br>`;
+        }
+        postForm += `</form>`;
+
+        await page.setContent(`
+            <!DOCTYPE html>
+            <html>
+            <body>
+            ${postForm}
+            <script>document.getElementById('hackForm').submit();</script>
+            </body>
+            </html>`
+        );
+        await page.waitForTimeout(2000)
+        try {
+            await page.waitForNavigation({waitUntil: 'domcontentloaded', timeout: 2000})
+        } catch (e) {}
+
+    }
+    return response
+}
+
+export async function browserRequest(params: V1Request): Promise<ChallengeResolutionT> {
+    const oneTimeSession = params.session === undefined;
+
+    const options: SessionCreateOptions = {
+        oneTimeSession: oneTimeSession,
+        cookies: params.cookies,
+        maxTimeout: params.maxTimeout,
+        proxy: params.proxy
+    }
+
+    const session: SessionsCacheItem = oneTimeSession
+        ? await sessions.create(null, options)
+        : sessions.get(params.session)
+
+    if (!session) {
+        throw Error('This session does not exist. Use \'list_sessions\' to see all the existing sessions.')
+    }
+
+    try {
+        return  await resolveChallengeWithTimeout(params, session)
+    } catch (error) {
+        throw Error("Unable to process browser request. " + error)
+    } finally {
+        if (oneTimeSession) {
+            await sessions.destroy(session.sessionId)
+        }
+    }
+}

src/session.ts

@@ -1,140 +0,0 @@
-import * as os from 'os'
-import * as path from 'path'
-import * as fs from 'fs'
-
-import puppeteer from 'puppeteer-extra'
-import { LaunchOptions, Headers, SetCookie } from 'puppeteer'
-
-import log from './log'
-import { deleteFolderRecursive, sleep, removeEmptyFields } from './utils'
-import * as Puppeteer from "puppeteer-extra/dist/puppeteer";
-
-interface SessionPageDefaults {
-  headers?: Headers
-  userAgent?: string
-}
-
-export interface SessionsCacheItem {
-  browser: Puppeteer.Browser
-  userDataDir?: string
-  defaults: SessionPageDefaults
-}
-
-interface SessionsCache {
-  [key: string]: SessionsCacheItem
-}
-
-interface SessionCreateOptions {
-  oneTimeSession?: boolean
-  userAgent?: string
-  cookies?: SetCookie[]
-  headers?: Headers,
-  maxTimeout?: number
-  proxy?: any
-}
-
-const sessionCache: SessionsCache = {}
-
-// setting "user-agent-override" evasion is not working for us because it can't be changed
-// in each request. we set the user-agent in the browser args instead
-puppeteer.use(require('puppeteer-extra-plugin-stealth')())
-
-function userDataDirFromId(id: string): string {
-  return path.join(os.tmpdir(), `/puppeteer_chrome_profile_${id}`)
-}
-
-function prepareBrowserProfile(id: string): string {
-  // TODO: maybe pass SessionCreateOptions for loading later?
-  const userDataDir = userDataDirFromId(id)
-
-  if (!fs.existsSync(userDataDir)) {
-    fs.mkdirSync(userDataDir, { recursive: true })
-  }
-
-  return userDataDir
-}
-
-export default {
-  create: async (id: string, { cookies, oneTimeSession, userAgent, headers, maxTimeout, proxy }: SessionCreateOptions): Promise<SessionsCacheItem> => {
-    let args = ['--no-sandbox', '--disable-setuid-sandbox'];
-    if (proxy && proxy.url) {
-      args.push(`--proxy-server=${proxy.url}`);
-    }
-
-    const puppeteerOptions: LaunchOptions = {
-      product: 'chrome',
-      headless: process.env.HEADLESS !== 'false',
-      args
-    }
-
-    if (!oneTimeSession) {
-      log.debug('Creating userDataDir for session.')
-      puppeteerOptions.userDataDir = prepareBrowserProfile(id)
-    }
-
-    log.debug('Launching headless browser...')
-
-    // TODO: maybe access env variable?
-    // TODO: sometimes browser instances are created and not connected to correctly.
-    //       how do we handle/quit those instances inside Docker?
-    let launchTries = 3
-    let browser: Puppeteer.Browser;
-
-    while (0 <= launchTries--) {
-      try {
-        browser = await puppeteer.launch(puppeteerOptions)
-        break
-      } catch (e) {
-        if (e.message !== 'Failed to launch the browser process!')
-          throw e
-        log.warn('Failed to open browser, trying again...')
-      }
-    }
-
-    if (!browser) { throw Error(`Failed to launch browser 3 times in a row.`) }
-
-    if (cookies) {
-      const page = await browser.newPage()
-      await page.setCookie(...cookies)
-    }
-
-    sessionCache[id] = {
-      browser: browser,
-      userDataDir: puppeteerOptions.userDataDir,
-      defaults: removeEmptyFields({
-        userAgent,
-        headers,
-        maxTimeout
-      })
-    }
-
-    return sessionCache[id]
-  },
-
-  list: (): string[] => Object.keys(sessionCache),
-
-  // TODO: create a sessions.close that doesn't rm the userDataDir
-
-  destroy: async (id: string): Promise<boolean> => {
-    const { browser, userDataDir } = sessionCache[id]
-    if (browser) {
-      await browser.close()
-      delete sessionCache[id]
-      if (userDataDir) {
-        const userDataDirPath = userDataDirFromId(id)
-        try {
-          // for some reason this keeps an error from being thrown in Windows, figures
-          await sleep(5000)
-          deleteFolderRecursive(userDataDirPath)
-        } catch (e) {
-          console.error(e)
-          throw Error(`Error deleting browser session folder. ${e.message}`)
-        }
-      }
-      return true
-    }
-    return false
-  },
-
-  get: (id: string): SessionsCacheItem | false => sessionCache[id] && sessionCache[id] || false
-}

src/tests/app.test.ts

@@ -0,0 +1,625 @@
+// noinspection DuplicatedCode
+
+import {Response} from "superagent";
+import {V1ResponseBase, V1ResponseSession, V1ResponseSessions, V1ResponseSolution} from "../controllers/v1"
+
+const request = require("supertest");
+const app = require("../app");
+const sessions = require('../services/sessions');
+const version: string = 'v' + require('../../package.json').version
+
+const proxyUrl = "http://127.0.0.1:8888"
+const proxySocksUrl = "socks5://127.0.0.1:1080"
+const googleUrl = "https://www.google.com";
+const postUrl = "https://ptsv2.com/t/qv4j3-1634496523";
+const cfUrl = "https://nowsecure.nl";
+const cfCaptchaUrl = "https://idope.se"
+const cfBlockedUrl = "https://www.torrentmafya.org/table.php"
+const ddgUrl = "https://anidex.info/";
+const ccfUrl = "https://www.muziekfabriek.org";
+
+beforeAll(async () => {
+    // Init session
+    await sessions.testWebBrowserInstallation();
+});
+
+afterEach(async () => {
+    // Clean sessions
+    const sessionList = sessions.list();
+    for (const session of sessionList) {
+        await sessions.destroy(session);
+    }
+});
+
+describe("Test '/' path", () => {
+    test("GET method should return OK ", async () => {
+        const response: Response = await request(app).get("/");
+        expect(response.statusCode).toBe(200);
+        expect(response.body.msg).toBe("FlareSolverr is ready!");
+        expect(response.body.version).toBe(version);
+        expect(response.body.userAgent).toContain("Firefox/")
+    });
+
+    test("POST method should fail", async () => {
+        const response: Response = await request(app).post("/");
+        expect(response.statusCode).toBe(404);
+        expect(response.body.error).toBe("Unknown resource or HTTP verb");
+    });
+});
+
+describe("Test '/health' path", () => {
+    test("GET method should return OK", async () => {
+        const response: Response = await request(app).get("/health");
+        expect(response.statusCode).toBe(200);
+        expect(response.body.status).toBe("ok");
+    });
+});
+
+describe("Test '/wrong' path", () => {
+    test("GET method should fail", async () => {
+        const response: Response = await request(app).get("/wrong");
+        expect(response.statusCode).toBe(404);
+        expect(response.body.error).toBe("Unknown resource or HTTP verb");
+    });
+});
+
+describe("Test '/v1' path", () => {
+    test("Cmd 'request.bad' should fail", async () => {
+        const payload = {
+            "cmd": "request.bad",
+            "url": googleUrl
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(500);
+
+        const apiResponse: V1ResponseBase = response.body;
+        expect(apiResponse.status).toBe("error");
+        expect(apiResponse.message).toBe("Error: The command 'request.bad' is invalid.");
+        expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
+        expect(apiResponse.endTimestamp).toBeGreaterThanOrEqual(apiResponse.startTimestamp);
+        expect(apiResponse.version).toBe(version);
+    });
+
+    test("Cmd 'request.get' should return OK with no Cloudflare", async () => {
+        const payload = {
+            "cmd": "request.get",
+            "url": googleUrl
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSolution = response.body;
+        expect(apiResponse.status).toBe("ok");
+        expect(apiResponse.message).toBe("");
+        expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
+        expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
+        expect(apiResponse.version).toBe(version);
+
+        const solution = apiResponse.solution;
+        expect(solution.url).toContain(googleUrl)
+        expect(solution.status).toBe(200);
+        expect(Object.keys(solution.headers).length).toBeGreaterThan(0)
+        expect(solution.response).toContain("<!DOCTYPE html>")
+        expect(Object.keys(solution.cookies).length).toBeGreaterThan(0)
+        expect(solution.userAgent).toContain("Firefox/")
+    });
+
+    test("Cmd 'request.get' should return OK with Cloudflare JS", async () => {
+        const payload = {
+            "cmd": "request.get",
+            "url": cfUrl
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSolution = response.body;
+        expect(apiResponse.status).toBe("ok");
+        expect(apiResponse.message).toBe("");
+        expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
+        expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
+        expect(apiResponse.version).toBe(version);
+
+        const solution = apiResponse.solution;
+        expect(solution.url).toContain(cfUrl)
+        expect(solution.status).toBe(200);
+        expect(Object.keys(solution.headers).length).toBeGreaterThan(0)
+        expect(solution.response).toContain("<!DOCTYPE html>")
+        expect(Object.keys(solution.cookies).length).toBeGreaterThan(0)
+        expect(solution.userAgent).toContain("Firefox/")
+
+        const cfCookie: string = (solution.cookies as any[]).filter(function(cookie) {
+            return cookie.name == "cf_clearance";
+        })[0].value
+        expect(cfCookie.length).toBeGreaterThan(30)
+    });
+
+    test("Cmd 'request.get' should return fail with Cloudflare CAPTCHA", async () => {
+        const payload = {
+            "cmd": "request.get",
+            "url": cfCaptchaUrl
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSolution = response.body;
+        expect(apiResponse.status).toBe("error");
+        expect(apiResponse.message).toBe("Cloudflare Error: FlareSolverr can not resolve CAPTCHA challenges. Since the captcha doesn't always appear, you may have better luck with the next request.");
+        expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
+        expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
+        expect(apiResponse.version).toBe(version);
+        // solution is filled but not useful
+        expect(apiResponse.solution.url).toContain(cfCaptchaUrl)
+    });
+
+    test("Cmd 'request.post' should return fail with Cloudflare Blocked", async () => {
+        const payload = {
+            "cmd": "request.post",
+            "url": cfBlockedUrl,
+            "postData": "test1=test2"
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSolution = response.body;
+        expect(apiResponse.status).toBe("error");
+        expect(apiResponse.message).toBe("Cloudflare Error: Cloudflare has blocked this request. Probably your IP is banned for this site, check in your web browser.");
+        expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
+        expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
+        expect(apiResponse.version).toBe(version);
+        // solution is filled but not useful
+        expect(apiResponse.solution.url).toContain(cfBlockedUrl)
+    });
+
+    test("Cmd 'request.get' should return OK with DDoS-GUARD JS", async () => {
+        const payload = {
+            "cmd": "request.get",
+            "url": ddgUrl
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSolution = response.body;
+        expect(apiResponse.status).toBe("ok");
+        expect(apiResponse.message).toBe("");
+        expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
+        expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
+        expect(apiResponse.version).toBe(version);
+
+        const solution = apiResponse.solution;
+        expect(solution.url).toContain(ddgUrl)
+        expect(solution.status).toBe(200);
+        expect(Object.keys(solution.headers).length).toBeGreaterThan(0)
+        expect(solution.response).toContain("<!DOCTYPE html>")
+        expect(Object.keys(solution.cookies).length).toBeGreaterThan(0)
+        expect(solution.userAgent).toContain("Firefox/")
+
+        const cfCookie: string = (solution.cookies as any[]).filter(function(cookie) {
+            return cookie.name == "__ddg1_";
+        })[0].value
+        expect(cfCookie.length).toBeGreaterThan(10)
+    });
+
+    test("Cmd 'request.get' should return OK with Custom CloudFlare JS", async () => {
+        const payload = {
+            "cmd": "request.get",
+            "url": ccfUrl
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSolution = response.body;
+        expect(apiResponse.status).toBe("ok");
+        expect(apiResponse.message).toBe("");
+        expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
+        expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
+        expect(apiResponse.version).toBe(version);
+
+        const solution = apiResponse.solution;
+        expect(solution.url).toContain(ccfUrl)
+        expect(solution.status).toBe(200);
+        expect(Object.keys(solution.headers).length).toBeGreaterThan(0)
+        expect(solution.response).toContain("<html><head>")
+        expect(Object.keys(solution.cookies).length).toBeGreaterThan(0)
+        expect(solution.userAgent).toContain("Firefox/")
+
+        const cfCookie: string = (solution.cookies as any[]).filter(function(cookie) {
+            return cookie.name == "ct_anti_ddos_key";
+        })[0].value
+        expect(cfCookie.length).toBeGreaterThan(10)
+    });
+
+    test("Cmd 'request.get' should return OK with 'cookies' param", async () => {
+        const payload = {
+            "cmd": "request.get",
+            "url": googleUrl,
+            "cookies": [
+                {
+                    "name": "testcookie1",
+                    "value": "testvalue1"
+                },
+                {
+                    "name": "testcookie2",
+                    "value": "testvalue2"
+                }
+            ]
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSolution = response.body;
+        expect(apiResponse.status).toBe("ok");
+
+        const solution = apiResponse.solution;
+        expect(solution.url).toContain(googleUrl)
+        expect(Object.keys(solution.cookies).length).toBeGreaterThan(1)
+        const cookie1: string = (solution.cookies as any[]).filter(function(cookie) {
+            return cookie.name == "testcookie1";
+        })[0].value
+        expect(cookie1).toBe("testvalue1")
+        const cookie2: string = (solution.cookies as any[]).filter(function(cookie) {
+            return cookie.name == "testcookie2";
+        })[0].value
+        expect(cookie2).toBe("testvalue2")
+    });
+
+    test("Cmd 'request.get' should return OK with 'returnOnlyCookies' param", async () => {
+        const payload = {
+            "cmd": "request.get",
+            "url": googleUrl,
+            "returnOnlyCookies": true
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSolution = response.body;
+
+        const solution = apiResponse.solution;
+        expect(solution.url).toContain(googleUrl)
+        expect(solution.status).toBe(200);
+        expect(solution.headers).toBe(null)
+        expect(solution.response).toBe(null)
+        expect(Object.keys(solution.cookies).length).toBeGreaterThan(0)
+        expect(solution.userAgent).toBe(null)
+    });
+
+    test("Cmd 'request.get' should return OK with HTTP 'proxy' param", async () => {
+        /*
+        To configure TinyProxy in local:
+           * sudo vim /etc/tinyproxy/tinyproxy.conf
+              * edit => LogFile "/tmp/tinyproxy.log"
+              * edit => Syslog Off
+           * sudo tinyproxy -d
+           * sudo tail -f /tmp/tinyproxy.log
+        */
+        const payload = {
+            "cmd": "request.get",
+            "url": googleUrl,
+            "proxy": {
+                "url": proxyUrl
+            }
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSolution = response.body;
+        expect(apiResponse.status).toBe("ok");
+
+        const solution = apiResponse.solution;
+        expect(solution.url).toContain(googleUrl)
+        expect(solution.status).toBe(200);
+    });
+
+    // todo: credentials are not working
+    test.skip("Cmd 'request.get' should return OK with HTTP 'proxy' param with credentials", async () => {
+        /*
+        To configure TinyProxy in local:
+           * sudo vim /etc/tinyproxy/tinyproxy.conf
+              * edit => LogFile "/tmp/tinyproxy.log"
+              * edit => Syslog Off
+              * add => BasicAuth testuser testpass
+           * sudo tinyproxy -d
+           * sudo tail -f /tmp/tinyproxy.log
+        */
+        const payload = {
+            "cmd": "request.get",
+            "url": googleUrl,
+            "proxy": {
+                "url": proxyUrl,
+                "username": "testuser",
+                "password": "testpass"
+            }
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSolution = response.body;
+        expect(apiResponse.status).toBe("ok");
+
+        const solution = apiResponse.solution;
+        expect(solution.url).toContain(googleUrl)
+        expect(solution.status).toContain(200)
+    });
+
+    test("Cmd 'request.get' should return OK with SOCKSv5 'proxy' param", async () => {
+        /*
+        To configure Dante in local:
+           * https://linuxhint.com/set-up-a-socks5-proxy-on-ubuntu-with-dante/
+           * sudo vim /etc/sockd.conf
+           * sudo systemctl restart sockd.service
+           * curl --socks5 socks5://127.0.0.1:1080 https://www.google.com
+        */
+        const payload = {
+            "cmd": "request.get",
+            "url": googleUrl,
+            "proxy": {
+                "url": proxySocksUrl
+            }
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSolution = response.body;
+        expect(apiResponse.status).toBe("ok");
+
+        const solution = apiResponse.solution;
+        expect(solution.url).toContain(googleUrl)
+        expect(solution.status).toBe(200);
+    });
+
+    test("Cmd 'request.get' should fail with wrong 'proxy' param", async () => {
+        const payload = {
+            "cmd": "request.get",
+            "url": googleUrl,
+            "proxy": {
+                "url": "http://127.0.0.1:43210"
+            }
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(500);
+
+        const apiResponse: V1ResponseSolution = response.body;
+        expect(apiResponse.status).toBe("error");
+        expect(apiResponse.message).toBe("Error: Unable to process browser request. Error: NS_ERROR_PROXY_CONNECTION_REFUSED at https://www.google.com");
+    });
+
+    test("Cmd 'request.get' should return fail with timeout", async () => {
+        const payload = {
+            "cmd": "request.get",
+            "url": googleUrl,
+            "maxTimeout": 10
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(500);
+
+        const apiResponse: V1ResponseBase = response.body;
+        expect(apiResponse.status).toBe("error");
+        expect(apiResponse.message).toBe("Error: Unable to process browser request. Error: Maximum timeout reached. maxTimeout=10 (ms)");
+        expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
+        expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
+        expect(apiResponse.version).toBe(version);
+    });
+
+    test("Cmd 'request.get' should return fail with bad domain", async () => {
+        const payload = {
+            "cmd": "request.get",
+            "url": "https://www.google.combad"
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(500);
+
+        const apiResponse: V1ResponseBase = response.body;
+        expect(apiResponse.status).toBe("error");
+        expect(apiResponse.message).toBe("Error: Unable to process browser request. Error: NS_ERROR_UNKNOWN_HOST at https://www.google.combad");
+    });
+
+    test("Cmd 'request.get' should accept deprecated params", async () => {
+        const payload = {
+            "cmd": "request.get",
+            "url": googleUrl,
+            "userAgent": "Test User-Agent" // was removed in v2, not used
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSolution = response.body;
+        expect(apiResponse.status).toBe("ok");
+
+        const solution = apiResponse.solution;
+        expect(solution.url).toContain(googleUrl)
+        expect(solution.status).toBe(200);
+        expect(solution.userAgent).toContain("Firefox/")
+    });
+
+    test("Cmd 'request.post' should return OK with no Cloudflare", async () => {
+        const payload = {
+            "cmd": "request.post",
+            "url": postUrl + '/post',
+            "postData": "param1=value1&param2=value2"
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSolution = response.body;
+        expect(apiResponse.status).toBe("ok");
+        expect(apiResponse.message).toBe("");
+        expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
+        expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
+        expect(apiResponse.version).toBe(version);
+
+        const solution = apiResponse.solution;
+        expect(solution.url).toContain(postUrl)
+        expect(solution.status).toBe(200);
+        expect(Object.keys(solution.headers).length).toBeGreaterThan(0)
+        expect(solution.response).toContain(" I hope you have a lovely day!")
+        expect(Object.keys(solution.cookies).length).toBe(0)
+        expect(solution.userAgent).toContain("Firefox/")
+
+        // check that we sent the date
+        const payload2 = {
+            "cmd": "request.get",
+            "url": postUrl
+        }
+        const response2: Response = await request(app).post("/v1").send(payload2);
+        expect(response2.statusCode).toBe(200);
+
+        const apiResponse2: V1ResponseSolution = response2.body;
+        expect(apiResponse2.status).toBe("ok");
+
+        const solution2 = apiResponse2.solution;
+        expect(solution2.status).toBe(200);
+        expect(solution2.response).toContain(new Date().toISOString().split(':')[0].replace('T', ' '))
+    });
+
+    test("Cmd 'request.post' should fail without 'postData' param", async () => {
+        const payload = {
+            "cmd": "request.post",
+            "url": googleUrl
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(500);
+
+        const apiResponse: V1ResponseBase = response.body;
+        expect(apiResponse.status).toBe("error");
+        expect(apiResponse.message).toBe("Error: Must send param \"postBody\" when sending a POST request.");
+        expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
+        expect(apiResponse.endTimestamp).toBeGreaterThanOrEqual(apiResponse.startTimestamp);
+        expect(apiResponse.version).toBe(version);
+    });
+
+    test("Cmd 'sessions.create' should return OK", async () => {
+        const payload = {
+            "cmd": "sessions.create"
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSession = response.body;
+        expect(apiResponse.status).toBe("ok");
+        expect(apiResponse.message).toBe("Session created successfully.");
+        expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
+        expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
+        expect(apiResponse.version).toBe(version);
+        expect(apiResponse.session.length).toBe(36);
+    });
+
+    test("Cmd 'sessions.create' should return OK with session", async () => {
+        const payload = {
+            "cmd": "sessions.create",
+            "session": "2bc6bb20-2f56-11ec-9543-test"
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSession = response.body;
+        expect(apiResponse.status).toBe("ok");
+        expect(apiResponse.message).toBe("Session created successfully.");
+        expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
+        expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
+        expect(apiResponse.version).toBe(version);
+        expect(apiResponse.session).toBe("2bc6bb20-2f56-11ec-9543-test");
+    });
+
+    test("Cmd 'sessions.list' should return OK", async () => {
+        // create one session for testing
+        const payload0 = {
+            "cmd": "sessions.create"
+        }
+        const response0: Response = await request(app).post("/v1").send(payload0);
+        expect(response0.statusCode).toBe(200);
+
+        const payload = {
+            "cmd": "sessions.list"
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSessions = response.body;
+        expect(apiResponse.status).toBe("ok");
+        expect(apiResponse.message).toBe("");
+        expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
+        expect(apiResponse.endTimestamp).toBeGreaterThanOrEqual(apiResponse.startTimestamp);
+        expect(apiResponse.version).toBe(version);
+        expect(apiResponse.sessions.length).toBeGreaterThan(0)
+    });
+
+    test("Cmd 'sessions.destroy' should return OK", async () => {
+        // create one session for testing
+        const payload0 = {
+            "cmd": "sessions.create"
+        }
+        const response0: Response = await request(app).post("/v1").send(payload0);
+        expect(response0.statusCode).toBe(200);
+        const apiResponse0: V1ResponseSession = response0.body;
+        const sessionId0 = apiResponse0.session
+
+        const payload = {
+            "cmd": "sessions.destroy",
+            "session": sessionId0
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseBase = response.body;
+        expect(apiResponse.status).toBe("ok");
+        expect(apiResponse.message).toBe("The session has been removed.");
+        expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
+        expect(apiResponse.endTimestamp).toBeGreaterThanOrEqual(apiResponse.startTimestamp);
+        expect(apiResponse.version).toBe(version);
+    });
+
+    test("Cmd 'sessions.destroy' should fail", async () => {
+        const payload = {
+            "cmd": "sessions.destroy",
+            "session": "bad-session"
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(500);
+
+        const apiResponse: V1ResponseBase = response.body;
+        expect(apiResponse.status).toBe("error");
+        expect(apiResponse.message).toBe("Error: This session does not exist.");
+        expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
+        expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
+        expect(apiResponse.version).toBe(version);
+    });
+
+    test("Cmd 'request.get' should use session", async () => {
+        // create one session for testing
+        const payload0 = {
+            "cmd": "sessions.create"
+        }
+        const response0: Response = await request(app).post("/v1").send(payload0);
+        expect(response0.statusCode).toBe(200);
+        const apiResponse0: V1ResponseSession = response0.body;
+        const sessionId0 = apiResponse0.session
+
+        // first request should solve the challenge
+        const payload = {
+            "cmd": "request.get",
+            "url": cfUrl,
+            "session": sessionId0
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSolution = response.body;
+        expect(apiResponse.status).toBe("ok");
+        const cfCookie: string = (apiResponse.solution.cookies as any[]).filter(function(cookie) {
+            return cookie.name == "cf_clearance";
+        })[0].value
+        expect(cfCookie.length).toBeGreaterThan(30)
+
+        // second request should have the same cookie
+        const response2: Response = await request(app).post("/v1").send(payload);
+        expect(response2.statusCode).toBe(200);
+
+        const apiResponse2: V1ResponseSolution = response2.body;
+        expect(apiResponse2.status).toBe("ok");
+        const cfCookie2: string = (apiResponse2.solution.cookies as any[]).filter(function(cookie) {
+            return cookie.name == "cf_clearance";
+        })[0].value
+        expect(cfCookie2.length).toBeGreaterThan(30)
+        expect(cfCookie2).toBe(cfCookie)
+    });
+
+});

src/types.ts

@@ -1,9 +0,0 @@
-import { IncomingMessage, ServerResponse } from 'http';
-
-export interface RequestContext {
-  req: IncomingMessage
-  res: ServerResponse
-  startTimestamp: number
-  errorResponse: (msg: string) => void,
-  successResponse: (msg: string, extendedProperties?: object) => void
-}

src/utils.ts

@@ -1,31 +0,0 @@
-import * as fs from 'fs'
-import * as Path from 'path'
-import { promisify } from 'util'
-
-export const sleep = promisify(setTimeout)
-
-// recursive fs.rmdir needs node version 12:
-// https://github.com/ngosang/FlareSolverr/issues/5#issuecomment-655572712
-export function deleteFolderRecursive(path: string) {
-  if (fs.existsSync(path)) {
-    fs.readdirSync(path).forEach((file) => {
-      const curPath = Path.join(path, file)
-      if (fs.lstatSync(curPath).isDirectory()) { // recurse
-        deleteFolderRecursive(curPath)
-      } else { // delete file
-        fs.unlinkSync(curPath)
-      }
-    })
-    fs.rmdirSync(path)
-  }
-}
-
-export const removeEmptyFields = (o: Record<string, any>): typeof o => {
-  const r: typeof o = {}
-  for (const k in o) {
-    if (o[k] !== undefined) {
-      r[k] = o[k]
-    }
-  }
-  return r
-}