Bump version 2.2.7

credit: @realivanjx
https://github.com/FlareSolverr/FlareSolverr/issues/481#issuecomment-1242951929

.dockerignore

@@ -1,6 +1,7 @@
-node_modules
+.git/
-npm-debug.log
+.github/
-Dockerfile
+.idea/
-.dockerignore
+bin/
-.git
+dist/
-.gitignore
+node_modules/
+resources/

.github/ISSUE_TEMPLATE.md

@@ -1,31 +1,26 @@
 **Please use the search bar** at the top of the page and make sure you are not creating an already submitted issue.
 Check closed issues as well, because your issue may have already been fixed.
 
-### Instruction on how to enable debug and html trace
+### How to enable debug and html traces
 
 [Follow the instructions from this wiki page](https://github.com/FlareSolverr/FlareSolverr/wiki/How-to-enable-debug-and-html-trace)
 
 ### Environment
 
-**FlareSolverr Version**:
+* **FlareSolverr version**:
-
+* **Last working FlareSolverr version**:
-**Docker**: [yes/no]
+* **Operating system**:
-
+* **Are you using Docker**: [yes/no]
-**OS**: 
+* **FlareSolverr User-Agent (see log traces or / endpoint)**:
-
+* **Are you using a proxy or VPN?** [yes/no]
-**Last Working FlareSolverr Version**:
+* **Are you using Captcha Solver:** [yes/no]
-
+* **If using captcha solver, which one:**
-**Are you using a proxy or VPN?** [yes/no]
+* **URL to test this issue:**
-
-**Using Captcha Solver:** [yes/no]
-
-**If using captcha solver, which one:**
 
 ### Description
 
 [List steps to reproduce the error and details on what happens and what you expected to happen]
 
-
 ### Logged Error Messages
 
 [Place any relevant error messages you noticed from the logs here.]

.github/workflows/release-docker.yml

@@ -24,7 +24,7 @@ jobs:
           tag-sha: false
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v1.0.1
       -
         name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1

.github/workflows/release.yml

@@ -18,7 +18,7 @@ jobs:
       - name: Setup Node
         uses: actions/setup-node@v2
         with:
-          node-version: '14'
+          node-version: '16'
 
       - name: Build artifacts
         run: |

Dockerfile

@@ -1,30 +1,29 @@
-FROM --platform=${TARGETPLATFORM:-linux/amd64} node:15.2.1-alpine3.11
+FROM node:16-alpine3.15
 
-# Print build information
+# Install the web browser (package firefox-esr is available too)
-ARG TARGETPLATFORM
+RUN apk update && \
-ARG BUILDPLATFORM
+    apk add --no-cache firefox dumb-init && \
-RUN printf "I am running on ${BUILDPLATFORM:-linux/amd64}, building for ${TARGETPLATFORM:-linux/amd64}\n$(uname -a)\n"
+    rm -Rf /var/cache
-
-# Install Chromium, dumb-init and remove all locales but en-US
-RUN apk add --no-cache chromium dumb-init && \
-    find /usr/lib/chromium/locales -type f ! -name 'en-US.*' -delete
 
 # Copy FlareSolverr code
 USER node
 RUN mkdir -p /home/node/flaresolverr
 WORKDIR /home/node/flaresolverr
-COPY --chown=node:node package.json package-lock.json tsconfig.json ./
+COPY --chown=node:node package.json package-lock.json tsconfig.json install.js ./
 COPY --chown=node:node src ./src/
 
-# Install package. Skip installing Chrome, we will use the installed package.
+# Install package. Skip installing the browser, we will use the installed package.
-ENV PUPPETEER_PRODUCT=chrome \
+ENV PUPPETEER_PRODUCT=firefox \
     PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
-    PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
+    PUPPETEER_EXECUTABLE_PATH=/usr/bin/firefox
 RUN npm install && \
     npm run build && \
-    rm -rf src tsconfig.json && \
+    npm prune --production && \
-    npm prune --production
+    rm -rf /home/node/.npm
 
 EXPOSE 8191
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]
-CMD ["npm", "start"]
+CMD ["node", "./dist/server.js"]
+
+# docker build -t flaresolverr:custom .
+# docker run -p 8191:8191 -e LOG_LEVEL=debug flaresolverr:custom

README.md

@@ -4,20 +4,18 @@
 [![Docker Pulls](https://img.shields.io/docker/pulls/flaresolverr/flaresolverr)](https://hub.docker.com/r/flaresolverr/flaresolverr/)
 [![GitHub issues](https://img.shields.io/github/issues/FlareSolverr/FlareSolverr)](https://github.com/FlareSolverr/FlareSolverr/issues)
 [![GitHub pull requests](https://img.shields.io/github/issues-pr/FlareSolverr/FlareSolverr)](https://github.com/FlareSolverr/FlareSolverr/pulls)
-[![Donate PayPal](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=X5NJLLX5GLTV6&source=url)
+[![Donate PayPal](https://img.shields.io/badge/Donate-PayPal-yellow.svg)](https://www.paypal.com/paypalme/diegoheras0xff)
-[![Donate Buy Me A Coffee](https://img.shields.io/badge/Donate-Buy%20me%20a%20coffee-yellow.svg)](https://www.buymeacoffee.com/ngosang)
+[![Donate Bitcoin](https://img.shields.io/badge/Donate-Bitcoin-f7931a.svg)](https://www.blockchain.com/btc/address/13Hcv77AdnFWEUZ9qUpoPBttQsUT7q9TTh)
-[![Donate Bitcoin](https://img.shields.io/badge/Donate-Bitcoin-orange.svg)](https://en.cryptobadges.io/donate/13Hcv77AdnFWEUZ9qUpoPBttQsUT7q9TTh)
+[![Donate Ethereum](https://img.shields.io/badge/Donate-Ethereum-8c8c8c.svg)](https://www.blockchain.com/eth/address/0x0D1549BbB00926BF3D92c1A8A58695e982f1BE2E)
 
-FlareSolverr is a proxy server to bypass Cloudflare protection
+FlareSolverr is a proxy server to bypass Cloudflare and DDoS-GUARD protection.
-
-:warning: This project is in beta state. Some things may not work and the API can change at any time.
 
 ## How it works
 
-FlareSolverr starts a proxy server and it waits for user requests in an idle state using few resources.
+FlareSolverr starts a proxy server, and it waits for user requests in an idle state using few resources.
 When some request arrives, it uses [puppeteer](https://github.com/puppeteer/puppeteer) with the
 [stealth plugin](https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin-stealth)
-to create a headless browser (Chrome). It opens the URL with user parameters and waits until the Cloudflare challenge
+to create a headless browser (Firefox). It opens the URL with user parameters and waits until the Cloudflare challenge
 is solved (or timeout). The HTML code and the cookies are sent back to the user, and those cookies can be used to
 bypass Cloudflare using other HTTP clients.
 
@@ -58,21 +56,31 @@ docker run -d \
   ghcr.io/flaresolverr/flaresolverr:latest
 ```
 
+If your host OS is Debian, make sure `libseccomp2` version is 2.5.x. You can check the version with `sudo apt-cache policy libseccomp2` 
+and update the package with `sudo apt install libseccomp2=2.5.1-1~bpo10+1` or `sudo apt install libseccomp2=2.5.1-1+deb11u1`.
+Remember to restart the Docker daemon and the container after the update.
+
 ### Precompiled binaries
 
 This is the recommended way for Windows users.
 * Download the [FlareSolverr zip](https://github.com/FlareSolverr/FlareSolverr/releases) from the release's assets. It is available for Windows and Linux.
-* Extract the zip file. FlareSolverr executable and chrome folder must be in the same directory.
+* Extract the zip file. FlareSolverr executable and firefox folder must be in the same directory.
 * Execute FlareSolverr binary. In the environment variables section you can find how to change the configuration.
 
 ### From source code
 
-This is the recommended way for MacOS users and for developers.
+This is the recommended way for macOS users and for developers.
-* Install [NodeJS](https://nodejs.org/)
+* Install [NodeJS](https://nodejs.org/) 16.
-* Clone this repository and open a shell in that path
+* Clone this repository and open a shell in that path.
-* Run `npm install` command to install FlareSolverr dependencies
+* Run `export PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true` (Linux/macOS) or `set PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true` (Windows).
-* Run `npm run build` command to compile TypeScript code
+* Run `npm install` command to install FlareSolverr dependencies.
-* Run `npm start` command to start FlareSolverr
+* Run `npm start` command to compile TypeScript code and start FlareSolverr.
+
+If you get errors related to firefox not installed try running `node install.js` to install Firefox.
+
+### Systemd service
+
+We provide an example Systemd unit file `flaresolverr.service` as reference. You have to modify the file to suit your needs: paths, user and environment variables.
 
 ## Usage
 
@@ -83,11 +91,7 @@ curl -L -X POST 'http://localhost:8191/v1' \
 --data-raw '{
   "cmd": "request.get",
   "url":"http://www.google.com/",
-  "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleW...",
+  "maxTimeout": 60000
-  "maxTimeout": 60000,
-  "headers": {
-    "X-Test": "Testing 123..."
-  }
 }'
 ```
 
@@ -104,7 +108,7 @@ This also speeds up the requests since it won't have to launch a new browser ins
 Parameter | Notes
 |--|--|
 session | Optional. The session ID that you want to be assigned to the instance. If isn't set a random UUID will be assigned.
-userAgent | Optional. Will be used by the headless browser.
+proxy | Optional, default disabled. Eg: `"proxy": {"url": "http://127.0.0.1:8888"}`. You must include the proxy schema in the URL: `http://`, `socks4://` or `socks5://`. Authorization (username/password) is not supported.
 
 #### + `sessions.list`
 
@@ -139,10 +143,12 @@ Parameter | Notes
 |--|--|
 url | Mandatory
 session | Optional. Will send the request from and existing browser instance. If one is not sent it will create a temporary instance that will be destroyed immediately after the request is completed.
-headers | Optional. To specify user headers.
 maxTimeout | Optional, default value 60000. Max timeout to solve the challenge in milliseconds.
 cookies | Optional. Will be used by the headless browser. Follow [this](https://github.com/puppeteer/puppeteer/blob/v3.3.0/docs/api.md#pagesetcookiecookies) format.
 returnOnlyCookies | Optional, default false. Only returns the cookies. Response data, headers and other parts of the response are removed.
+proxy | Optional, default disabled. Eg: `"proxy": {"url": "http://127.0.0.1:8888"}`. You must include the proxy schema in the URL: `http://`, `socks4://` or `socks5://`. Authorization (username/password) is not supported. (When the `session` parameter is set, the proxy is ignored; a session specific proxy can be set in `sessions.create`.)
+
+:warning: If you want to use Cloudflare clearance cookie in your scripts, make sure you use the FlareSolverr User-Agent too. If they don't match you will see the challenge.
 
 Example response from running the `curl` above:
 
@@ -209,27 +215,21 @@ This is the same as `request.get` but it takes one more param:
 
 Parameter | Notes
 |--|--|
-postData | Must be a string. If you want to POST a form, don't forget to set the `Content-Type` header to `application/x-www-form-urlencoded` or the server might not understand your request.
+postData | Must be a string with `application/x-www-form-urlencoded`. Eg: `a=b&c=d`
-
-### Download small files
-
-If you need to access an image/pdf or small file, you should pass the `download` parameter to `request.get` setting it
-to `true`. Rather than access the html and return text it will return the buffer **base64** encoded which you will be
-able to decode and save the image/pdf.
-
-This method isn't recommended for videos or anything larger. As that should be streamed back to the client and at the
-moment there is nothing setup to do so. If this is something you need feel free to create an issue and/or submit a PR.
 
 ## Environment variables
 
 Name | Default | Notes
 |--|--|--|
-LOG_LEVEL | info | Used to change the verbosity of the logging. Use `LOG_LEVEL=debug` for more information.
+LOG_LEVEL | info | Verbosity of the logging. Use `LOG_LEVEL=debug` for more information.
-LOG_HTML | false | Used for debugging. If `true` all HTML that passes through the proxy will be logged to the console in `debug` level.
+LOG_HTML | false | Only for debugging. If `true` all HTML that passes through the proxy will be logged to the console in `debug` level.
-PORT | 8191 | Change this if you already have a process running on port `8191`.
+CAPTCHA_SOLVER | none | Captcha solving method. It is used when a captcha is encountered. See the Captcha Solvers section.
-HOST | 0.0.0.0 | This shouldn't need to be messed with but if you insist, it's here!
+TZ | UTC | Timezone used in the logs and the web browser. Example: `TZ=Europe/London`.
-CAPTCHA_SOLVER | none | This is used to select which captcha solving method it used when a captcha is encountered.
+HEADLESS | true | Only for debugging. To run the web browser in headless mode or visible.
-HEADLESS | true | This is used to debug the browser by not running it in headless mode.
+BROWSER_TIMEOUT | 40000 | If you are experiencing errors/timeouts because your system is slow, you can try to increase this value. Remember to increase the `maxTimeout` parameter too.
+TEST_URL | https://www.google.com | FlareSolverr makes a request on start to make sure the web browser is working. You can change that URL if it is blocked in your country.
+PORT | 8191 | Listening port. You don't need to change this if you are running on Docker.
+HOST | 0.0.0.0 | Listening interface. You don't need to change this if you are running on Docker.
 
 Environment variables are set differently depending on the operating system. Some examples:
 * Docker: Take a look at the Docker section in this document. Environment variables can be set in the `docker-compose.yml` file or in the Docker CLI command.
@@ -247,37 +247,6 @@ If this is the case, FlareSolverr will return the error `Captcha detected but no
 FlareSolverr can be customized to solve the captchas automatically by setting the environment variable `CAPTCHA_SOLVER`
 to the file name of one of the adapters inside the [/captcha](src/captcha) directory.
 
-### hcaptcha-solver
-
-This method makes use of the [hcaptcha-solver](https://github.com/JimmyLaurent/hcaptcha-solver) project.
-
-NOTE: This solver works picking random images so it will fail in a lot of requests and it's hard to know if it is
-working or not. In a real use case with Sonarr/Radarr + Jackett it is still useful because those apps make a new request
-each 15 minutes. Eventually one of the requests is going to work and Jackett saves the cookie forever (until it stops
-working).
-
-To use this solver you must set the environment variable:
-
-```bash
-CAPTCHA_SOLVER=hcaptcha-solver
-```
-
-### CaptchaHarvester
-
-This method makes use of the [CaptchaHarvester](https://github.com/NoahCardoza/CaptchaHarvester) project which allows
-users to collect their own tokens from ReCaptcha V2/V3 and hCaptcha for free.
-
-To use this method you must set these environment variables:
-
-```bash
-CAPTCHA_SOLVER=harvester
-HARVESTER_ENDPOINT=https://127.0.0.1:5000/token
-```
-
-**Note**: above I set `HARVESTER_ENDPOINT` to the default configuration of the captcha harvester's server, but that
-could change if you customize the command line flags. Simply put, `HARVESTER_ENDPOINT` should be set to the URI of the
-route that returns a token in plain text when called.
-
 ## Related projects
 
 * C# implementation => https://github.com/FlareSolverr/FlareSolverrSharp

build-binaries.js

@@ -1,7 +1,8 @@
 const fs = require('fs')
 const path = require('path')
-const {execSync} = require('child_process')
+const { execSync } = require('child_process')
 const archiver = require('archiver')
+const https = require('https')
 const puppeteer = require('puppeteer')
 const version = 'v' + require('./package.json').version;
 
@@ -9,50 +10,57 @@ const version = 'v' + require('./package.json').version;
   const builds = [
     {
       platform: 'linux',
-      version: 756035,
+      firefoxFolder: 'firefox',
-      chromeFolder: 'chrome-linux',
       fsExec: 'flaresolverr-linux',
       fsZipExec: 'flaresolverr',
-      fsZipName: 'linux-x64'
+      fsZipName: 'linux-x64',
+      fsLicenseName: 'LICENSE'
     },
     {
       platform: 'win64',
-      version: 756035,
+      firefoxFolder: 'firefox',
-      chromeFolder: 'chrome-win',
       fsExec: 'flaresolverr-win.exe',
       fsZipExec: 'flaresolverr.exe',
-      fsZipName: 'windows-x64'
+      fsZipName: 'windows-x64',
+      fsLicenseName: 'LICENSE.txt'
     }
-    // TODO: this is working but changes are required in session.ts to find chrome path
+    // todo: this has to be build in macOS (hdiutil is required). changes required in sessions.ts too
     // {
     //   platform: 'mac',
-    //   version: 756035,
+    //   firefoxFolder: 'firefox',
-    //   chromeFolder: 'chrome-mac',
     //   fsExec: 'flaresolverr-macos',
     //   fsZipExec: 'flaresolverr',
-    //   fsZipName: 'macos'
+    //   fsZipName: 'macos',
+    //   fsLicenseName: 'LICENSE'
     // }
   ]
 
   // generate executables
   console.log('Generating executables...')
   if (fs.existsSync('bin')) {
-    fs.rmdirSync('bin', {recursive: true})
+    fs.rmSync('bin', { recursive: true })
   }
-  execSync('pkg -t node14-win-x64,node14-linux-x64 --out-path bin .')
+  execSync('./node_modules/.bin/pkg -t node16-win-x64,node16-linux-x64 --out-path bin .')
-  // execSync('pkg -t node14-win-x64,node14-mac-x64,node14-linux-x64 --out-path bin .')
+  // execSync('./node_modules/.bin/pkg -t node16-win-x64,node16-mac-x64,node16-linux-x64 --out-path bin .')
 
-  // download Chrome and zip together
+  // Puppeteer does not allow to download Firefox revisions, just the last Nightly
+  // We this script we can download any version
+  const revision = '94.0a1';
+  const downloadHost = 'https://archive.mozilla.org/pub/firefox/nightly/2021/10/2021-10-01-09-33-23-mozilla-central';
+
+  // download firefox and zip together
   for (const os of builds) {
     console.log('Building ' + os.fsZipName + ' artifact')
 
-    // download chrome
+    // download firefox
-    console.log('Downloading Chrome...')
+    console.log(`Downloading firefox ${revision} for ${os.platform} ...`)
     const f = puppeteer.createBrowserFetcher({
+      product: 'firefox',
       platform: os.platform,
+      host: downloadHost,
       path: path.join(__dirname, 'bin', 'puppeteer')
     })
-    await f.download(os.version)
+    await f.download(revision)
 
     // compress in zip
     console.log('Compressing zip file...')
@@ -70,9 +78,13 @@ const version = 'v' + require('./package.json').version;
 
     archive.pipe(output)
 
+    archive.file('LICENSE', { name: 'flaresolverr/' + os.fsLicenseName })
     archive.file('bin/' + os.fsExec, { name: 'flaresolverr/' + os.fsZipExec })
-    archive.directory('bin/puppeteer/' + os.platform + '-' + os.version + '/' + os.chromeFolder, 'flaresolverr/chrome')
+    archive.directory('bin/puppeteer/' + os.platform + '-' + revision + '/' + os.firefoxFolder, 'flaresolverr/firefox')
+    if (os.platform === 'linux') {
+      archive.file('flaresolverr.service', { name: 'flaresolverr/flaresolverr.service' })
+    }
 
-    archive.finalize()
+    await archive.finalize()
   }
 })()

docker-compose.yml

@@ -9,6 +9,7 @@ services:
       - LOG_LEVEL=${LOG_LEVEL:-info}
       - LOG_HTML=${LOG_HTML:-false}
       - CAPTCHA_SOLVER=${CAPTCHA_SOLVER:-none}
+      - TZ=Europe/London
     ports:
       - "${PORT:-8191}:8191"
     restart: unless-stopped

flaresolverr.service

@@ -0,0 +1,19 @@
+[Unit]
+Description=FlareSolverr
+After=network.target
+
+[Service]
+SyslogIdentifier=flaresolverr
+Restart=always
+RestartSec=5
+Type=simple
+User=flaresolverr
+Group=flaresolverr
+Environment="LOG_LEVEL=info"
+Environment="CAPTCHA_SOLVER=none"
+WorkingDirectory=/opt/flaresolverr
+ExecStart=/opt/flaresolverr/flaresolverr
+TimeoutStopSec=30
+
+[Install]
+WantedBy=multi-user.target

html_samples/cloudflare_captcha_hcaptcha_v1.html

@@ -0,0 +1,219 @@
+<!DOCTYPE html>
+<html lang="en-US">
+
+<head>
+    <title>Just a moment...</title>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=Edge">
+    <meta name="robots" content="noindex,nofollow">
+    <meta name="viewport" content="width=device-width,initial-scale=1">
+    <link href="Just%20a%20moment_files/cf-errors.css" rel="stylesheet">
+
+    <script>
+        (function () {
+            window._cf_chl_opt = {
+                cvId: '2',
+                cType: 'managed',
+                cNounce: '67839',
+                cRay: '732fbc436ab471ed',
+                cHash: 'dce5bd920f3aa51',
+                cUPMDTk: "\/search?q=2022&__cf_chl_tk=lkycIb1jDXlmFqiB7AXTwy38_EzYPvu79CCQyU9lhUE-1659201316-0-gaNycGzNCf0",
+                cFPWv: 'g',
+                cTTimeMs: '1000',
+                cTplV: 2,
+                cRq: {
+                    ru: 'aHR0cHM6Ly8wbWFnbmV0LmNvbS9zZWFyY2g/cT0yMDIy',
+                    ra: 'TW96aWxsYS81LjAgKFgxMTsgTGludXggeDg2XzY0OyBydjoxMDUuMCkgR2Vja28vMjAxMDAxMDEgRmlyZWZveC8xMDUuMA==',
+                    rm: 'R0VU',
+                    d: 'MqxNbbGfWazPaVMZ7GQRz02TV/pSUL9POWx0y4e7HFRwP1RTAxLc1RZRuHg+N/bGMuPj08kSx0UpcjEjMkSOqiU6I/64IDYbCJvey5rY07fkkljpZaYGTDZIoWdOWlgP3ky15ybZ42xMK4tfI1yJ+iFZCVgR6VBjJzi5I56j9Ijog2AvsoQW2TrguGpgKaT1LkhxWNElzBbvXWt1uyRgE19UQ9J/5vtxEwoh5wodHh7WE297n8uI1hpDgge2bDYQvwe+RDq3QAyhQOmymg+IIlt1y115v9R8k5ehT9TFY3vYvYnoJu9cOyHYprf9Z0jTNGxSTvLHYJbfq30Samu5fKfE0oZREZizvPUgUsJm2rRKkCY9VCdBkpO8vaUgIwIYkeWavtqdudjb3zEDBCD4cAH/xv3Bl1VRy2Qf7XlcbpElCOq06TDTQ1uGjyCqbVbvjesrOy0Dp2nXTjdfbkWvnN7mWpFlPUD7/41MUo9lc6V1Aj1Kjg6AKfVV4DUHpq6ZVnMHzrcPQLy4qD7CptcMpQKArZtJCRsUpgq8GWKJcU4dU8ZmyROAA+l+JEVnGbh2bsRdif4azh57OdjZfEKSa5c+AL3i66vyWAZCw9Wl6CAQdFTA+ixkbl8zKbCm8ulv',
+                    t: 'MTY1OTIwMTMxNi4zOTIwMDA=',
+                    m: '3l81qRkXiMTbjTzBtc0v1XwSheF46UfagbXVhYgbAVw=',
+                    i1: 'Iu5a1gH3p9igzqBwncow9g==',
+                    i2: 'PmNXozjc73unhnp/X0+kUQ==',
+                    zh: 'qP4bnGc6j96JlnjNSE7HmQci3S9L50bHFtm4bQRjjKU=',
+                    uh: 'SK3PXNkeRzZtkRARhJpbmZpCIiWQw6+5gpOE7vojWx4=',
+                    hh: 'azXzJl8Ou22g0nN/9idVUoB9EqZ7fLmkSdDRHM3Lkmw=',
+                }
+            }
+            window._cf_chl_enter = function () { window._cf_chl_opt.p = 1 };
+        })();
+    </script>
+
+    <script src="Just%20a%20moment_files/v1.js"></script>
+    <script type="text/javascript" src="Just%20a%20moment_files/api.js"></script>
+</head>
+
+<body class="no-js">
+
+    <div class="privacy-pass">
+        <a rel="noopener noreferrer" href="https://addons.mozilla.org/en-US/firefox/addon/privacy-pass/"
+            target="_blank">
+            Privacy Pass
+            <span class="privacy-pass-icon-wrapper">
+                <div class="privacy-pass-icon"></div>
+            </span>
+        </a>
+    </div>
+
+    <div class="main-wrapper" role="main">
+        <div class="main-content">
+            <h1 class="zone-name-title h1">
+                <img class="heading-favicon" src="Just%20a%20moment_files/favicon.ico"
+                    onerror="this.onerror=null;this.parentNode.removeChild(this)">
+                0MAGNET.COM
+            </h1>
+            <h2 class="h2" id="cf-challenge-running">
+                Checking if the site connection is secure
+            </h2>
+            <div id="cf-challenge-stage" style="display: block;">
+                <div id="cf-challenge-hcaptcha-wrapper" class="captcha-prompt spacer">
+                    <div style="display: none;" class="hcaptcha-box"><iframe src="Just%20a%20moment_files/hcaptcha.html"
+                            title="widget containing checkbox for hCaptcha security challenge" tabindex="0"
+                            scrolling="no" data-hcaptcha-widget-id="0tiueg8lyuj" data-hcaptcha-response=""
+                            style="width: 303px; height: 78px; overflow: hidden;" frameborder="0"></iframe><textarea
+                            id="h-captcha-response-0tiueg8lyuj" name="h-captcha-response"
+                            style="display: none;"></textarea></div>
+                    <div class="hcaptcha-box"><iframe src="Just%20a%20moment_files/hcaptcha_002.html"
+                            title="widget containing checkbox for hCaptcha security challenge" tabindex="0"
+                            scrolling="no" data-hcaptcha-widget-id="10tlmhzz0qyq" data-hcaptcha-response=""
+                            style="width: 303px; height: 78px; overflow: hidden;" frameborder="0"></iframe><textarea
+                            id="h-captcha-response-10tlmhzz0qyq" name="h-captcha-response"
+                            style="display: none;"></textarea></div>
+                </div>
+            </div>
+            <div id="cf-challenge-spinner" class="spacer loading-spinner" style="display: none; visibility: hidden;">
+                <div class="lds-ring">
+                    <div></div>
+                    <div></div>
+                    <div></div>
+                    <div></div>
+                </div>
+            </div>
+            <noscript>
+                <div id="cf-challenge-error-title">
+                    <div class="h2">
+                        <span class="icon-wrapper">
+                            <div class="heading-icon warning-icon"></div>
+                        </span>
+                        <span id="cf-challenge-error-text">
+                            Enable JavaScript and cookies to continue
+                        </span>
+                    </div>
+                </div>
+            </noscript>
+            <div
+                style="display:none;background-image:url('/cdn-cgi/images/trace/captcha/nojs/transparent.gif?ray=732fbc436ab471ed')">
+            </div>
+            <div id="cf-challenge-body-text" class="core-msg spacer">
+                0magnet.com needs to review the security of your connection before
+                proceeding.
+            </div>
+            <div id="cf-challenge-fact-wrapper" style="display: block; visibility: visible;" class="fact spacer hidden">
+                <span class="fact-title">Did you know</span> <span id="cf-challenge-fact" class="body-text">the first
+                    botnet in 2003 took over 500-1000 devices? Today, botnets take over millions of devices at
+                    once.</span>
+            </div>
+            <div id="cf-challenge-explainer-expandable" class="hidden expandable body-text spacer"
+                style="display: block; visibility: visible;">
+                <div class="expandable-title" id="cf-challenge-explainer-summary"><button class="expandable-summary-btn"
+                        id="cf-challenge-explainer-btn" type="button"> Why am I seeing this page? <span
+                            class="caret-icon-wrapper">
+                            <div class="caret-icon"></div>
+                        </span> </button> </div>
+                <div class="expandable-details" id="cf-challenge-explainer-details">
+                    Requests from malicious bots can pose as legitimate traffic.
+                    Occasionally, you may see this page while the site ensures that the
+                    connection is secure.</div>
+            </div>
+            <div id="cf-challenge-success" style="display: none;">
+                <div class="h2"><span class="icon-wrapper"><img class="heading-icon" alt="Success icon"
+                            src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADQAAAA0CAMAAADypuvZAAAANlBMVEUAAAAxMTEwMDAxMTExMTEwMDAwMDAwMDAxMTExMTExMTEwMDAwMDAxMTExMTEwMDAwMDAxMTHB9N+uAAAAEXRSTlMA3zDvfyBAEJC/n3BQz69gX7VMkcMAAAGySURBVEjHnZZbFoMgDEQJiDzVuv/NtgbtFGuQ4/zUKpeMIQbUhXSKE5l1XSn4pFWHRm/WShT1HRLWC01LGxFEVkCc30eYkLJ1Sjk9pvkw690VY6k8DWP9OM9yMG0Koi+mi8XA36NXmW0UXra4eJ3iwHfrfXVlgL0NqqGBHdqfeQhMmyJ48WDuKP81h3+SMPeRKkJcSXiLUK4XTHCjESOnz1VUXQoc6lgi2x4cI5aTQ201Mt8wHysI5fc05M5c81uZEtHcMKhxZ7iYEty1GfhLvGKpm+EYkdGxm1F5axmcB93DoORIbXfdN7f+hlFuyxtDP+sxtBnF43cIYwaZAWRgzxIoiXEMESoPlMhwLRDXeK772CAzXEdBRV7cmnoVBp0OSlyGidEzJTFq5hhcsA5388oSGM6b5p+qjpZrBlMS9xj4AwXmz108ukU1IomM3ceiW0CDwHCqp1NjAqXlFrbga+xuloQJ+tuyfbIBPNpqnmxqT7dPaOnZqBfhSBCteJAxWj58zLk2xgg+SPGYM6dRO6WczSnIxxwEExRaO+UyCUhbOp7CGQ+kxSUfNtLQFC+Po29vvy7jj4y0yAAAAABJRU5ErkJggg=="></span>Connection
+                    is secure</div>
+                <div class="core-msg spacer">Proceeding...</div>
+            </div>
+            <form id="challenge-form"
+                action="/search?q=2022&amp;__cf_chl_f_tk=lkycIb1jDXlmFqiB7AXTwy38_EzYPvu79CCQyU9lhUE-1659201316-0-gaNycGzNCf0"
+                method="POST" enctype="application/x-www-form-urlencoded">
+                <input type="hidden" name="md"
+                    value="P4fDbSohR3e3VZmGdBSN0Gd8t8ueht.ZVgSdQYwa45Y-1659201316-0-AesEKnKN8eJLiLESJle3R0T3fwKbVMlX09CR0sIU1LruDXen0nSlT2a5OpMUFYR7HQMGcF9Ja227n2p2D2ffUlWHPVeFX-YSNiewLZA3XuAQmOn-1DyWKA-SaMH_MW2vOSC7PCHAdJDhoRWjM_o3MyKziopj3WmDcaCI_ikk68bJTIValZ_e9tO7hmHC8zjsxDC8kXmI0tbrhyW5nyS2hRlx_ZVRcRHbHsVRN0-FGtEbCoaHmnp-q0N4AYhCJXofYRunPcSG_Y1iWMk-7ofOXON_gO7oGG_8-WWD5EG1jaz2ldpNO1RTkS7dQvTiC1Io1qAsVnQtokEaDR2zoWK_MF-hz6tOmuJIDgnAoH6vPFAa9EyJOUiG2RV-3q1CKTUgr82XRJw5CaXpN0QeBq0xHxFl5mzkFO8xqQsRnPkGUKtxBQ58syPIhR4AvNp8HA028gUNmaztJZ9i2UcWydut4VghHsoJjS5DEKTamjJhNrrkargjXUekXTfKXMVKCXxo0NFObTmKwzsNB5hrk3M43KzZCOOgTnqsrVUk54bAeDsr4qmTVW2wVk-0u78QpV2JFFOIJxRLikPmqo9CUokgUJ_IPsEjA5Q3kjrf9yq2OHU0MkwzLFNOAyc5N3A4WSYp91kESwxM98qFetpAZ0R3LID2c2-MraHnpOI2Xn4bxbDIdUPmjy6VB8Huuuf6M-o3Tw">
+                <input type="hidden" name="r"
+                    value="bdZ7.nm8dGOZxq3EDOv_Kx7nKVv68q7b0RARXAlR9kQ-1659201316-0-AawyK3x4GgWasA2OtBBEp9Ea52qs8zEWwnQJxLWUnC+1jqlxaKHTIHeVQjvrTl/ccu6QA41yrSTKvazKiv6zQEiDj/6ziYkhldx+oJ7SqgMzPozzza1jofsGpCCPAzIlDicF+7sh4WKOxUJOeHgHCgfEZF/MPNsaahvbQ10U8Ei9tmvj8c2tkoybya75Bj5XHPPu0S9hnOH7S24ltm9vmyHlttI7uuI962FzPCTGjuAl4R/5+06WVAzBCJrS4biDNIuyYe22PtLl4b3Yf55eW7AFgyzKgddsohZJuNNliKyD6cusHDhm7MYpnXc5zwTdCbt6KGK/tBaylNyYwH/WBAUhyRYN5EVt9/iIKHrb+P6Z0RL4nO3BtQE/Zwx1VC3g1Wy4PPQJjqLixQptzl5eIzu43JIO/LBvT/mWuheH4eoPlghvyMYwfHcs7B4d7FCv1Tj9Skp9Fcj6HBAZlq/ss/eIwk7oOcTviQs+EUF9/yYatgtpXX9RCyvhMU6/ghOLfXRmOpAzsmoGnVqEpc2IMlZegYtieLveXU35cGJMI6wCR2ciCJIX995vLuL/4BdCAMEhyMAUWxtaCD2ZfRHyOWKNuf80w9k6/Ofhu7RevCr2mjQJAVTyE2OWWgOUuYJ4pZim93J7slMXieL3S5/JM08Q8g179Of7dzpN/oG7s80ljxAiCprpUAwpEmNiqNJN//v0e9KxknhCHeAWSAe8IeXbp5PSEQHXTmsqOFRkpud1pTsETcNbdonk8XMyv8mZRcFPVWRRWUb8hupn/d+x9r6mOdKdJkH8ZZ0R30LG0SLPYEvsVr2yU9o+uCZrRWkuE3SP3Lq3BIx+0vtm0DOvj6cODxy5/4Zm4x7LIpSa9wr69Rs2x+t+U5ydUupZ7oiAbWfYZSXHpmB0zJYOLMPJZcut50J/IgWuTMda8QBcTG3jRr4BTwpcmBZRmddfOJYgD7EMpOi1HgwLnS7l5QELafaMn0Hl6G774GVy4lEK2jURG9IEE3PV1m5Y903pqldFkJQsMxdisJWOzVjbtf41fxxnt4cQgiDQhktqCwg8xP6ijzPeWgvQHL5fMq61cQ5/4HB+yt9wKWMlBfUJR+ocI2MYx2nUWz+0BwCnTU29D9bx1xkir9bsnUnfOlfRDO2OEvI0iTe82666rVQO9XqTEz3POxrJYzLcSC9fTHpHfmCVwT2zWGGLi6pW5kqZh/uzSQ12MSvF5+dwvhe7yRks5gwMhnDHMQFyxKw3Xxm+dq2Ix/1uUucOhCu2L72j/NIwkF3Z7O7afY9nIu+NqMe8PbPJjq5ovEluosQfAMzWJH5Va8iur8o6K1y6hm7XFdNYAR+uCtxMw6WzF58QWVXXrDvfPeBMaNz+VVCnGP9elAwv62tc4Uh2SCbKbWdZchtLHgrJgYgQtCMhBDh0AXzE6ubbtfm9jE2vWcPj5jbo8U72i1pL2j8Xfr562Xc2WrQ7tKvSFQepGxfu2XgF7q55XKVqrnrBeXxZViUkB/gyXxI26CfrVfPLW+sYUo3JS+eCjyn2K7phv+630ixdpKrRJCTmkP3G8tcoLTJCB67/pbz+dXiNSB4JlHf4i3FVRkr8TAWS2zuMjJhB+ZyxnrGq/m7KSwpEEqgSCpOrQ5nkeoKIOyITfe9EPBSy9QtYDK+SAhUiLnICVURK7kGgrhZuKyK5/nyK9l7ffg16aaChJBisPBeiYsTDHlAeq0GbW7VR/jQDAVtVldeyD/dM5rJ4X+wl3A+faYD1OUxYT3n8dMs+E/1jLnYixXJpo7iXCqlTV3phOatg4XDQ5Bj6EYQIljVI4x2e8XHspcETIa0WepLsZF7WUtY7KbN8ZyDBFXgTMb4lzPmWyY8hZ05uX8EBKqUJhWh91AUob/OpJdf+u3axDDeRgsjl8K6CFM/5uQKo93co3KPqGZiqx0JoVj1t6KGxkrzYsgwlrTyeL44cEgr0zRQz5oFExuwHGYyogbHZ8EvU1eoiJ3IuQFxUH/1ULidfGtB371RYfz9gqONi1KiVzJ+zLjw+4HgMKXOV+ra09Dyg+eyUNfHillLXkhKWVoDpUhc+r46W6vXFp3oMKUWTRM0dE7iHofo+0tHb73d3ID2blRXgUeoMCQwOptoAYlFBIUYjggrIhd1AMC8TiZmiNULyP5imDePwcfq+ZjGH3o8VKRI2FcoFmChQegGco6pEbB5DxCguDuJbFRwGH4t9T0y74ZhlZiTNKA4xXsQnfIBEC5qz3mkcDAWoe73zqFAjp35JRVBjo3UDvehJppxzuoCXt9UbeuNEGll5/YJR4lfbUsEai0U6TFVleTTY53ofYCWEM6EnNDIToTFbm514YFTUSc4h8Qlq2fPeqC3IcCmirNT4Kf0FCO7MQrtGNFPJme2cpb/pZguS3pxxkKb4lOS+eGiUBGcSs1v3zHroJ+hum4wTJFRG0Yb99aCVQU44wgV3nKW7FZkXzwO3QY7nnkFI2kaAXerCPF4+Ho463g==">
+
+                <span style="display: none;"><span class="text-gray-600" data-translate="error">error code:
+                        1020</span></span>
+            </form>
+        </div>
+    </div>
+    <script>
+        (function () {
+            var trkjs = document.createElement('img');
+            trkjs.setAttribute('src', '/cdn-cgi/images/trace/captcha/js/transparent.gif?ray=732fbc436ab471ed');
+            trkjs.setAttribute('style', 'display: none');
+            document.body.appendChild(trkjs);
+            var cpo = document.createElement('script');
+            cpo.src = '/cdn-cgi/challenge-platform/h/g/orchestrate/managed/v1?ray=732fbc436ab471ed';
+            window._cf_chl_opt.cOgUHash = location.hash === '' && location.href.indexOf('#') !== -1 ? '#' : location.hash;
+            window._cf_chl_opt.cOgUQuery = location.search === '' && location.href.slice(0, -window._cf_chl_opt.cOgUHash.length).indexOf('?') !== -1 ? '?' : location.search;
+            if (window.history && window.history.replaceState) {
+                var ogU = location.pathname + window._cf_chl_opt.cOgUQuery + window._cf_chl_opt.cOgUHash;
+                history.replaceState(null, null, "\/search?q=2022&__cf_chl_rt_tk=lkycIb1jDXlmFqiB7AXTwy38_EzYPvu79CCQyU9lhUE-1659201316-0-gaNycGzNCf0" + window._cf_chl_opt.cOgUHash);
+                cpo.onload = function () {
+                    history.replaceState(null, null, ogU);
+                };
+            }
+            document.getElementsByTagName('head')[0].appendChild(cpo);
+        }());
+    </script><img src="Just%20a%20moment_files/transparent.gif" style="display: none">
+
+    <div class="footer" role="contentinfo">
+        <div class="footer-inner">
+            <div class="clearfix diagnostic-wrapper">
+                <div class="ray-id">Ray ID: <code>732fbc436ab471ed</code></div>
+            </div>
+            <div class="text-center">
+                Performance &amp; security by
+                <a rel="noopener noreferrer" href="https://www.cloudflare.com/" target="_blank">Cloudflare</a>
+            </div>
+        </div>
+    </div>
+
+
+    <div style="background-color: rgb(255, 255, 255); border: 1px solid rgb(215, 215, 215); box-shadow: rgba(0, 0, 0, 0.1) 0px 0px 4px; border-radius: 4px; left: -10000px; top: -10000px; z-index: -2147483648; position: absolute; transition: opacity 0.15s ease-out 0s; opacity: 0; visibility: hidden;"
+        aria-hidden="true">
+        <div style="position: relative; z-index: 1;"><iframe src="Just%20a%20moment_files/hcaptcha_003.html"
+                title="Main content of the hCaptcha challenge" scrolling="no"
+                style="border: 0px none; z-index: 2000000000; position: relative;" frameborder="0"></iframe></div>
+        <div
+            style="width: 100%; height: 100%; position: fixed; pointer-events: none; top: 0px; left: 0px; z-index: 0; background-color: rgb(255, 255, 255); opacity: 0.05;">
+        </div>
+        <div
+            style="border-width: 11px; position: absolute; pointer-events: none; margin-top: -11px; z-index: 1; right: 100%;">
+            <div
+                style="border-width: 10px; border-style: solid; border-color: transparent rgb(255, 255, 255) transparent transparent; position: relative; top: 10px; z-index: 1;">
+            </div>
+            <div
+                style="border-width: 11px; border-style: solid; border-color: transparent rgb(215, 215, 215) transparent transparent; position: relative; top: -11px; z-index: 0;">
+            </div>
+        </div>
+    </div>
+    <div style="background-color: rgb(255, 255, 255); border: 1px solid rgb(215, 215, 215); box-shadow: rgba(0, 0, 0, 0.1) 0px 0px 4px; border-radius: 4px; left: -10000px; top: -10000px; z-index: -2147483648; position: absolute; transition: opacity 0.15s ease-out 0s; opacity: 0; visibility: hidden;"
+        aria-hidden="true">
+        <div style="position: relative; z-index: 1;"><iframe src="Just%20a%20moment_files/hcaptcha_004.html"
+                title="Main content of the hCaptcha challenge" scrolling="no"
+                style="border: 0px none; z-index: 2000000000; position: relative;" frameborder="0"></iframe></div>
+        <div
+            style="width: 100%; height: 100%; position: fixed; pointer-events: none; top: 0px; left: 0px; z-index: 0; background-color: rgb(255, 255, 255); opacity: 0.05;">
+        </div>
+        <div
+            style="border-width: 11px; position: absolute; pointer-events: none; margin-top: -11px; z-index: 1; right: 100%;">
+            <div
+                style="border-width: 10px; border-style: solid; border-color: transparent rgb(255, 255, 255) transparent transparent; position: relative; top: 10px; z-index: 1;">
+            </div>
+            <div
+                style="border-width: 11px; border-style: solid; border-color: transparent rgb(215, 215, 215) transparent transparent; position: relative; top: -11px; z-index: 0;">
+            </div>
+        </div>
+    </div>
+</body>
+
+</html>

html_samples/cloudflare_captcha_norobot_v1.html

@@ -0,0 +1,170 @@
+<!DOCTYPE html>
+<html lang="en-US">
+
+<head>
+    <title>Just a moment...</title>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=Edge">
+    <meta name="robots" content="noindex,nofollow">
+    <meta name="viewport" content="width=device-width,initial-scale=1">
+    <link href="Just%20a%20moment2_files/cf-errors.css" rel="stylesheet">
+
+    <script>
+        (function () {
+            window._cf_chl_opt = {
+                cvId: '2',
+                cType: 'managed',
+                cNounce: '94250',
+                cRay: '732fc1c74f757330',
+                cHash: '8c4978fa93c1751',
+                cUPMDTk: "\/search?q=2022&__cf_chl_tk=6E3KpS5eCzuCMJG64ch2shvOMHdwQ8ioliqACpoQqM8-1659201542-0-gaNycGzNCeU",
+                cFPWv: 'g',
+                cTTimeMs: '1000',
+                cTplV: 2,
+                cRq: {
+                    ru: 'aHR0cHM6Ly8wbWFnbmV0LmNvbS9zZWFyY2g/cT0yMDIy',
+                    ra: 'TW96aWxsYS81LjAgKFgxMTsgTGludXggeDg2XzY0OyBydjoxMDUuMCkgR2Vja28vMjAxMDAxMDEgRmlyZWZveC8xMDUuMA==',
+                    rm: 'R0VU',
+                    d: 'C4CtJo9JDMtUWZ0r+/s2CwjYdSTdqGYK3qFo1OXpvSc9v7/3d5QuMwmvG3e5oV1BpjlQb8eJJ23gVRxavjw/gpPp1brmKoHuvcJEmAP3Sof38vqcpF91/9NHe3JbmCM2xshiGvJdbpJXb5wXdYKYPMqy7NUHL1VU4hupa3Da3tBq9zyuMa1NcZaiyeE6piSl7n96m+VziRdwyG+SBUldIG/Fsv9J1yl+Gj19wbX1XEneMXChcClGgRrSe1MTd9thLkq2NGFqROnsUmpA8b+2Eqi+IPYQfkPcydWkHmJqQixN9ZFTIBChIC60hGHOQ7O354ju65tVGAhB/nBRREpdqvwoYzgufgg83+dbPHVdQasiuLRHvftOtHhS5/iaBOVoEBH+rElTSk/OYjU2Yh6gkQj0FjkbebEBptFeVAxgqoYZljOrhamWYYZ14tOKeonzc1rz/FXNTM5qVtrWCwAlt9SsXDjM/GYXZMTbOdNLnLZGlLNQCx+l6hMC0OQC45sWFzZECljbjXwiYfodKobeqe11lUXnskj8AN5Qc7O8OqtALsxoNCLZ7ou+ORY0lauremeuu3U3WqadgSGFGA+TZZw2VcCA3BIUKCGlsNLBlJ8wQS2UAGJfGLOVuhErmtsM',
+                    t: 'MTY1OTIwMTU0Mi4yOTUwMDA=',
+                    m: 'eWHHJ28v6yOyvSePVqcdyHxAYkkc3xq3VJ8YiDCk5nk=',
+                    i1: 'M3dMvem+HcwSbNQrJbaYdQ==',
+                    i2: 'ebY327qYCu6NZKHSQXkbaQ==',
+                    zh: 'qP4bnGc6j96JlnjNSE7HmQci3S9L50bHFtm4bQRjjKU=',
+                    uh: 'SK3PXNkeRzZtkRARhJpbmZpCIiWQw6+5gpOE7vojWx4=',
+                    hh: 'azXzJl8Ou22g0nN/9idVUoB9EqZ7fLmkSdDRHM3Lkmw=',
+                }
+            }
+            window._cf_chl_enter = function () { window._cf_chl_opt.p = 1 };
+        })();
+    </script>
+
+    <script src="Just%20a%20moment2_files/v1.js"></script>
+    <script type="text/javascript" src="Just%20a%20moment2_files/api.js"></script>
+</head>
+
+<body class="no-js">
+
+    <div class="privacy-pass">
+        <a rel="noopener noreferrer" href="https://addons.mozilla.org/en-US/firefox/addon/privacy-pass/"
+            target="_blank">
+            Privacy Pass
+            <span class="privacy-pass-icon-wrapper">
+                <div class="privacy-pass-icon"></div>
+            </span>
+        </a>
+    </div>
+
+    <div class="main-wrapper" role="main">
+        <div class="main-content">
+            <h1 class="zone-name-title h1">
+                <img class="heading-favicon" src="Just%20a%20moment2_files/favicon.ico"
+                    onerror="this.onerror=null;this.parentNode.removeChild(this)">
+                0MAGNET.COM
+            </h1>
+            <h2 class="h2" id="cf-challenge-running">
+                Checking if the site connection is secure
+            </h2>
+            <div id="cf-challenge-stage" style="display: block;">
+                <div id="cf-norobot-container" style="display: flex;"><input type="button" value="Verify you are human"
+                        class="big-button pow-button" style="cursor: pointer;"></div>
+            </div>
+            <div id="cf-challenge-spinner" class="spacer loading-spinner" style="display: none; visibility: hidden;">
+                <div class="lds-ring">
+                    <div></div>
+                    <div></div>
+                    <div></div>
+                    <div></div>
+                </div>
+            </div>
+            <noscript>
+                <div id="cf-challenge-error-title">
+                    <div class="h2">
+                        <span class="icon-wrapper">
+                            <div class="heading-icon warning-icon"></div>
+                        </span>
+                        <span id="cf-challenge-error-text">
+                            Enable JavaScript and cookies to continue
+                        </span>
+                    </div>
+                </div>
+            </noscript>
+            <div
+                style="display:none;background-image:url('/cdn-cgi/images/trace/captcha/nojs/transparent.gif?ray=732fc1c74f757330')">
+            </div>
+            <div id="cf-challenge-body-text" class="core-msg spacer">
+                0magnet.com needs to review the security of your connection before
+                proceeding.
+            </div>
+            <div id="cf-challenge-fact-wrapper" style="display: block; visibility: visible;" class="fact spacer hidden">
+                <span class="fact-title">Did you know</span> <span id="cf-challenge-fact" class="body-text">botnets can
+                    be used to shutdown popular websites?</span>
+            </div>
+            <div id="cf-challenge-explainer-expandable" class="hidden expandable body-text spacer"
+                style="display: block; visibility: visible;">
+                <div class="expandable-title" id="cf-challenge-explainer-summary"><button class="expandable-summary-btn"
+                        id="cf-challenge-explainer-btn" type="button"> Why am I seeing this page? <span
+                            class="caret-icon-wrapper">
+                            <div class="caret-icon"></div>
+                        </span> </button> </div>
+                <div class="expandable-details" id="cf-challenge-explainer-details">
+                    Requests from malicious bots can pose as legitimate traffic.
+                    Occasionally, you may see this page while the site ensures that the
+                    connection is secure.</div>
+            </div>
+            <div id="cf-challenge-success" style="display: none;">
+                <div class="h2"><span class="icon-wrapper"><img class="heading-icon" alt="Success icon"
+                            src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADQAAAA0CAMAAADypuvZAAAANlBMVEUAAAAxMTEwMDAxMTExMTEwMDAwMDAwMDAxMTExMTExMTEwMDAwMDAxMTExMTEwMDAwMDAxMTHB9N+uAAAAEXRSTlMA3zDvfyBAEJC/n3BQz69gX7VMkcMAAAGySURBVEjHnZZbFoMgDEQJiDzVuv/NtgbtFGuQ4/zUKpeMIQbUhXSKE5l1XSn4pFWHRm/WShT1HRLWC01LGxFEVkCc30eYkLJ1Sjk9pvkw690VY6k8DWP9OM9yMG0Koi+mi8XA36NXmW0UXra4eJ3iwHfrfXVlgL0NqqGBHdqfeQhMmyJ48WDuKP81h3+SMPeRKkJcSXiLUK4XTHCjESOnz1VUXQoc6lgi2x4cI5aTQ201Mt8wHysI5fc05M5c81uZEtHcMKhxZ7iYEty1GfhLvGKpm+EYkdGxm1F5axmcB93DoORIbXfdN7f+hlFuyxtDP+sxtBnF43cIYwaZAWRgzxIoiXEMESoPlMhwLRDXeK772CAzXEdBRV7cmnoVBp0OSlyGidEzJTFq5hhcsA5388oSGM6b5p+qjpZrBlMS9xj4AwXmz108ukU1IomM3ceiW0CDwHCqp1NjAqXlFrbga+xuloQJ+tuyfbIBPNpqnmxqT7dPaOnZqBfhSBCteJAxWj58zLk2xgg+SPGYM6dRO6WczSnIxxwEExRaO+UyCUhbOp7CGQ+kxSUfNtLQFC+Po29vvy7jj4y0yAAAAABJRU5ErkJggg=="></span>Connection
+                    is secure</div>
+                <div class="core-msg spacer">Proceeding...</div>
+            </div>
+            <form id="challenge-form"
+                action="/search?q=2022&amp;__cf_chl_f_tk=6E3KpS5eCzuCMJG64ch2shvOMHdwQ8ioliqACpoQqM8-1659201542-0-gaNycGzNCeU"
+                method="POST" enctype="application/x-www-form-urlencoded">
+                <input type="hidden" name="md"
+                    value="UPeuijc1TS5ZQ21GIY6wjg6HHN_jWKH9sqolcSJABwg-1659201542-0-AR_ZxgiwVB4GwEgAjllIrmnGAumHNwuvfpFBddySYLh6CWexrUnxVYlX_wlB19Yndm45fs-KngMxbYB4dEOuf4MOJ_yL_BsNG3_cIPybV0bNn9WQXecJg3FfFrIBuMFIappZOX4hdDjLtRo9f4JsVsU6FzD9sUoKJRd4BTkjTAm25yFbqmPgV15XZhnJ5HRux044u0IIOVZCwTTzgRLCqToVb-OfiuUcHBzt4W7_wNlF1ObUi2oEr00DA1zZvzzY2KnXdZVN8m2OaNY_f2zkk9uDlLQRob_Ti6MHPNDr4eRkyMqZMZ1XDCxe-9lBkcEfpqtg6_4yac9ZiIEoNdJnJVE6cuNzb59DcBooXAq3IWp6fK4y4UIBStjqOXk4bxQb5yt1COfdPuQ9iLE_7yYOPG_t7n5I-4mjwvG7_U337A17oeEemXHfJkGC88Vm3SQdEHiW96VJuOA_X-rb7p3iOMlLYB5DKJ5DaBoPnP86uAWhoHWE6nrVzeAxeQ1y0uBHYPioJba5Kn9d-e2HsTMuAi7ZgSKuk90ApclIiW3owI4bLc4wxO5cu3ZIz7sZfbdvIKDhf9ESZhpQrITU_4Hgqjz0s3lt-MVeNP_0bz31XSeA--pdiulzUpQWLx1jhC4s7Av6STUb9bmbHpE41283KbbpuzBbmHN1UczNiaaquYZiEXRHKYyEMhKD782nWTJwQA">
+                <input type="hidden" name="r"
+                    value="i1ShtnCs9Zs8QexeFnp6EFtrWs3WbGEVQGXbVfYwpRI-1659201542-0-AbDM6G9qkbgoH+BqDdr1tzCDHr/DU9Sdxelapvp2/FZN6VqYfpDkJGv+HxhBQng6aVktcEobxp2ouOxJZxPQrR6tVFIhOW6uPOAdy5kh2BBJWUHfER13aq8LQ86fvDyRh3AThEHj6bgs2udacfvOrDrHT2j/KHBPePlGKbh8rzDTJBKw0ejUleHk8eKX/BQ1bVULgxT+ZZY721lyn2wrjsde1j1OAsiiCDkVvQ4Rs+Bas7UApD5HeWzyrCu2VFk/Qf+Rk+6spM+StYenQUAKXXrekJoIeNxPf/W9ZRsJfwUoY0JUK2thOWiwQOtw21nVDpiCFB9nhhOsmBzBoRQGjckZyu/O5U7jMIFdS9ThCFC0Kffg0MEr5xTkmgw+CNSwN7AlI9v3GS2XdTFOPXe29b68fZXzYfbm2CjqYhmxomZjCGTAmkzXWaVnMOs9Vl/8VurCUEu8SAt5k9Za/vFrEurX1edXNCviVuTOBSLHjqBiLui9FbufzGLq6BaHYi3WIFA1nMkoxduxbErP+Eqyi8UNvzvmEqUbj2COalXcQzkbHkyyLo33MNHZEi1zhhHjwCm1lp6mm4BRe60kRgTHb8X7oxBpY4vEcMz4jQQdsW15xBPAjsH8m9cj1H2ujpd7kfo8JGTyZ7FcoxOzGOuZr8XRpGkH72HaWYz7M+GIb3BBZ1v2Za7sSrzinNLFjHCCVXq68MqOmZ6RhgeexGoKJzcMHsHvgGXB8CisyyNTtA3OQOujybNUnNzlW7vJ/wDreTHkko6jQ/Lm/X2GnLg85BIg6IeROzt3eInAYsCaNKpST/h5bvSGCyzRoOW46oO8ZzZrV2FI2rEr0xLTIVWzQ//K2iGOCz58RisCfxWiF2n+fzj/5nE/0cjTPzYP68TM5BxB058EO7ZEFbgqhUji8IR9V2ahy7kI9dUhwd2S4IyjL+O6hCNPwjpRohkt93wXUCZDMgNoxi1BIylqqtAxYBfodyjFz8mB8GgcqBaCHN3tI0BINENVfvSJwKniYxL73frTX5KEqniT9GdT15o4F7QLf4S1atwYzF6ezJTYgLf6fOWUZKpaFMSRzEsxmZDmOFZeiss8lj7bKS6drOpkaOYzZiSgp5t5VwLKT0yQ+PDWQmqkpZ5WOa9/ayXLyOCunzk1IUO6VkvgFe0P2LZC9XEZUfwAFakYemej8/SZx0EknoPob1il3MMsbfHNAvcvUJK9xDbdAQ7rz34r4D5zO2aPnmYw1yv9K36z78I2dZpjVT9kpiKFwaOTkuSDUDtcmnhKM1XE+goG/C66G6PsChpGKLCeaDw4Rp7BxlumiSGB4Mp/bs8pTz3gez7pSu1oNodr7Tr1wJvCK8T5nVJ5GRO/tQ+Ff2K2s67udoV0CFtKufJyRsGCEv/0u5sArg3uwtwIz1W0JtAVjhe+J2nUihLa0Gqm7AwcCwfhsLHOhMG28V2NAw19iVq8RuMN7A2kGg5PH6bUeilWUxxZvWyDfyRSJZYMQytwAJdt4gQ++Qnl3mcaSk1N3pSiltVUDpfLcYb5gd35m+mKQWtPnIDlJMAtGoBeqROQPLNDg+LYdI/dnJzIOHjI3J+pTWhbAlF7B7NtccZOHmI9Cl3vS6Fpqs5aSPEDoDENTap6JN1kgm5NszMay9tAm66AcKF95W6QhwgQsyRrwScgRaPUtCx9ZJcbav6T/CAulcBB85MjwAd8+HF1g+UZT9VvChZoxh7NzfMoR53pVbxvW6acO8oVN5ITTP8mNAIisRvWi2KVdi4KqaLjYtLFNN8AMzjAC0vBIaFyGZlIbFsB44MRiMufD64b/66dqeC0l0WrUlUG/DgrnSQr6lgK2gONJKPQZGXoaK0Ga8O8xMOkaFLNaqH5UH5KpHvIQ8nwhuXk/MS/7Gdp1W02OEB4l0hhKFytgWdo9QmCquSatvOjuFyRPa6tV8ceGmuDnQw22bJM9BwzdKlHn/2/mHjCz7gcEA3Hb/CbeP8V8mF1mc5R8HEEdz/rx+BTESmGiRivv+WQYpRKNh77iqbYvCvkduK4b3UErNbvcS10aTt8zDFF/oIwjDpsniJsrIUcC0FdQRs2dqPIfkoSHvs7YGmOjx9QThCAiTkPKUE9C5C4YPY4CWRV3nYAFJrTq0F047PkzDYm0AJMCahWK7Vq/Ra3l3nRHu9yI+P0HiruUbkzLgiEJYnAuUtxvpC/Vj0uhr+A0R9Obs1MHkwtDuMs/ETh3ZymeFtWLj70StkslJxTzKGimZSsqQXRFYGHY6CHqHwIXGrArYNjTty48VIfbfaEu58KQp6roOdFmx90AcK2lV0V5UdyuzDJeH/V5ERAmxWLrXQKWgiDrY4ZqecnRk5XEAVMq/ChPts9gR7xsQK5WsHtQNKLfltkL8YvAoS+jZvxzfUUBg99YSC4J/HzQS+FQAnkDxCgeroahXysNN1bgDASXOrn3NsC3LYpUiZ2AVTLPkj1roR9r65O">
+
+                <span style="display: none;"><span style="display: none;" class="text-gray-600"
+                        data-translate="error">error code: 1020</span></span>
+            </form>
+        </div>
+    </div>
+    <script>
+        (function () {
+            var trkjs = document.createElement('img');
+            trkjs.setAttribute('src', '/cdn-cgi/images/trace/captcha/js/transparent.gif?ray=732fc1c74f757330');
+            trkjs.setAttribute('style', 'display: none');
+            document.body.appendChild(trkjs);
+            var cpo = document.createElement('script');
+            cpo.src = '/cdn-cgi/challenge-platform/h/g/orchestrate/managed/v1?ray=732fc1c74f757330';
+            window._cf_chl_opt.cOgUHash = location.hash === '' && location.href.indexOf('#') !== -1 ? '#' : location.hash;
+            window._cf_chl_opt.cOgUQuery = location.search === '' && location.href.slice(0, -window._cf_chl_opt.cOgUHash.length).indexOf('?') !== -1 ? '?' : location.search;
+            if (window.history && window.history.replaceState) {
+                var ogU = location.pathname + window._cf_chl_opt.cOgUQuery + window._cf_chl_opt.cOgUHash;
+                history.replaceState(null, null, "\/search?q=2022&__cf_chl_rt_tk=6E3KpS5eCzuCMJG64ch2shvOMHdwQ8ioliqACpoQqM8-1659201542-0-gaNycGzNCeU" + window._cf_chl_opt.cOgUHash);
+                cpo.onload = function () {
+                    history.replaceState(null, null, ogU);
+                };
+            }
+            document.getElementsByTagName('head')[0].appendChild(cpo);
+        }());
+    </script><img src="Just%20a%20moment2_files/transparent.gif" style="display: none">
+
+    <div class="footer" role="contentinfo">
+        <div class="footer-inner">
+            <div class="clearfix diagnostic-wrapper">
+                <div class="ray-id">Ray ID: <code>732fc1c74f757330</code></div>
+            </div>
+            <div class="text-center">
+                Performance &amp; security by
+                <a rel="noopener noreferrer" href="https://www.cloudflare.com/" target="_blank">Cloudflare</a>
+            </div>
+        </div>
+    </div>
+
+
+</body>
+
+</html>

html_samples/cloudflare_init_v1.html

@@ -0,0 +1,120 @@
+<!DOCTYPE html>
+<html lang="en-US">
+
+<head>
+    <title>Just a moment...</title>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+    <meta http-equiv="X-UA-Compatible" content="IE=Edge" />
+    <meta name="robots" content="noindex,nofollow" />
+    <meta name="viewport" content="width=device-width,initial-scale=1" />
+    <link href="/cdn-cgi/styles/cf-errors.css" rel="stylesheet" />
+
+    <script>
+        (function () {
+            window._cf_chl_opt = {
+                cvId: '2',
+                cType: 'managed',
+                cNounce: '46449',
+                cRay: '732fd3bc9c1d72de',
+                cHash: '8838fcad2a7f56c',
+                cUPMDTk: "\/search?q=2022&__cf_chl_tk=y4XnN88eYeUiXmFkQeqEipve1VuK0jJA.G4Hz6xztsM-1659202277-0-gaNycGzNBz0",
+                cFPWv: 'g',
+                cTTimeMs: '1000',
+                cTplV: 2,
+                cRq: {
+                    ru: 'aHR0cHM6Ly8wbWFnbmV0LmNvbS9zZWFyY2g/cT0yMDIy',
+                    ra: 'Y3VybC83Ljg0LjA=',
+                    rm: 'R0VU',
+                    d: '+SdFLvm4kJf8Z9BVci1ZbUOY6ab/Dm5Zzyb0IvscIzmY9PnAAcvPfJ/3TD9YJViBxB/ArnbCQrOUfbSkq4odyaZmW19gm+exRuL8Z3POm1ABs7y6jwMshM19q4Gr3eFY/MUO/IYWuyA2F9q94hRCI6ZNb7dLEh9yh6hORbKRd62pdn59h1xCx8tNdKDtP7VXPXo85nYmJJPLOdXTnII+YxZ03a4isAmBHbi+lGoQN/bCV0K006VmpfPElAfAO9jm45o7pc1NgPQhZSKWpTyI/nHMueH6wacPREzN5RtREoQfKuwYpV++Gq56qr5bAe/SKeF+rI0x7OSqC4HQvrNwbA+kHZzaxgOKeiMFjDxmro/GyC/+sxeZmrxnSIAh4BScjPxEl1FLLkg/6D0JH6HmxoT8N/Jgpi9447Am4WeX+WQxJ9+uDs5WrFIahx7pWrgcZUTRPh+UCu3allJ2Q3cAfwK6BclhES/HhBBbJv0pnR1R2RfKDM/gr1MpLuhaK4mFEO/kSyNUjOnCjOfd+5d7Qb0DZn7sHpF2SVc+zNv5OWSvCRDUcNHjIOV6fq0datVyVWmxD6unPS0MMUFO+ZZNiB4ionrhVCiLrb2FjPQ8tzyCqXg+tnV7WtZ0h4+JuK3rxcaQ8PQy60/As8dKHqVTnw==',
+                    t: 'MTY1OTIwMjI3Ny44NjMwMDA=',
+                    m: 'zvAOPvfoONkW1BzH+jMnKOPtDpPpZijRP52DVDWH+i8=',
+                    i1: 'dDlQDNhOEuHzFEPo/etoAA==',
+                    i2: '+LTK9hchBRjTTQk1WQU1Vw==',
+                    zh: 'qP4bnGc6j96JlnjNSE7HmQci3S9L50bHFtm4bQRjjKU=',
+                    uh: 'IdIU2i4FhVxxcYhzSFWdjoBuQm7qnyVK65JGofJuWV4=',
+                    hh: 'azXzJl8Ou22g0nN/9idVUoB9EqZ7fLmkSdDRHM3Lkmw=',
+                }
+            }
+            window._cf_chl_enter = function () { window._cf_chl_opt.p = 1 };
+        })();
+    </script>
+
+</head>
+
+<body class="no-js">
+
+    <div class="main-wrapper" role="main">
+        <div class="main-content">
+            <h1 class="zone-name-title h1">
+                <img class="heading-favicon" src="/favicon.ico"
+                    onerror="this.onerror=null;this.parentNode.removeChild(this)" />
+                0MAGNET.COM
+            </h1>
+            <h2 class="h2" id="cf-challenge-running">
+                Checking if the site connection is secure
+            </h2>
+            <noscript>
+                <div id="cf-challenge-error-title">
+                    <div class="h2">
+                        <span class="icon-wrapper">
+                            <div class="heading-icon warning-icon"></div>
+                        </span>
+                        <span id="cf-challenge-error-text">
+                            Enable JavaScript and cookies to continue
+                        </span>
+                    </div>
+                </div>
+            </noscript>
+            <div
+                style="display:none;background-image:url('/cdn-cgi/images/trace/captcha/nojs/transparent.gif?ray=732fd3bc9c1d72de')">
+            </div>
+            <div id="cf-challenge-body-text" class="core-msg spacer">
+                0magnet.com needs to review the security of your connection before
+                proceeding.
+            </div>
+            <form id="challenge-form"
+                action="/search?q=2022&amp;__cf_chl_f_tk=y4XnN88eYeUiXmFkQeqEipve1VuK0jJA.G4Hz6xztsM-1659202277-0-gaNycGzNBz0"
+                method="POST" enctype="application/x-www-form-urlencoded">
+                <input type="hidden" name="md"
+                    value="DpGhFnuVRfDhqsQNASrgdT4WiiJ8m6lqTIs03.l6RLc-1659202277-0-AfUEAk9DsJ4rmpVI_Al7-eogy2CmM3YgWe4-31iw0oG2CcDIbYvauEW2IvK9m27_gq1FvdH-UPaGHR0q6Q2haXlX4pgmQK5rlQUSEd5HquGdtWMasHWqL_Q_TZGdOKz30bE2FEk8wLHRErHJRJloDRj0tiG8MreT2La_GLvovNK1XbMXDxFZT2Cc-DThBvxbgbDffw3okYfdl1ECXhLw9G6L4o8xgLsz3QZQG3dNZNhm5n4mf55-BBsFDzDTEN1_1BgORVw3mtbsodedktcACsVBCRupyBpTev9MML1jHzk06ZT9dhcCP4zXvsMS4-gG212LFu79Cpl0MHifKvPk0DTJQja1ulaT4gVuIvmLPihPh1IYMGbEcdX4MFH0Wu_RL6UPINE6esf-oAx8-imKhKITB_R4974rpq9XJk65Kf9R6AJhu072CyOqW1YcmYMkUCqFjdZnRyNgHRT2Q5bMEJ8fv0DwfFV6ynG7n6JGMd_pEnZp0nEvjWXpK6Ft8ZZGOXtFMfmW4vNgFhs6xJ1wnaJWuLXae3V6gTZYxMkeIsyMzlvRSzYBz_rgRBNkvvAwbNvOZ369tKbaElS39hOI1WTaoOsnY2d0Z4mDe4AVbSs3fVJGikzZSa3Ctr1RnqqOztVIRYL1Q7IYRJ02P6egL7sn7RniJ6znNAoPhaWJLYzynWXeQF5YO5U0Zf779qkm3A" />
+                <input type="hidden" name="r"
+                    value="QJznOl.RWpNvkdG1Pf6TAzaNhRIFpH8DJ0w1yAuwRLw-1659202277-0-AcOBapBisncM3qf1RYdkTNlIXCth/TmoAMnk3vJozFlG8/vYeLPpjG389mhQu01aSlpJqFWn0VQf9c/7w3yh85jHmrpaxJtpxTiSL9k+AWm61kE6DkHgJBl5jUc7gu4W3oHdmP4FyOUzhbBpIkOAntSkVJJmgu6SaIE3I9fRAFu7bPxBveT8zZGyVUJSPKpwx/w4rNPzs2VnCeEVL1eOdbLInHYR1kqC8M4JyBynwdVXxIX+j5o/rTrNK8E/W4UZMhuWqIaOnX7FzmceglyBSjDqJFLCt0TOhc66m82Y25Obi8Gvsqn34bjwPA2G8qOvgrHA2RFH6lEQFSdGMzLrF4qU5P9j9FzU1CPSTfGtkKbsMnGcMrtzmyQ7LdMIfghYvnCBXTi82iIzaSwzY3sEnW9KZs24Akxu/AV1E03sqW1CAA1UCRURpX4GKXvD6UYpSgc6++q8naLdRozkLP81T/CvHyIRdQx8vylmVN9u/rPvMbW1jWtniDmuAjBQDUd058YH+IRmm4lREG5JN2yeX083h/BG6tssEQVdTcIgwZRNDB+kK8vtOmywmo5qTAX1VE/sgfPCw5+3Xxu+hhZON3C7VGfrCQI5ZSb6+YvBLXmO26Nlp3fSOyeBwZy3pVuGwv/TrEo+e8USIlIs1T6MQJYQeX/4vOdy89npo6KBqY23giTFDh8EMZo//93hfBsRUbHrY/It6kp42qzsnWTjbkyiqd1zBSpQhuMyuMPeKpQ63oVI2tlGyioLg3HcfhbHQcdpUAWDn8lZ4+GTFVMix+20fGbErkVeBs7WvFSLlZ1YtYpCXrgVaomj7WCr8Icb7ASXKfvEuqC1ZnZgn6Lb6x3dUBGiDtnSFnixHFElIF6nPedVIV0+TxccjlV/LJeyNM58GHtRo4NcmIo1a6kN3vzPAjTUhgDJe4aYP6oVKCRNDcrHlGlLubu6XuIvBFM5Sq401xxahOe3VP2u7JovkzXwfl+yUxQOYaoq1LR+wnDhXgVbBNbM2QfIhez578zu2TN5bu5H14UXZ1E78KA6Op9b/PUgA1AsgTJVRk4M6OQSpa5wRIKkXzxpGIRz6+YBxSjIaX2I220GH4s6Te4CBpq77g6V4CVIkEvqZwbN9hIoAoljWVbEEdb3WmYZqoPxN/8ZIjU7uwQUyDgnCOlc7Z52TgG6nVvj7RVyxv5ugskW+fcOI12o35iYNNpXTh1boHyn7nlPG7wtSsl9UlTss27nd04AIzbH0qyX3kn77yPsobMDYUJ3IGhOujV8Cg08XHFIlYSYGPbqqpog+CuuWtzvwyk5mmHXkJNPyFEZL/irApJbatpGNqgGNnQL+5KYp+/U8/kROLTOWa8tG5609MF+wdrScsfPT9eE+HYh7tEFURnwm8kJtdAadcxYjzO60PFcUI1R5SMGHRflAnpY2gvAzbsSssk1WIF+6eHSe6FLHMXCHMp0w1XkNKpny5Ce3YTKhJ4TRg7HfN1pvet2Duj4G04A328uYUppPlU7Spz0fj5N/FHJf3sPaqJC8jn74L0mT92ecGaxS3ZGvytw51ulA00wgzfZDWL4pirzgYVjUQTqVl9FzWYua4Vk4l3BX0opWKA4FloLTP3ekrvmO/zkztMBV4fvK+F8JIOzLOs4AuoCv8uXl7Ny9wLQI3a0hJAdbXJpI3WV/iuV7da4fQao2Z2HiatQh3ZtdLqWmGtqQlcVtsBrac82eo7mKAfwltTfLlX9Drtp4ohwoFe0Upm+YsfY6DK7zHrk3k9GN7gm6cMi1neNFaqWZR9s8ABDBg==" />
+
+            </form>
+        </div>
+    </div>
+    <script>
+        (function () {
+            var trkjs = document.createElement('img');
+            trkjs.setAttribute('src', '/cdn-cgi/images/trace/captcha/js/transparent.gif?ray=732fd3bc9c1d72de');
+            trkjs.setAttribute('style', 'display: none');
+            document.body.appendChild(trkjs);
+            var cpo = document.createElement('script');
+            cpo.src = '/cdn-cgi/challenge-platform/h/g/orchestrate/managed/v1?ray=732fd3bc9c1d72de';
+            window._cf_chl_opt.cOgUHash = location.hash === '' && location.href.indexOf('#') !== -1 ? '#' : location.hash;
+            window._cf_chl_opt.cOgUQuery = location.search === '' && location.href.slice(0, -window._cf_chl_opt.cOgUHash.length).indexOf('?') !== -1 ? '?' : location.search;
+            if (window.history && window.history.replaceState) {
+                var ogU = location.pathname + window._cf_chl_opt.cOgUQuery + window._cf_chl_opt.cOgUHash;
+                history.replaceState(null, null, "\/search?q=2022&__cf_chl_rt_tk=y4XnN88eYeUiXmFkQeqEipve1VuK0jJA.G4Hz6xztsM-1659202277-0-gaNycGzNBz0" + window._cf_chl_opt.cOgUHash);
+                cpo.onload = function () {
+                    history.replaceState(null, null, ogU);
+                };
+            }
+            document.getElementsByTagName('head')[0].appendChild(cpo);
+        }());
+    </script>
+
+    <div class="footer" role="contentinfo">
+        <div class="footer-inner">
+            <div class="clearfix diagnostic-wrapper">
+                <div class="ray-id">Ray ID: <code>732fd3bc9c1d72de</code></div>
+            </div>
+            <div class="text-center">
+                Performance &amp; security by
+                <a rel="noopener noreferrer" href="https://www.cloudflare.com" target="_blank">Cloudflare</a>
+            </div>
+        </div>
+    </div>
+</body>
+
+</html>

html_samples/cloudflare_spinner_v1.html

@@ -0,0 +1,167 @@
+<html lang="en-US">
+
+<head>
+    <title>Just a moment...</title>
+    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=Edge">
+    <meta name="robots" content="noindex,nofollow">
+    <meta name="viewport" content="width=device-width,initial-scale=1">
+    <link href="/cdn-cgi/styles/cf-errors.css" rel="stylesheet">
+
+    <script>
+        (function () {
+            window._cf_chl_opt = {
+                cvId: '2',
+                cType: 'managed',
+                cNounce: '52875',
+                cRay: '732fa2449b567521',
+                cHash: '79cce74ebb92671',
+                cUPMDTk: "\/search?q=2022&__cf_chl_tk=1qWQAgl8.irfEoDb73Rb0pUm1SXbis3ZamDAIoTcPks-1659200251-0-gaNycGzNCFE",
+                cFPWv: 'g',
+                cTTimeMs: '1000',
+                cTplV: 2,
+                cRq: {
+                    ru: 'aHR0cHM6Ly8wbWFnbmV0LmNvbS9zZWFyY2g/cT0yMDIy',
+                    ra: 'TW96aWxsYS81LjAgKFgxMTsgTGludXggeDg2XzY0KSBBcHBsZVdlYktpdC81MzcuMzYgKEtIVE1MLCBsaWtlIEdlY2tvKSBDaHJvbWUvMTAzLjAuNTA2MC4xMzQgU2FmYXJpLzUzNy4zNg==',
+                    rm: 'R0VU',
+                    d: 'UfK0k9mFeKGEdqoWAUIbk3OXbXe9DOHoYXdKLPyxbICSIQBS4GSNYar0DtbPI7+UQ7UeBZ2XCdQinvgH0pgzJCF1qB0nkXtu0qlLk6EwkrGAKD/pMGFFQF2EaCw3m00/xoRCDgLZRl/wUkRGz3HUOkTuPeKgZjsFyPoPv7MbYSMUtH7QU6ruIh+O3hvDOT2oA/BOKbRMSTnFedTIXADXL6GE8ZyNZ33wJlef5KzT0MHlN+3eZTAt6urCvJaY3MdTXKVye6fwyjqGEksaJ6B85vwrifLTYEU4/bORwXx8mTQTqjo3kh1rATlmthQwBpcQtWXmgDUcJ5gPrOk1fzhqrhO4b++HiIx3P5YZ9Ko2D0NNWeg1AYIwDjh9rZg5m0MmCXh1VqXDbnpseQW1vPkkZAADxyvLf/eEc1o2EpYGpK+qSpMZ4RcngnU0o8A2nS+j/CNsid0315OrYVyOIZcw6L3ovu6yfAAAALyOmg5ctXCqjzRthoibUb58u+myxOtfX1ew9IzNq8Z6t6RlomjR7Iy/7BJiJQNCF98dllNbODHz//TymlI1m8D9w+CYlZFIpiWJVH1M4h+tabH5YrqDVbkJgY6yVAfnr/NI6d6NHrhN+eSW30jkvAmZ6JRMhVWW',
+                    t: 'MTY1OTIwMDI1MS42MjAwMDA=',
+                    m: '/e8nTBb03IHZzN/DSkoHPRu0Ndm3ynYs8g6ZC+VxHcc=',
+                    i1: 'tx+ntPfeE2Gv81s52vIOlA==',
+                    i2: 'fpw8a/EO+Fo2t/ZiNKxEcg==',
+                    zh: 'qP4bnGc6j96JlnjNSE7HmQci3S9L50bHFtm4bQRjjKU=',
+                    uh: 'Eex9UQDjphKtV6LyVQ95F/MC5kBA3Rj4lC6CudiU3Vs=',
+                    hh: 'azXzJl8Ou22g0nN/9idVUoB9EqZ7fLmkSdDRHM3Lkmw=',
+                }
+            }
+            window._cf_chl_enter = function () { window._cf_chl_opt.p = 1 };
+        })();
+    </script>
+
+    <script src="/cdn-cgi/challenge-platform/h/g/orchestrate/managed/v1?ray=732fa2449b567521"></script>
+    <script type="text/javascript"
+        src="https://cloudflare.hcaptcha.com/1/api.js?endpoint=https%3A%2F%2Fcloudflare.hcaptcha.com&amp;assethost=https%3A%2F%2Fcf-assets.hcaptcha.com&amp;imghost=https%3A%2F%2Fcf-imgs.hcaptcha.com&amp;render=explicit&amp;recaptchacompat=off&amp;onload=_cf_chl_hload"></script>
+</head>
+
+<body class="no-js">
+
+    <div class="privacy-pass">
+        <a rel="noopener noreferrer"
+            href="https://chrome.google.com/webstore/detail/privacy-pass/ajhmfdgkijocedmfjonnpjfojldioehi"
+            target="_blank">
+            Privacy Pass
+            <span class="privacy-pass-icon-wrapper">
+                <div class="privacy-pass-icon"></div>
+            </span>
+        </a>
+    </div>
+
+    <div class="main-wrapper" role="main">
+        <div class="main-content">
+            <h1 class="zone-name-title h1">
+                <img class="heading-favicon" src="/favicon.ico"
+                    onerror="this.onerror=null;this.parentNode.removeChild(this)">
+                0MAGNET.COM
+            </h1>
+            <h2 class="h2" id="cf-challenge-running">
+                Checking if the site connection is secure
+            </h2>
+            <div id="cf-challenge-stage" style="display: none;"></div>
+            <div id="cf-challenge-spinner" class="spacer loading-spinner" style="display: block; visibility: visible;">
+                <div class="lds-ring">
+                    <div></div>
+                    <div></div>
+                    <div></div>
+                    <div></div>
+                </div>
+            </div>
+            <noscript>
+                <div id="cf-challenge-error-title">
+                    <div class="h2">
+                        <span class="icon-wrapper">
+                            <div class="heading-icon warning-icon"></div>
+                        </span>
+                        <span id="cf-challenge-error-text">
+                            Enable JavaScript and cookies to continue
+                        </span>
+                    </div>
+                </div>
+            </noscript>
+            <div
+                style="display:none;background-image:url('/cdn-cgi/images/trace/captcha/nojs/transparent.gif?ray=732fa2449b567521')">
+            </div>
+            <div id="cf-challenge-body-text" class="core-msg spacer">
+                0magnet.com needs to review the security of your connection before
+                proceeding.
+            </div>
+            <div id="cf-challenge-fact-wrapper" class="fact spacer hidden" style="display: block; visibility: visible;">
+                <span class="fact-title">Did you know</span> <span id="cf-challenge-fact" class="body-text">bots
+                    historically made up nearly 40% of all internet traffic?</span>
+            </div>
+            <div id="cf-challenge-explainer-expandable" class="hidden expandable body-text spacer"
+                style="display: none;">
+                <div class="expandable-title" id="cf-challenge-explainer-summary"><button class="expandable-summary-btn"
+                        id="cf-challenge-explainer-btn" type="button"> Why am I seeing this page? <span
+                            class="caret-icon-wrapper">
+                            <div class="caret-icon"></div>
+                        </span> </button> </div>
+                <div class="expandable-details" id="cf-challenge-explainer-details"> Requests from malicious bots can
+                    pose as legitimate traffic. Occasionally, you may see this page while the site ensures that the
+                    connection is secure.</div>
+            </div>
+            <div id="cf-challenge-success" style="display: none;">
+                <div class="h2"><span class="icon-wrapper"><img class="heading-icon" alt="Success icon"
+                            src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAADQAAAA0CAMAAADypuvZAAAANlBMVEUAAAAxMTEwMDAxMTExMTEwMDAwMDAwMDAxMTExMTExMTEwMDAwMDAxMTExMTEwMDAwMDAxMTHB9N+uAAAAEXRSTlMA3zDvfyBAEJC/n3BQz69gX7VMkcMAAAGySURBVEjHnZZbFoMgDEQJiDzVuv/NtgbtFGuQ4/zUKpeMIQbUhXSKE5l1XSn4pFWHRm/WShT1HRLWC01LGxFEVkCc30eYkLJ1Sjk9pvkw690VY6k8DWP9OM9yMG0Koi+mi8XA36NXmW0UXra4eJ3iwHfrfXVlgL0NqqGBHdqfeQhMmyJ48WDuKP81h3+SMPeRKkJcSXiLUK4XTHCjESOnz1VUXQoc6lgi2x4cI5aTQ201Mt8wHysI5fc05M5c81uZEtHcMKhxZ7iYEty1GfhLvGKpm+EYkdGxm1F5axmcB93DoORIbXfdN7f+hlFuyxtDP+sxtBnF43cIYwaZAWRgzxIoiXEMESoPlMhwLRDXeK772CAzXEdBRV7cmnoVBp0OSlyGidEzJTFq5hhcsA5388oSGM6b5p+qjpZrBlMS9xj4AwXmz108ukU1IomM3ceiW0CDwHCqp1NjAqXlFrbga+xuloQJ+tuyfbIBPNpqnmxqT7dPaOnZqBfhSBCteJAxWj58zLk2xgg+SPGYM6dRO6WczSnIxxwEExRaO+UyCUhbOp7CGQ+kxSUfNtLQFC+Po29vvy7jj4y0yAAAAABJRU5ErkJggg=="></span>Connection
+                    is secure</div>
+                <div class="core-msg spacer">Proceeding...</div>
+            </div>
+            <form id="challenge-form"
+                action="/search?q=2022&amp;__cf_chl_f_tk=1qWQAgl8.irfEoDb73Rb0pUm1SXbis3ZamDAIoTcPks-1659200251-0-gaNycGzNCFE"
+                method="POST" enctype="application/x-www-form-urlencoded">
+                <input type="hidden" name="md"
+                    value="OghUU_ltYW6I0fpWl7rE4yHBGBPHfpZQIKZRSEpJKjE-1659200251-0-AWB-KR-MabhObmvYa3mR5-xDk3qZVV73547wjnl-QtfPoTxe017AXt4WUskEcVzEIUKC7dsJoiy8ec1NA0fxdnI8X9OfPhtynl00ReWBVZc_3Gba_wigWMmM_9e8PX9vpVDcXpCRbz1BJ5_YLsba9TJM1sp14U9RtIce-tRBB53qoxLxJRz9QFmckEVBvsba4RfoycOvYPMMsfAqSkq13qtsA3Kd6RDB5Rb5-qF8674DsB4AMvd9xu_fBplQqKjOpEtrThCUtw8M2DHY8FUr_owUo1NIS1s6fSBEyHh6ehz9CidJ7zpRwYZFwgz_Pq9i8LmQG_AajozOJJhLp-tox0dptbUZnRNGt3hGQgrNu3jlCfwPC2XVp7xgLvmZoPYrzzrZoi_wErnIvVgyGCw9-sDPblPdvLBUz6uXNreWwThEW6PeRtMXnePO9UwcZmj_2awhwcVSHSLz1t1z22LtVsQ8xNpMbiE7xDvI2D5LNHAPIUC7Wp4AcehWD-fEm0w5jnVTWOmFlVRxtcnYZSMfDSaRUxsZ3hg5B1-ghVMEX6M-r_hAd6pLKNmjIfdl_Nvdm6veQvV-gTFaULbfuhmQQjYEb9G2IptDiNTZs5S7FtmjqVBAA7PmvwBTQwxw86J0cV3v_4pT1Oj8tigwiPny35HMTrKRmRZWaAZudCmWxDZkJIW8Eir7KQ57ba-u9cHh0A">
+                <input type="hidden" name="r"
+                    value="q.UUtPBFcFi4IkcVw3l4U_xJJKDIbHJj7xmuB43IIAI-1659200251-0-Aa4lU5RipD+d4of3hcdQ0rVmZ4ulb3siZYKwhm1jGNiA+/9b8IW1HL8k1GrsYEVexDW7ycP5UINQZ1sYJvZBTCQe3lhyGLHdLZ7KdI9RXKEbPx1NUOR/HthCD0Wbo7H41jbAf7l+HhH0zTLjm77/6NpJZHcgfsbBwwubl4R3oLarzPSByV2PVBnkuMyKCYgibriuMUt2iJHoMLx7Cr+Bmjx1KEFCrPYP0t7vgQs2APTylhL7ebP77XB9ndxU6Of3r4eHnTwLIcomFJ3+jqL6pzFaNoXdUBHrv9oZs/33KZjf2NB8cu5KUpAdM2lp3t5oTSQE19fJVroxmf91hcTdele3F2DAeawFGDwncm/Jo725SlyNk4TqsmR+il7DLkS/FTcCNzQe4cQM6DRWdmF9I1OohAl1/uGXYqUJSK1F45n3gec/pyPTQyZI0OLc7sCGYXfn3VPFsGATkg5mxE9rgZIB2b6ID9JggzIlDdYxlQRWecpruu07KOgk3m7g95lyHNZTohqemo4T8Z2MOZECjmXGMAuwvvk4d5sakVHr39kmAY6aSfXrRB+iONCOKkahbumrVmjLsnMvrpTb0DFE5pRAxwANPZKzb6Ikmlvxh7oJIPOB0mG9hDeoc/AJVlvZJV4CrpDLulNjHetAWXMwMptZuYJGEhcDXxmYj0ybntTCU4Y3JJQc5K+7ehSdnluTvMueWfs628854r4PcOONZzsO337j+3lUxrP5vDUCzYD25FNxvs8jGfqRivqHMOq2z9iOs0sHQTlHroLLSt2G7M50yRJBGTfxrIsvLq+ML3e/mRIkYIQxOcp8ugoPoT4c9gex3OyY0cnnA2/9OibQs9kevwf9DSnutMRRcbIXZI0XO6FY07+MykWqUcXygwMHs1vQxhaQ26NFYwolEWfOL7EQpp4GKyN30nL4nPNil/7GsXIr5SC+o55KI0l3AOEYE1jirVx2G0U7Br7SW80Ih4Fn5U/+4qFfW57GAJrpuk9qjFfJehe7wFBu5bHghEGRhKAu0wvpY7UTc9AiacMfP7ujVWi4DIbTCfOzOgVT8E0T6KaUurBppPJflLQE41c8n29ULyKmki9t8lIKvxYmv/3/AauhXFAExh+JrdnaeSDxhJYjWEUDJiaNvnkDCHMxPs/bePhSg4DYRMh4ngcOHCRkkRlDjipUUgeCrwNBY0qu2DIqLZXI1ZMwU+R0nuWnwc5xJuMHtrLkWbziP0FQcGaF0B6SaFIcOLnWG7YjJZFzxjFpvLb8GnZxk7i2YHCDTn0Stq3JDZHCkjJQjaPMmuK+5KYzfaSHcKOcaQbkbyDjn3t/XQX3a7lknngVchIJVsVn8osqgKvOx3aAdCicYKR6QaukrXHhR9uIEbPdoBYqZPKFz0uvVOShsUx2f65CaI8wWMjOBRWxTK1xUPNsetOiyYSvNwjeULaCXPKLi2qv/cZRRbsr3g5ghdHvNTpD/O0/xUgiziev3/9CpNopyr6VzLar9dJ/s++imXY1w1TCRJ2uCI2H70XGBGWxSZdbnfxU+j3zNCL0dBuabwhDd4ZnOZmlFZjGBiOUpsWdRrHd3c+QpwXdxB3QurRwX6J+LhmkqcWsPhP7LlMnN7dr2HUFZ5FS4LASl5AOf8hjCnO06FT8fWLl1eKVVjCugx9w54qjGqOV8A0v/PdWr7Ic0WfriyYbmwn/XnH8t0ri3bqDZsDkfhQMMF9JSWHEdoGD60a7McGDxr4g9s3LZhq5KozgSvyG+RUBPla8g2zB253hR7amWE5WO4IChl7AXmRB89F9u2+AoDIbefseb3pwG7GfkYpSBwmgJ4Ju4LAWoSfBhZSPMQadHZOCg36R11KesUy+NAy9bvD1bE3UMx9e2NbFohu6sXlilpnxINHp0sFEeulreEjWSQreri1eZeKxV2QfKIzWUiMoNdyT0JzM+/brYzddBpO2DrlnK5bEPWgtu0D7d4Kfm+0T7S//Fq+hxf40lSMPP8cBlan6sEd2iWmZ6gW3z43wNbJaPQIUDgb58ELxaEKQN4tOOy75/XXfISNnhG0K8M79a175WUb8v0A=">
+
+                <span style="display: none;"><span class="text-gray-600" data-translate="error">error code:
+                        1020</span></span>
+            </form>
+        </div>
+    </div>
+    <script>
+        (function () {
+            var trkjs = document.createElement('img');
+            trkjs.setAttribute('src', '/cdn-cgi/images/trace/captcha/js/transparent.gif?ray=732fa2449b567521');
+            trkjs.setAttribute('style', 'display: none');
+            document.body.appendChild(trkjs);
+            var cpo = document.createElement('script');
+            cpo.src = '/cdn-cgi/challenge-platform/h/g/orchestrate/managed/v1?ray=732fa2449b567521';
+            window._cf_chl_opt.cOgUHash = location.hash === '' && location.href.indexOf('#') !== -1 ? '#' : location.hash;
+            window._cf_chl_opt.cOgUQuery = location.search === '' && location.href.slice(0, -window._cf_chl_opt.cOgUHash.length).indexOf('?') !== -1 ? '?' : location.search;
+            if (window.history && window.history.replaceState) {
+                var ogU = location.pathname + window._cf_chl_opt.cOgUQuery + window._cf_chl_opt.cOgUHash;
+                history.replaceState(null, null, "\/search?q=2022&__cf_chl_rt_tk=1qWQAgl8.irfEoDb73Rb0pUm1SXbis3ZamDAIoTcPks-1659200251-0-gaNycGzNCFE" + window._cf_chl_opt.cOgUHash);
+                cpo.onload = function () {
+                    history.replaceState(null, null, ogU);
+                };
+            }
+            document.getElementsByTagName('head')[0].appendChild(cpo);
+        }());
+    </script><img src="/cdn-cgi/images/trace/captcha/js/transparent.gif?ray=732fa2449b567521" style="display: none">
+
+    <div class="footer" role="contentinfo">
+        <div class="footer-inner">
+            <div class="clearfix diagnostic-wrapper">
+                <div class="ray-id">Ray ID: <code>732fa2449b567521</code></div>
+            </div>
+            <div class="text-center">
+                Performance &amp; security by
+                <a rel="noopener noreferrer" href="https://www.cloudflare.com" target="_blank">Cloudflare</a>
+            </div>
+        </div>
+    </div>
+
+
+</body>
+
+</html>

install.js

@@ -0,0 +1,40 @@
+const fs = require('fs');
+const puppeteer = require('puppeteer');
+
+(async () => {
+
+  // Puppeteer does not allow to download Firefox revisions, just the last Nightly
+  // We this script we can download any version
+  const revision = '94.0a1';
+  const downloadHost = 'https://archive.mozilla.org/pub/firefox/nightly/2021/10/2021-10-01-09-33-23-mozilla-central';
+
+  // skip installation (for Dockerfile)
+  if (process.env.PUPPETEER_EXECUTABLE_PATH) {
+    console.log('Skipping Firefox installation because the environment variable "PUPPETEER_EXECUTABLE_PATH" is set.');
+    return;
+  }
+
+  // check if Firefox is already installed
+  const f = puppeteer.createBrowserFetcher({
+    product: 'firefox',
+    host: downloadHost
+  })
+  if (fs.existsSync(f._getFolderPath(revision))) {
+    console.log(`Firefox ${revision} already installed...`)
+    return;
+  }
+
+  console.log(`Installing firefox ${revision} ...`)
+  const downloadPath = f._downloadsFolder;
+  console.log(`Download path: ${downloadPath}`)
+  if (fs.existsSync(downloadPath)) {
+    console.log(`Removing previous downloads...`)
+    fs.rmSync(downloadPath, { recursive: true })
+  }
+
+  console.log(`Downloading firefox ${revision} ...`)
+  await f.download(revision)
+
+  console.log('Installation complete...')
+
+})()

jest.config.js

@@ -0,0 +1,12 @@
+module.exports = {
+  // A list of paths to directories that Jest should use to search for files in
+  roots: [
+    "./src/"
+  ],
+  // Compile Typescript
+  transform: {
+    '^.+\\.(ts|tsx)$': 'ts-jest'
+  },
+  // Default value for FlareSolverr maxTimeout is 60000
+  testTimeout: 70000
+}

package.json

@@ -1,59 +1,46 @@
 {
   "name": "flaresolverr",
-  "version": "1.2.3",
+  "version": "2.2.7",
   "description": "Proxy server to bypass Cloudflare protection.",
   "scripts": {
-    "start": "node ./dist/index.js",
+    "install": "node install.js",
+    "start": "tsc && node ./dist/server.js",
     "build": "tsc",
-    "dev": "nodemon -e ts --exec ts-node src/index.ts",
+    "dev": "nodemon -e ts --exec ts-node src/server.ts",
-    "package": "node build-binaries.js"
+    "package": "tsc && node build-binaries.js",
+    "test": "jest --runInBand"
   },
   "author": "Diego Heras (ngosang)",
-  "contributors": [
-    {
-      "name": "Noah Cardoza",
-      "url": "https://github.com/NoahCardoza/CloudProxy.git"
-    }
-  ],
   "license": "MIT",
   "repository": {
     "type": "git",
     "url": "https://github.com/ngosang/FlareSolverr"
   },
-  "pkg": {
-    "assets": [
-      "node_modules/puppeteer-extra-plugin-stealth/**/*.*"
-    ]
-  },
   "bin": {
-    "flaresolverr": "dist/index.js"
+    "flaresolverr": "dist/server.js"
   },
   "dependencies": {
     "await-timeout": "^1.1.1",
+    "body-parser": "^1.20.0",
     "console-log-level": "^1.4.1",
-    "got": "^11.5.1",
+    "express": "^4.18.1",
-    "hcaptcha-solver": "^1.0.2",
+    "puppeteer": "^13.7.0",
-    "puppeteer": "^3.3.0",
+    "uuid": "^8.3.2"
-    "puppeteer-extra": "^3.1.15",
-    "puppeteer-extra-plugin-stealth": "^2.6.5",
-    "uuid": "^8.2.0"
   },
   "devDependencies": {
     "@types/await-timeout": "^0.3.1",
-    "@types/node": "^14.0.23",
+    "@types/body-parser": "^1.19.2",
-    "@types/puppeteer": "^3.0.1",
+    "@types/express": "^4.17.13",
-    "@types/uuid": "^8.0.0",
+    "@types/jest": "^28.1.6",
-    "archiver": "^5.2.0",
+    "@types/node": "^18.6.2",
-    "eslint": "^7.5.0",
+    "@types/supertest": "^2.0.12",
-    "eslint-config-airbnb-base": "^14.2.0",
+    "@types/uuid": "^8.3.4",
-    "eslint-config-standard": "^14.1.1",
+    "archiver": "^5.3.1",
-    "eslint-plugin-import": "^2.22.0",
+    "nodemon": "^2.0.19",
-    "eslint-plugin-node": "^11.1.0",
+    "pkg": "^5.8.0",
-    "eslint-plugin-promise": "^4.2.1",
+    "supertest": "^6.2.4",
-    "eslint-plugin-standard": "^4.0.1",
+    "ts-jest": "^28.0.7",
-    "nodemon": "^2.0.4",
+    "ts-node": "^10.9.1",
-    "pkg": "^4.4.9",
+    "typescript": "^4.7.4"
-    "ts-node": "^8.10.2",
-    "typescript": "^3.9.7"
   }
 }

resources/flaresolverr_logo.svg

@@ -0,0 +1,180 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="256"
+   height="256"
+   viewBox="0 0 256 256"
+   id="svg2"
+   version="1.1"
+   inkscape:version="0.91 r13725"
+   sodipodi:docname="flaresolverr_logo.svg"
+   inkscape:export-filename="C:\Users\Diego\Desktop\flaresolverr_logo.png"
+   inkscape:export-xdpi="90"
+   inkscape:export-ydpi="90">
+  <defs
+     id="defs4" />
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="0.64"
+     inkscape:cx="-88.263072"
+     inkscape:cy="-93.571587"
+     inkscape:document-units="px"
+     inkscape:current-layer="layer1"
+     showgrid="false"
+     inkscape:window-width="2560"
+     inkscape:window-height="1377"
+     inkscape:window-x="-8"
+     inkscape:window-y="-8"
+     inkscape:window-maximized="1"
+     units="px"
+     showborder="true" />
+  <metadata
+     id="metadata7">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Capa 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(0,-796.36219)">
+    <g
+       id="g4177"
+       transform="matrix(0.51436047,0,0,0.59495735,-334.60687,650.43877)">
+      <g
+         id="g4141" />
+      <g
+         id="g4143" />
+      <g
+         id="g4145" />
+      <g
+         id="g4147" />
+      <g
+         id="g4149" />
+      <g
+         id="g4151" />
+      <g
+         id="g4153" />
+      <g
+         id="g4155" />
+      <g
+         id="g4157" />
+      <g
+         id="g4159" />
+      <g
+         id="g4161" />
+      <g
+         id="g4163" />
+      <g
+         id="g4165" />
+      <g
+         id="g4167" />
+      <g
+         id="g4169" />
+      <g
+         id="g4263"
+         transform="matrix(0.94954959,0,0,0.94954959,-111.49858,393.65111)">
+        <g
+           id="g4269" />
+        <g
+           id="g4342"
+           transform="translate(736.24631,-345.97247)">
+          <path
+             style="fill:#9dc6fb"
+             inkscape:connector-curvature="0"
+             d="m 584.32729,454.42324 c -0.995,-51.995 -44.49,-93.257 -96.488,-91.376 -7.616,0.273 -14.792,-3.862 -18.446,-10.55 -22.605,-41.376 -66.519,-69.441 -116.989,-69.441 -51.757,0 -96.596,29.528 -118.647,72.648 -6.423,12.56 -19.224,10.9 -24.689,10.9 -40.126,0 -74.199,25.852 -86.512,61.804 -2.046,5.973 -6.938,10.463 -12.894,12.556 -22.389998,7.87 -38.250998,29.605 -37.275998,54.902 1.163,30.174 26.849,53.631 57.044998,53.631 l 359.817,0 c 52.291,0 96.08,-42.793 95.079,-95.074 z"
+             id="path4285" />
+          <path
+             style="fill:#80b4fb"
+             inkscape:connector-curvature="0"
+             d="m 190.59629,495.86724 c -0.975,-25.298 14.885,-47.033 37.276,-54.902 5.956,-2.094 10.848,-6.584 12.894,-12.556 12.313,-35.952 46.385,-61.804 86.512,-61.804 5.465,0 18.265,1.66 24.688,-10.9 13.005,-25.43 33.94,-46.125 59.541,-58.832 -17.812,-8.834 -37.873,-13.816 -59.103,-13.816 -51.757,0 -96.596,29.528 -118.647,72.648 -6.423,12.56 -19.224,10.9 -24.689,10.9 -40.126,0 -74.199,25.852 -86.512,61.804 -2.046,5.973 -6.938,10.463 -12.894,12.556 -22.389998,7.87 -38.250998,29.605 -37.275998,54.902 1.163,30.174 26.849,53.63 57.044998,53.63 l 118.21,0 c -30.196,0 -55.881,-23.457 -57.045,-53.63 z"
+             id="path4287" />
+        </g>
+      </g>
+    </g>
+    <g
+       id="g4241"
+       transform="matrix(0.1453379,0,0,0.1453379,47.012211,854.83732)"
+       style="fill:#4d4d4d">
+      <g
+         id="g4197"
+         style="fill:#4d4d4d;fill-opacity:1">
+        <path
+           id="path4201"
+           d="m 867.699,356.238 -31.5,-26.6 c -9.699,-8.2 -24,-7.8 -33.199,0.9 l -17.4,16.3 c -14.699,-7.1 -30.299,-12.1 -46.4,-15 l -4.898,-24 c -2.5,-12.4 -14,-21 -26.602,-20 l -41.1,3.5 c -12.6,1.1 -22.5,11.4 -22.9,24.1 l -0.799,24.4 c -15.801,5.7 -30.701,13.5 -44.301,23.3 l -20.799,-13.8 c -10.602,-7 -24.701,-5 -32.9,4.7 l -26.6,31.7 c -8.201,9.7 -7.801,24 0.898,33.2 l 18.201,19.399 c -6.301,14.2 -10.801,29.101 -13.4,44.4 l -26,5.3 c -12.4,2.5 -21,14 -20,26.601 l 3.5,41.1 c 1.1,12.6 11.4,22.5 24.1,22.9 l 28.1,0.899 c 5.102,13.4 11.801,26.101 19.9,38 l -15.699,23.7 c -7,10.6 -5,24.7 4.699,32.9 l 31.5,26.6 c 9.701,8.2 24,7.8 33.201,-0.9 l 20.6,-19.3 c 13.5,6.3 27.699,11 42.299,13.8 l 5.701,28.2 c 2.5,12.4 14,21 26.6,20 l 41.1,-3.5 c 12.6,-1.1 22.5,-11.399 22.9,-24.1 l 0.9,-27.601 c 15,-5.3 29.199,-12.5 42.299,-21.399 l 22.701,15 c 10.6,7 24.699,5 32.9,-4.7 l 26.6,-31.5 c 8.199,-9.7 7.799,-24 -0.9,-33.2 L 872.7,592.138 c 6.701,-14.2 11.602,-29.2 14.4,-44.601 l 25,-5.1 c 12.4,-2.5 21,-14 20,-26.601 l -3.5,-41.1 c -1.1,-12.6 -11.4,-22.5 -24.1,-22.9 l -25.1,-0.8 c -5.201,-14.6 -12.201,-28.399 -20.9,-41.2 l 13.699,-20.6 c 7.201,-10.598 5.201,-24.798 -4.5,-32.998 z M 712.801,593.837 c -44.4,3.801 -83.602,-29.3 -87.301,-73.699 -3.801,-44.4 29.301,-83.601 73.699,-87.301 44.4,-3.8 83.602,29.301 87.301,73.7 3.801,44.401 -29.301,83.601 -73.699,87.3 z"
+           inkscape:connector-curvature="0"
+           style="fill:#4d4d4d;fill-opacity:1" />
+        <path
+           id="path4203"
+           d="m 205,704.438 c -12.6,1.3 -22.3,11.899 -22.4,24.6 l -0.3,25.3 c -0.2,12.7 9.2,23.5 21.8,25.101 l 18.6,2.399 c 3.1,11.301 7.5,22.101 13.2,32.301 l -12,14.8 c -8,9.899 -7.4,24.1 1.5,33.2 l 17.7,18.1 c 8.9,9.1 23.1,10.1 33.2,2.3 l 14.899,-11.5 c 10.5,6.2 21.601,11.101 33.2,14.5 l 2,19.2 c 1.3,12.6 11.9,22.3 24.6,22.4 l 25.301,0.3 c 12.699,0.2 23.5,-9.2 25.1,-21.8 l 2.3,-18.2 c 12.601,-3.101 24.601,-7.8 36,-14 l 14,11.3 c 9.9,8 24.101,7.4 33.201,-1.5 l 18.1,-17.7 c 9.1,-8.899 10.1,-23.1 2.301,-33.2 L 496.6,818.438 c 6.6,-11 11.701,-22.7 15.201,-35 l 16.6,-1.7 c 12.6,-1.3 22.299,-11.9 22.4,-24.6 l 0.299,-25.301 c 0.201,-12.699 -9.199,-23.5 -21.799,-25.1 l -16.201,-2.1 c -3.1,-12.2 -7.699,-24 -13.699,-35 l 10.1,-12.4 c 8,-9.9 7.4,-24.1 -1.5,-33.2 l -17.699,-18.1 c -8.9,-9.101 -23.102,-10.101 -33.201,-2.3 l -12.101,9.3 c -11.399,-6.9 -23.6,-12.2 -36.399,-15.8 L 407,581.437 c -1.3,-12.601 -11.899,-22.3 -24.6,-22.4 l -25.3,-0.3 c -12.7,-0.2 -23.5,9.2 -25.101,21.8 l -2,15.601 c -13.199,3.399 -25.899,8.6 -37.699,15.399 l -12.5,-10.2 c -9.9,-8 -24.101,-7.399 -33.201,1.5 l -18.2,17.801 c -9.1,8.899 -10.1,23.1 -2.3,33.199 l 10.7,13.801 c -6.2,11 -11.1,22.699 -14.3,35 l -17.499,1.8 z m 163.3,-28.601 c 36.3,0.4 65.399,30.301 65,66.601 -0.4,36.3 -30.301,65.399 -66.601,65 -36.3,-0.4 -65.399,-30.3 -65,-66.601 0.401,-36.299 30.301,-65.399 66.601,-65 z"
+           inkscape:connector-curvature="0"
+           style="fill:#4d4d4d;fill-opacity:1" />
+      </g>
+      <g
+         id="g4205"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4207"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4209"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4211"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4213"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4215"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4217"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4219"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4221"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4223"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4225"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4227"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4229"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4231"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4233"
+         style="fill:#4d4d4d" />
+    </g>
+  </g>
+</svg>

src/app.ts

@@ -0,0 +1,83 @@
+import log from './services/log'
+import {NextFunction, Request, Response} from 'express';
+import {getUserAgent} from "./services/sessions";
+import {controllerV1} from "./controllers/v1";
+
+const express = require('express');
+const app = express();
+const bodyParser = require('body-parser');
+const version: string = 'v' + require('../package.json').version
+
+// Convert request objects to JSON
+app.use(bodyParser.json({
+    limit: '50mb',
+    verify(req: Request, res: Response, buf: any) {
+        req.body = buf;
+    }
+}));
+
+// Access log
+app.use(function(req: Request, res: Response, next: NextFunction) {
+    if (req.url != '/health') {
+        // count the request for the log prefix
+        log.incRequests()
+        // build access message
+        let body = "";
+        if (req.method == 'POST' && req.body) {
+            body += " body: "
+            try {
+                body += JSON.stringify(req.body)
+            } catch(e) {
+                body += req.body
+            }
+        }
+        log.info(`Incoming request => ${req.method} ${req.url}${body}`);
+    }
+    next();
+});
+
+// *********************************************************************************************************************
+// Routes
+
+// Show welcome message
+app.get("/", ( req: Request, res: Response ) => {
+    res.send({
+        "msg": "FlareSolverr is ready!",
+        "version": version,
+        "userAgent": getUserAgent()
+    });
+});
+
+// Health endpoint. this endpoint is special because it doesn't print traces
+app.get("/health", ( req: Request, res: Response ) => {
+    res.send({
+        "status": "ok"
+    });
+});
+
+// Controller v1
+app.post("/v1", async( req: Request, res: Response ) => {
+    await controllerV1(req, res);
+});
+
+// *********************************************************************************************************************
+
+// Unknown paths or verbs
+app.use(function (req : Request, res : Response) {
+    res.status(404)
+        .send({"error": "Unknown resource or HTTP verb"})
+})
+
+// Errors
+app.use(function (err: any, req: Request, res: Response, next: NextFunction) {
+    if (err) {
+        let msg = 'Invalid request: ' + err;
+        msg = msg.replace("\n", "").replace("\r", "")
+        log.error(msg)
+        res.send({"error": msg})
+    } else {
+        next()
+    }
+})
+
+module.exports = app;

src/captcha/harvester.ts

@@ -1,31 +0,0 @@
-import got from 'got'
-import { sleep } from '../utils'
-
-/*
-    This method uses the captcha-harvester project:
-        https://github.com/NoahCardoza/CaptchaHarvester
-
-    While the function must take url/sitekey/type args,
-    they aren't used because the harvester server must
-    be preconfigured.
-
-    ENV:
-        HARVESTER_ENDPOINT: This must be the full path
-        to the /token endpoint of the harvester.
-        E.G. "https://127.0.0.1:5000/token"
-*/
-
-export default async function solve(): Promise<string> {
-  const endpoint = process.env.HARVESTER_ENDPOINT
-  if (!endpoint) { throw Error('ENV variable `HARVESTER_ENDPOINT` must be set.') }
-  while (true) {
-    try {
-      return (await got.get(process.env.HARVESTER_ENDPOINT, {
-        https: { rejectUnauthorized: false }
-      })).body
-    } catch (e) {
-      if (e.response.statusCode !== 418) { throw e }
-    }
-    await sleep(3000)
-  }
-}

src/captcha/hcaptcha-solver.ts

@@ -1,21 +0,0 @@
-const solveCaptcha = require('hcaptcha-solver');
-import { SolverOptions } from '.'
-/*
-    This method uses the hcaptcha-solver project:
-        https://github.com/JimmyLaurent/hcaptcha-solver
-
-    TODO: allow user pass custom options to the solver.
-
-    ENV:
-        There are no other variables that must be set to get this to work
-*/
-
-export default async function solve({ url }: SolverOptions): Promise<string> {
-  try {
-    const token = await solveCaptcha(url)
-    return token
-  } catch (e) {
-    console.error(e)
-    return null
-  }
-}

src/captcha/index.ts

@@ -1,4 +1,4 @@
-import log from "../log";
+import log from "../services/log";
 
 export enum CaptchaType {
   re = 'reCaptcha',
@@ -30,12 +30,12 @@ export default (): Solver => {
         throw Error(`The solver '${method}' is not a valid captcha solving method.`)
       } else {
         console.error(e)
-        throw Error(`An error occured loading the solver '${method}'.`)
+        throw Error(`An error occurred loading the solver '${method}'.`)
       }
     }
   }
 
-  log.info(`Using '${method} to solve the captcha.`);
+  log.info(`Using '${method}' to solve the captcha.`);
 
   return captchaSolvers[method]
 }

src/controllers/v1.ts

@@ -0,0 +1,178 @@
+import {Request, Response} from 'express';
+import {Protocol} from "devtools-protocol";
+
+import log from '../services/log'
+import {browserRequest, ChallengeResolutionResultT, ChallengeResolutionT} from "../services/solver";
+import {SessionCreateOptions} from "../services/sessions";
+const sessions = require('../services/sessions')
+const version: string = 'v' + require('../../package.json').version
+
+interface V1Routes {
+  [key: string]: (params: V1RequestBase, response: V1ResponseBase) => Promise<void>
+}
+
+export interface Proxy {
+  url?: string
+  username?: string
+  password?: string
+}
+
+export interface V1RequestBase {
+  cmd: string
+  cookies?: Protocol.Network.CookieParam[],
+  maxTimeout?: number
+  proxy?: Proxy
+  session: string
+  headers?: Record<string, string> // deprecated v2, not used
+  userAgent?: string // deprecated v2, not used
+}
+
+interface V1RequestSession extends V1RequestBase {
+}
+
+export interface V1Request extends V1RequestBase {
+  url: string
+  method?: string
+  postData?: string
+  returnOnlyCookies?: boolean
+  download?: boolean // deprecated v2, not used
+  returnRawHtml?: boolean // deprecated v2, not used
+}
+
+export interface V1ResponseBase {
+  status: string
+  message: string
+  startTimestamp: number
+  endTimestamp: number
+  version: string
+}
+
+export interface V1ResponseSolution extends V1ResponseBase {
+  solution: ChallengeResolutionResultT
+}
+
+export interface V1ResponseSession extends V1ResponseBase {
+  session: string
+}
+
+export interface V1ResponseSessions extends V1ResponseBase {
+  sessions: string[]
+}
+
+export const routes: V1Routes = {
+  'sessions.create': async (params: V1RequestSession, response: V1ResponseSession): Promise<void> => {
+    const options: SessionCreateOptions = {
+      oneTimeSession: false,
+      cookies: params.cookies,
+      maxTimeout: params.maxTimeout,
+      proxy: params.proxy
+    }
+    const { sessionId, browser } = await sessions.create(params.session, options)
+    if (browser) {
+      response.status = "ok";
+      response.message = "Session created successfully.";
+      response.session = sessionId
+    } else {
+      throw Error('Error creating session.')
+    }
+  },
+  'sessions.list': async (params: V1RequestSession, response: V1ResponseSessions): Promise<void> => {
+    response.status = "ok";
+    response.message = "";
+    response.sessions = sessions.list();
+  },
+  'sessions.destroy': async (params: V1RequestSession, response: V1ResponseBase): Promise<void> => {
+    if (await sessions.destroy(params.session)) {
+      response.status = "ok";
+      response.message = "The session has been removed.";
+    } else {
+      throw Error('This session does not exist.')
+    }
+  },
+  'request.get': async (params: V1Request, response: V1ResponseSolution): Promise<void> => {
+    params.method = 'GET'
+    if (params.postData) {
+      throw Error('Cannot use "postBody" when sending a GET request.')
+    }
+    if (params.returnRawHtml) {
+      log.warn("Request parameter 'returnRawHtml' was removed in FlareSolverr v2.")
+    }
+    if (params.download) {
+      log.warn("Request parameter 'download' was removed in FlareSolverr v2.")
+    }
+    const result: ChallengeResolutionT = await browserRequest(params)
+
+    response.status = result.status;
+    response.message = result.message;
+    response.solution = result.result;
+    if (response.message) {
+      log.info(response.message)
+    }
+  },
+  'request.post': async (params: V1Request, response: V1ResponseSolution): Promise<void> => {
+    params.method = 'POST'
+    if (!params.postData) {
+      throw Error('Must send param "postBody" when sending a POST request.')
+    }
+    if (params.returnRawHtml) {
+      log.warn("Request parameter 'returnRawHtml' was removed in FlareSolverr v2.")
+    }
+    if (params.download) {
+      log.warn("Request parameter 'download' was removed in FlareSolverr v2.")
+    }
+    const result: ChallengeResolutionT = await browserRequest(params)
+
+    response.status = result.status;
+    response.message = result.message;
+    response.solution = result.result;
+    if (response.message) {
+      log.info(response.message)
+    }
+  },
+}
+
+export async function controllerV1(req: Request, res: Response): Promise<void> {
+  const response: V1ResponseBase = {
+    status: null,
+    message: null,
+    startTimestamp: Date.now(),
+    endTimestamp: 0,
+    version: version
+  }
+
+  try {
+    const params: V1RequestBase = req.body
+    // do some validations
+    if (!params.cmd) {
+      throw Error("Request parameter 'cmd' is mandatory.")
+    }
+    if (params.headers) {
+      log.warn("Request parameter 'headers' was removed in FlareSolverr v2.")
+    }
+    if (params.userAgent) {
+      log.warn("Request parameter 'userAgent' was removed in FlareSolverr v2.")
+    }
+
+    // set default values
+    if (!params.maxTimeout || params.maxTimeout < 1) {
+      params.maxTimeout = 60000;
+    }
+
+    // execute the command
+    const route = routes[params.cmd]
+    if (route) {
+      await route(params, response)
+    } else {
+      throw Error(`The command '${params.cmd}' is invalid.`)
+    }
+  } catch (e) {
+    res.status(500)
+    response.status = "error";
+    response.message = e.toString();
+    log.error(response.message)
+  }
+
+  response.endTimestamp = Date.now()
+  log.info(`Response in ${(response.endTimestamp - response.startTimestamp) / 1000} s`)
+  res.send(response)
+}

src/index.ts

@@ -1,115 +0,0 @@
-import log from './log'
-import { createServer, IncomingMessage, ServerResponse } from 'http';
-import { RequestContext } from './types'
-import Router, { BaseAPICall } from './routes'
-
-const version: string = "v" + require('../package.json').version
-const serverPort: number = Number(process.env.PORT) || 8191
-const serverHost: string = process.env.HOST || '0.0.0.0'
-
-
-function errorResponse(errorMsg: string, res: ServerResponse, startTimestamp: number) {
-  log.error(errorMsg)
-  const response = {
-    status: 'error',
-    message: errorMsg,
-    startTimestamp,
-    endTimestamp: Date.now(),
-    version
-  }
-  res.writeHead(500, {
-    'Content-Type': 'application/json'
-  })
-  res.write(JSON.stringify(response))
-  res.end()
-}
-
-function successResponse(successMsg: string, extendedProperties: object, res: ServerResponse, startTimestamp: number) {
-  const endTimestamp = Date.now()
-  log.info(`Successful response in ${(endTimestamp - startTimestamp) / 1000} s`)
-  if (successMsg) { log.info(successMsg) }
-
-  const response = Object.assign({
-    status: 'ok',
-    message: successMsg || '',
-    startTimestamp,
-    endTimestamp,
-    version
-  }, extendedProperties || {})
-  res.writeHead(200, {
-    'Content-Type': 'application/json'
-  })
-  res.write(JSON.stringify(response))
-  res.end()
-}
-
-function validateIncomingRequest(ctx: RequestContext, params: BaseAPICall) {
-  log.info(`Params: ${JSON.stringify(params)}`)
-
-  if (ctx.req.method !== 'POST') {
-    ctx.errorResponse('Only the POST method is allowed')
-    return false
-  }
-
-  if (ctx.req.url !== '/v1') {
-    ctx.errorResponse('Only /v1 endpoint is allowed')
-    return false
-  }
-
-  if (!params.cmd) {
-    ctx.errorResponse("Parameter 'cmd' is mandatory")
-    return false
-  }
-
-  return true
-}
-
-createServer((req: IncomingMessage, res: ServerResponse) => {
-  const startTimestamp = Date.now()
-
-  // count the request for the log prefix
-  log.incRequests()
-  log.info(`Incoming request: ${req.method} ${req.url}`)
-
-  // show welcome message
-  if (req.url == '/') {
-    successResponse("FlareSolverr is ready!", null, res, startTimestamp);
-    return;
-  }
-
-  // get request body
-  const bodyParts: any[] = []
-  req.on('data', chunk => {
-    bodyParts.push(chunk)
-  }).on('end', () => {
-    // parse params
-    const body = Buffer.concat(bodyParts).toString()
-    let params: BaseAPICall = null
-    try {
-      params = JSON.parse(body)
-    } catch (err) {
-      errorResponse('Body must be in JSON format', res, startTimestamp)
-      return
-    }
-
-    const ctx: RequestContext = {
-      req,
-      res,
-      startTimestamp,
-      errorResponse: (msg) => errorResponse(msg, res, startTimestamp),
-      successResponse: (msg, extendedProperties) => successResponse(msg, extendedProperties, res, startTimestamp)
-    }
-
-    // validate params
-    if (!validateIncomingRequest(ctx, params)) { return }
-
-    // process request
-    Router(ctx, params).catch(e => {
-      console.error(e)
-      ctx.errorResponse(e.message)
-    })
-  })
-}).listen(serverPort, serverHost, () => {
-  log.info(`FlareSolverr ${version} listening on http://${serverHost}:${serverPort}`);
-  log.debug('Debug log enabled');
-})

src/log.ts

@@ -1,19 +0,0 @@
-let requests = 0
-
-const LOG_HTML: boolean = Boolean(process.env.LOG_HTML) || false
-
-export default {
-  incRequests: () => { requests++ },
-  html(html: string) {
-    if (LOG_HTML)
-      this.debug(html)
-  },
-  ...require('console-log-level')(
-    {
-      level: process.env.LOG_LEVEL || 'info',
-      prefix(level: string) {
-        return `${new Date().toISOString()} ${level.toUpperCase()} REQ-${requests}`
-      }
-    }
-  )
-}

src/providers/cloudflare.ts

@@ -1,164 +1,163 @@
-import {Response} from 'puppeteer'
+import {Page, HTTPResponse} from 'puppeteer'
-import {Page} from "puppeteer-extra/dist/puppeteer";
 
-import log from "../log";
+import log from "../services/log";
-import getCaptchaSolver, {CaptchaType} from "../captcha";
 
 /**
  *  This class contains the logic to solve protections provided by CloudFlare
-**/
+ **/
 
-const CHALLENGE_SELECTORS = ['#trk_jschal_js', '.ray_id', '.attack-box'];
+const BAN_SELECTORS: string[] = [];
-const TOKEN_INPUT_NAMES = ['g-recaptcha-response', 'h-captcha-response'];
+const CHALLENGE_SELECTORS: string[] = [
+    // todo: deprecate  '#trk_jschal_js', '#cf-please-wait'
+    '#cf-challenge-running', '#trk_jschal_js', '#cf-please-wait', // CloudFlare
+    '#link-ddg', // DDoS-GUARD
+    'td.info #js_info' // Custom CloudFlare for EbookParadijs, Film-Paleis, MuziekFabriek and Puur-Hollands
+];
+const CAPTCHA_SELECTORS: string[] = [
+    // todo: deprecate 'input[name="cf_captcha_kind"]'
+    '#cf-challenge-hcaptcha-wrapper', '#cf-norobot-container', 'input[name="cf_captcha_kind"]'
+];
 
-export default async function resolveChallenge(url: string, page: Page, response: Response): Promise<Response> {
+const delay = (ms: number) => new Promise(resolve => setTimeout(resolve, ms))
+
+export default async function resolveChallenge(url: string, page: Page, response: HTTPResponse): Promise<HTTPResponse> {
 
   // look for challenge and return fast if not detected
-  if (!response.headers().server.startsWith('cloudflare')) {
+  let cfDetected = response.headers().server &&
+      (response.headers().server.startsWith('cloudflare') || response.headers().server.startsWith('ddos-guard'));
+  if (cfDetected) {
+    if (response.status() == 403 || response.status() == 503) {
+      cfDetected = true; // Defected CloudFlare and DDoS-GUARD
+    } else if (response.headers().vary && response.headers().vary.trim() == 'Accept-Encoding,User-Agent' &&
+        response.headers()['content-encoding'] && response.headers()['content-encoding'].trim() == 'br') {
+      cfDetected = true; // Detected Custom CloudFlare for EbookParadijs, Film-Paleis, MuziekFabriek and Puur-Hollands
+    } else {
+      cfDetected = false;
+    }
+  }
+
+  if (cfDetected) {
+    log.info('Cloudflare detected');
+  } else {
     log.info('Cloudflare not detected');
     return response;
   }
-  log.info('Cloudflare detected');
 
-  if (await page.$('.cf-error-code')) {
+  if (await findAnySelector(page, BAN_SELECTORS)) {
-    throw new Error('Cloudflare has blocked this request (Code 1020 Detected).')
+    throw new Error('Cloudflare has blocked this request. Probably your IP is banned for this site, check in your web browser.');
   }
 
-  let selectorFoundCount = 0;
+  // find Cloudflare selectors
-  if (response.status() > 400) {
+  let selectorFound = false;
-    // detect cloudflare wait 5s
+  let selector: string = await findAnySelector(page, CHALLENGE_SELECTORS)
-    for (const selector of CHALLENGE_SELECTORS) {
+  if (selector) {
-      const cfChallengeElem = await page.$(selector)
+    selectorFound = true;
-      if (cfChallengeElem) {
+    log.debug(`Javascript challenge element '${selector}' detected.`)
-        selectorFoundCount++
+    log.debug('Waiting for Cloudflare challenge...')
-        log.debug(`Javascript challenge element '${selector}' detected.`)
-        log.debug('Waiting for Cloudflare challenge...')
 
-        while (true) {
+    while (true) {
-          await page.waitFor(1000)
+      try {
-          try {
-            // catch exception timeout in waitForNavigation
-            response = await page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 5000 })
-          } catch (error) { }
 
-          try {
+        selector = await findAnySelector(page, CHALLENGE_SELECTORS)
-            // catch Execution context was destroyed
+        if (!selector) {
-            const cfChallengeElem = await page.$(selector)
+          // solved!
-            if (!cfChallengeElem) { break }
+          log.debug('Challenge element not found')
-            log.debug('Found challenge element again...')
+          break
-          } catch (error)
-          { }
 
-          response = await page.reload({ waitUntil: 'domcontentloaded' })
+        } else {
-          log.debug('Page reloaded.')
+          log.debug(`Javascript challenge element '${selector}' detected.`)
-          log.html(await page.content())
+
+          // check for CAPTCHA challenge
+          if (await findAnySelector(page, CAPTCHA_SELECTORS)) {
+            // captcha detected
+            break
+          }
+
+          // new Cloudflare Challenge #cf-please-wait
+          const displayStyle = await page.evaluate((selector) => {
+            return getComputedStyle(document.querySelector(selector)).getPropertyValue("display");
+          }, selector);
+          if (displayStyle == "none") {
+            // spinner is hidden, could be a captcha or not
+            log.debug('Challenge element is hidden')
+            log.debug("Waiting for 30 secs");
+            await delay(30 * 1000);
+            // wait until redirecting disappears
+            while (true) {
+              try {
+                await page.waitForTimeout(1000)
+                const displayStyle2 = await page.evaluate(() => {
+                  return getComputedStyle(document.querySelector('#cf-spinner-redirecting')).getPropertyValue("display");
+                });
+                if (displayStyle2 == "none") {
+                  break // hCaptcha detected
+                }
+              } catch (error) {
+                break // redirection completed
+              }
+            }
+            break
+          } else {
+            log.debug('Challenge element is visible')
+          }
         }
+        log.debug('Found challenge element again')
 
-        log.debug('Validating HTML code...')
+      } catch (error)
-        break
+      {
-      } else {
+        log.debug("Unexpected error: " + error);
-        log.debug(`No '${selector}' challenge element detected.`)
+        if (!error.toString().includes("Execution context was destroyed")) {
+          break
+        }
       }
+
+      log.debug('Waiting for Cloudflare challenge...')
+      await page.waitForTimeout(1000)
     }
-    log.debug("Javascript challenge selectors found: " + selectorFoundCount + ", total selectors: " + CHALLENGE_SELECTORS.length)
+
+    log.debug('Validating HTML code...')
   } else {
-    // some sites use cloudflare but there is no challenge
+    log.debug(`No challenge element detected.`)
-    log.debug(`Javascript challenge not detected. Status code: ${response.status()}`);
-    selectorFoundCount = 1;
   }
 
-  // it seems some captcha pages return 200 sometimes
+  // check for CAPTCHA challenge
-  if (await page.$('input[name="cf_captcha_kind"]')) {
+  if (await findAnySelector(page, CAPTCHA_SELECTORS)) {
-    log.info('Captcha challenge detected.');
+    log.info('CAPTCHA challenge detected');
-    const captchaSolver = getCaptchaSolver()
+    throw new Error('FlareSolverr can not resolve CAPTCHA challenges. Since the captcha doesn\'t always appear, you may have better luck with the next request.');
-    if (captchaSolver) {
-      const captchaStartTimestamp = Date.now()
-      const challengeForm = await page.$('#challenge-form')
-      if (challengeForm) {
-        const captchaTypeElm = await page.$('input[name="cf_captcha_kind"]')
-        const cfCaptchaType: string = await captchaTypeElm.evaluate((e: any) => e.value)
-        const captchaType: CaptchaType = (CaptchaType as any)[cfCaptchaType]
-        if (!captchaType) {
-          throw new Error('Unknown captcha type!');
-        }
 
-        let sitekey = null
+    // const captchaSolver = getCaptchaSolver()
-        if (captchaType != 'hCaptcha' && process.env.CAPTCHA_SOLVER != 'hcaptcha-solver') {
+    // if (captchaSolver) {
-          const sitekeyElem = await page.$('*[data-sitekey]')
+    //     // to-do: get the params
-          if (!sitekeyElem) {
+    //     log.info('Waiting to receive captcha token to bypass challenge...')
-            throw new Error('Could not find sitekey!');
+    //     const token = await captchaSolver({
-          }
+    //       url,
-          sitekey = await sitekeyElem.evaluate((e) => e.getAttribute('data-sitekey'))
+    //       sitekey,
-        }
+    //       type: captchaType
-
+    //     })
-        log.info('Waiting to receive captcha token to bypass challenge...')
+    //     log.debug(`Token received: ${token}`);
-        const token = await captchaSolver({
+    //     // to-do: send the token
-          url,
+    //   }
-          sitekey,
+    // } else {
-          type: captchaType
+    //   throw new Error('Captcha detected but no automatic solver is configured.');
-        })
+    // }
-        log.debug(`Token received: ${token}`);
-        if (!token) {
-          throw new Error('Token solver failed to return a token.')
-        }
-
-        let responseFieldsFoundCount = 0;
-        for (const name of TOKEN_INPUT_NAMES) {
-          const input = await page.$(`textarea[name="${name}"]`)
-          if (input) {
-            responseFieldsFoundCount ++;
-            log.debug(`Challenge response field '${name}' found in challenge form.`);
-            await input.evaluate((e: HTMLTextAreaElement, token) => { e.value = token }, token);
-          }
-        }
-        if (responseFieldsFoundCount == 0) {
-          throw new Error('Challenge response field not found in challenge form.');
-        }
-
-        // ignore preset event listeners on the form
-        await page.evaluate(() => {
-          window.addEventListener('submit', (e) => { event.stopPropagation() }, true)
-        })
-
-        // it seems some sites obfuscate their challenge forms
-        // TODO: look into how they do it and come up with a more solid solution
-        try {
-          // this element is added with js and we want to wait for all the js to load before submitting
-          await page.waitForSelector('#challenge-form', { timeout: 10000 })
-        } catch (err) {
-          throw new Error("No '#challenge-form' element detected.");
-        }
-
-        // calculates the time it took to solve the captcha
-        const captchaSolveTotalTime = Date.now() - captchaStartTimestamp
-
-        // generates a random wait time
-        const randomWaitTime = (Math.floor(Math.random() * 10) + 10) * 1000
-
-        // waits, if any, time remaining to appear human but stay as fast as possible
-        const timeLeft = randomWaitTime - captchaSolveTotalTime
-        if (timeLeft > 0) {
-          log.debug(`Waiting for '${timeLeft}' milliseconds.`);
-          await page.waitFor(timeLeft);
-        }
-
-        // submit captcha response
-        challengeForm.evaluate((e: HTMLFormElement) => e.submit())
-        response = await page.waitForNavigation({ waitUntil: 'domcontentloaded' })
-
-        if (await page.$('input[name="cf_captcha_kind"]')) {
-          throw new Error('Captcha service failed to solve the challenge.');
-        }
-      }
-    } else {
-      throw new Error('Captcha detected but no automatic solver is configured.');
-    }
   } else {
-    if (selectorFoundCount == 0)
+    if (!selectorFound)
     {
-      throw new Error('No challenge selectors found, unable to proceed')
+      throw new Error('No challenge selectors found, unable to proceed.')
+    } else {
+      log.info('Challenge solved');
     }
   }
 
   return response;
 }
+
+async function findAnySelector(page: Page, selectors: string[]) {
+  for (const selector of selectors) {
+    const cfChallengeElem = await page.$(selector)
+    if (cfChallengeElem) {
+      return selector;
+    }
+  }
+  return null;
+}

src/routes.ts

@@ -1,295 +0,0 @@
-import { v1 as UUIDv1 } from 'uuid'
-import { SetCookie, Request, Response, Headers, HttpMethod, Overrides } from 'puppeteer'
-import { Page, Browser } from "puppeteer-extra/dist/puppeteer";
-const Timeout = require('await-timeout');
-
-import log from './log'
-import sessions, { SessionsCacheItem } from './session'
-import { RequestContext } from './types'
-import cloudflareProvider from './providers/cloudflare';
-
-export interface BaseAPICall {
-  cmd: string
-}
-
-interface BaseSessionsAPICall extends BaseAPICall {
-  session?: string
-}
-
-interface SessionsCreateAPICall extends BaseSessionsAPICall {
-  userAgent?: string,
-  cookies?: SetCookie[],
-  headers?: Headers
-  maxTimeout?: number
-  proxy?: any
-}
-
-interface BaseRequestAPICall extends BaseAPICall {
-  url: string
-  method?: HttpMethod
-  postData?: string
-  session?: string
-  userAgent?: string
-  maxTimeout?: number
-  cookies?: SetCookie[],
-  headers?: Headers
-  proxy?: any, // TODO: use interface not any
-  download?: boolean
-  returnOnlyCookies?: boolean
-}
-
-
-interface Routes {
-  [key: string]: (ctx: RequestContext, params: BaseAPICall) => void | Promise<void>
-}
-
-interface ChallengeResolutionResultT {
-  url: string
-  status: number,
-  headers?: Headers,
-  response: string,
-  cookies: object[]
-  userAgent: string
-}
-
-interface ChallengeResolutionT {
-  status?: string
-  message: string
-  result: ChallengeResolutionResultT
-}
-
-interface OverrideResolvers {
-  method?: (request: Request) => HttpMethod,
-  postData?: (request: Request) => string,
-  headers?: (request: Request) => Headers
-}
-
-type OverridesProps =
-  'method' |
-  'postData' |
-  'headers'
-
-// We always set a Windows User-Agent because ARM builds are detected by Cloudflare
-const DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36"
-
-async function resolveChallengeWithTimeout(ctx: RequestContext, params: BaseRequestAPICall, page: Page) {
-  const maxTimeout = params.maxTimeout || 60000
-  const timer = new Timeout();
-  try {
-    const promise = resolveChallenge(ctx, params, page);
-    return await Promise.race([
-      promise,
-      timer.set(maxTimeout, `Maximum timeout reached. maxTimeout=${maxTimeout} (ms)`)
-    ]);
-  } finally {
-    timer.clear();
-  }
-}
-
-async function resolveChallenge(ctx: RequestContext, { url, proxy, download, returnOnlyCookies }: BaseRequestAPICall, page: Page): Promise<ChallengeResolutionT | void> {
-
-  let status = 'ok'
-  let message = ''
-
-  if (proxy) {
-    log.debug("Apply proxy");
-    if (proxy.username)
-      await page.authenticate({ username: proxy.username, password: proxy.password });
-  }
-
-  log.debug(`Navigating to... ${url}`)
-  let response: Response = await page.goto(url, { waitUntil: 'domcontentloaded' })
-  log.html(await page.content())
-
-  // Detect protection services and solve challenges
-  try {
-    response = await cloudflareProvider(url, page, response);
-  } catch (e) {
-    status = "error";
-    message = "Cloudflare " + e.toString();
-  }
-
-  const payload: ChallengeResolutionT = {
-    status,
-    message,
-    result: {
-      url: page.url(),
-      status: response.status(),
-      headers: response.headers(),
-      response: null,
-      cookies: await page.cookies(),
-      userAgent: await page.evaluate(() => navigator.userAgent)
-    }
-  }
-
-  if (returnOnlyCookies) {
-    payload.result.headers = null;
-    payload.result.userAgent = null;
-  } else {
-    if (download) {
-      // for some reason we get an error unless we reload the page
-      // has something to do with a stale buffer and this is the quickest
-      // fix since I am short on time
-      response = await page.goto(url, { waitUntil: 'domcontentloaded' })
-      payload.result.response = (await response.buffer()).toString('base64')
-    } else {
-      payload.result.response = await page.content()
-    }
-  }
-
-  // make sure the page is closed because if it isn't and error will be thrown
-  // when a user uses a temporary session, the browser make be quit before
-  // the page is properly closed.
-  await page.close()
-
-  return payload
-}
-
-function mergeSessionWithParams({ defaults }: SessionsCacheItem, params: BaseRequestAPICall): BaseRequestAPICall {
-  const copy = { ...defaults, ...params }
-
-  // custom merging logic
-  copy.headers = { ...defaults.headers || {}, ...params.headers || {} } || null
-
-  return copy
-}
-
-async function setupPage(ctx: RequestContext, params: BaseRequestAPICall, browser: Browser): Promise<Page> {
-  const page = await browser.newPage()
-
-  // merge session defaults with params
-  const { method, postData, userAgent, headers, cookies } = params
-
-  let overrideResolvers: OverrideResolvers = {}
-
-  if (method !== 'GET') {
-    log.debug(`Setting method to ${method}`)
-    overrideResolvers.method = request => method
-  }
-
-  if (postData) {
-    log.debug(`Setting body data to ${postData}`)
-    overrideResolvers.postData = request => postData
-  }
-
-  if (userAgent) {
-    log.debug(`Using custom UA: ${userAgent}`)
-    await page.setUserAgent(userAgent)
-  } else {
-    await page.setUserAgent(DEFAULT_USER_AGENT)
-  }
-
-  if (headers) {
-    log.debug(`Adding custom headers: ${JSON.stringify(headers, null, 2)}`,)
-    overrideResolvers.headers = request => Object.assign(request.headers(), headers)
-  }
-
-  if (cookies) {
-    log.debug(`Setting custom cookies: ${JSON.stringify(cookies, null, 2)}`,)
-    await page.setCookie(...cookies)
-  }
-
-  // if any keys have been set on the object
-  if (Object.keys(overrideResolvers).length > 0) {
-    log.debug(overrideResolvers)
-    let callbackRunOnce = false
-    const callback = (request: Request) => {
-
-      if (callbackRunOnce || !request.isNavigationRequest()) {
-        request.continue()
-        return
-      }
-
-      callbackRunOnce = true
-      const overrides: Overrides = {}
-
-      Object.keys(overrideResolvers).forEach((key: OverridesProps) => {
-        // @ts-ignore
-        overrides[key] = overrideResolvers[key](request)
-      });
-
-      log.debug(overrides)
-
-      request.continue(overrides)
-    }
-
-    await page.setRequestInterception(true)
-    page.on('request', callback)
-  }
-
-  return page
-}
-
-const browserRequest = async (ctx: RequestContext, params: BaseRequestAPICall) => {
-  const oneTimeSession = params.session === undefined
-  const sessionId = params.session || UUIDv1()
-  const session = oneTimeSession
-    ? await sessions.create(sessionId, {
-      userAgent: params.userAgent,
-      oneTimeSession
-    })
-    : sessions.get(sessionId)
-
-  if (session === false) {
-    return ctx.errorResponse('This session does not exist. Use \'list_sessions\' to see all the existing sessions.')
-  }
-
-  params = mergeSessionWithParams(session, params)
-
-  try {
-    const page = await setupPage(ctx, params, session.browser)
-    const data = await resolveChallengeWithTimeout(ctx, params, page)
-
-    if (data) {
-      const { status } = data
-      delete data.status
-      ctx.successResponse(data.message, {
-        ...(oneTimeSession ? {} : { session: sessionId }),
-        ...(status ? { status } : {}),
-        solution: data.result
-      })
-    }
-  } catch (error) {
-    log.error(error)
-    return ctx.errorResponse("Unable to process browser request. Error: " + error)
-  } finally {
-    if (oneTimeSession) { sessions.destroy(sessionId) }
-  }
-}
-
-export const routes: Routes = {
-  'sessions.create': async (ctx, { session, ...options }: SessionsCreateAPICall) => {
-    session = session || UUIDv1()
-    const { browser } = await sessions.create(session, options)
-    if (browser) { ctx.successResponse('Session created successfully.', { session }) }
-  },
-  'sessions.list': (ctx) => {
-    ctx.successResponse(null, { sessions: sessions.list() })
-  },
-  'sessions.destroy': async (ctx, { session }: BaseSessionsAPICall) => {
-    if (await sessions.destroy(session)) { return ctx.successResponse('The session has been removed.') }
-    ctx.errorResponse('This session does not exist.')
-  },
-  'request.get': async (ctx, params: BaseRequestAPICall) => {
-    params.method = 'GET'
-    if (params.postData) {
-      return ctx.errorResponse('Cannot use "postBody" when sending a GET request.')
-    }
-    await browserRequest(ctx, params)
-  },
-  'request.post': async (ctx, params: BaseRequestAPICall) => {
-    params.method = 'POST'
-
-    if (!params.postData) {
-      return ctx.errorResponse('Must send param "postBody" when sending a POST request.')
-    }
-
-    await browserRequest(ctx, params)
-  },
-}
-
-export default async function Router(ctx: RequestContext, params: BaseAPICall): Promise<void> {
-  const route = routes[params.cmd]
-  if (route) { return await route(ctx, params) }
-  return ctx.errorResponse(`The command '${params.cmd}' is invalid.`)
-}

src/server.ts

@@ -0,0 +1,63 @@
+import log from './services/log'
+import {testWebBrowserInstallation} from "./services/sessions";
+
+const app = require("./app");
+const version: string = 'v' + require('../package.json').version
+const serverPort: number = Number(process.env.PORT) || 8191
+const serverHost: string = process.env.HOST || '0.0.0.0'
+
+function validateEnvironmentVariables() {
+  // ip and port variables are validated by nodejs
+  if (process.env.LOG_LEVEL && ['error', 'warn', 'info', 'verbose', 'debug'].indexOf(process.env.LOG_LEVEL) == -1) {
+    log.error(`The environment variable 'LOG_LEVEL' is wrong. Check the documentation.`);
+    process.exit(1);
+  }
+  if (process.env.LOG_HTML && ['true', 'false'].indexOf(process.env.LOG_HTML) == -1) {
+    log.error(`The environment variable 'LOG_HTML' is wrong. Check the documentation.`);
+    process.exit(1);
+  }
+  if (process.env.HEADLESS && ['true', 'false'].indexOf(process.env.HEADLESS) == -1) {
+    log.error(`The environment variable 'HEADLESS' is wrong. Check the documentation.`);
+    process.exit(1);
+  }
+  // todo: fix resolvers
+  // try {
+  //   getCaptchaSolver();
+  // } catch (e) {
+  //   log.error(`The environment variable 'CAPTCHA_SOLVER' is wrong. ${e.message}`);
+  //   process.exit(1);
+  // }
+}
+
+// Init
+log.info(`FlareSolverr ${version}`);
+log.debug('Debug log enabled');
+
+process.on('SIGTERM', () => {
+  // Capture signal on Docker Stop #158
+  log.info("Process interrupted")
+  process.exit(0)
+})
+
+process.on('uncaughtException', function(err) {
+  // Avoid crashing in NodeJS 17 due to UnhandledPromiseRejectionWarning: Unhandled promise rejection.
+  log.error(err)
+})
+
+validateEnvironmentVariables();
+
+testWebBrowserInstallation().then(() => {
+  // Start server
+  app.listen(serverPort, serverHost, () => {
+    log.info(`Listening on http://${serverHost}:${serverPort}`);
+  })
+}).catch(function(e) {
+  log.error(e);
+  const msg: string = "" + e;
+  if (msg.includes('while trying to connect to the browser!')) {
+    log.error(`It seems that the system is too slow to run FlareSolverr. 
+    If you are running with Docker, try to remove CPU limits in the container. 
+    If not, try setting the 'BROWSER_TIMEOUT' environment variable and the 'maxTimeout' parameter to higher values.`);
+  }
+  process.exit(1);
+})

src/services/log.ts

@@ -0,0 +1,41 @@
+let requests = 0
+
+const LOG_HTML: boolean = process.env.LOG_HTML == 'true';
+
+function toIsoString(date: Date) {
+  // this function fixes Date.toISOString() adding timezone
+  let tzo = -date.getTimezoneOffset(),
+      dif = tzo >= 0 ? '+' : '-',
+      pad = function(num: number) {
+        let norm = Math.floor(Math.abs(num));
+        return (norm < 10 ? '0' : '') + norm;
+      };
+
+  return date.getFullYear() +
+      '-' + pad(date.getMonth() + 1) +
+      '-' + pad(date.getDate()) +
+      'T' + pad(date.getHours()) +
+      ':' + pad(date.getMinutes()) +
+      ':' + pad(date.getSeconds()) +
+      dif + pad(tzo / 60) +
+      ':' + pad(tzo % 60);
+}
+
+export default {
+  incRequests: () => {
+      requests++
+  },
+  html(html: string) {
+    if (LOG_HTML) {
+        this.debug(html)
+    }
+  },
+  ...require('console-log-level')(
+    {level: process.env.LOG_LEVEL || 'info',
+      prefix(level: string) {
+        const req = (requests > 0) ? ` REQ-${requests}` : '';
+        return `${toIsoString(new Date())} ${level.toUpperCase()}${req}`
+      }
+    }
+  )
+}

src/services/sessions.ts

@@ -0,0 +1,194 @@
+import {v1 as UUIDv1} from 'uuid'
+import * as path from 'path'
+import {Browser} from 'puppeteer'
+import {Protocol} from "devtools-protocol";
+
+import log from './log'
+import {Proxy} from "../controllers/v1";
+
+const os = require('os');
+const fs = require('fs');
+const puppeteer = require('puppeteer');
+
+export interface SessionsCacheItem {
+  sessionId: string
+  browser: Browser
+}
+
+interface SessionsCache {
+  [key: string]: SessionsCacheItem
+}
+
+export interface SessionCreateOptions {
+  oneTimeSession: boolean
+  cookies?: Protocol.Network.CookieParam[],
+  maxTimeout?: number
+  proxy?: Proxy
+}
+
+const sessionCache: SessionsCache = {}
+let webBrowserUserAgent: string;
+
+function buildExtraPrefsFirefox(proxy: Proxy): object {
+  // Default configurations are defined here
+  // https://github.com/puppeteer/puppeteer/blob/v3.3.0/src/Launcher.ts#L481
+  const extraPrefsFirefox = {
+    // Disable newtabpage
+    "browser.newtabpage.enabled": false,
+    "browser.startup.homepage": "about:blank",
+
+    // Do not warn when closing all open tabs
+    "browser.tabs.warnOnClose": false,
+
+    // Disable telemetry
+    "toolkit.telemetry.reportingpolicy.firstRun": false,
+
+    // Disable first-run welcome page
+    "startup.homepage_welcome_url": "about:blank",
+    "startup.homepage_welcome_url.additional": "",
+
+    // Detected !
+    // // Disable images to speed up load
+    // "permissions.default.image": 2,
+
+    // Limit content processes to 1
+    "dom.ipc.processCount": 1
+  }
+
+  // proxy.url format => http://<host>:<port>
+  if (proxy && proxy.url) {
+    log.debug(`Using proxy: ${proxy.url}`)
+    const [host, portStr] = proxy.url.replace(/.+:\/\//g, '').split(':');
+    const port = parseInt(portStr);
+    if (!host || !portStr || !port) {
+      throw new Error("Proxy configuration is invalid! Use the format: protocol://ip:port")
+    }
+
+    const proxyPrefs = {
+      "network.proxy.type": 1,
+      "network.proxy.share_proxy_settings": true
+    }
+    if (proxy.url.indexOf("socks") != -1) {
+      // SOCKSv4 & SOCKSv5
+      Object.assign(proxyPrefs, {
+        "network.proxy.socks": host,
+        "network.proxy.socks_port": port,
+        "network.proxy.socks_remote_dns": true
+      });
+      if (proxy.url.indexOf("socks4") != -1) {
+        Object.assign(proxyPrefs, {
+          "network.proxy.socks_version": 4
+        });
+      } else {
+        Object.assign(proxyPrefs, {
+          "network.proxy.socks_version": 5
+        });
+      }
+    } else {
+      // HTTP
+      Object.assign(proxyPrefs, {
+        "network.proxy.ftp": host,
+        "network.proxy.ftp_port": port,
+        "network.proxy.http": host,
+        "network.proxy.http_port": port,
+        "network.proxy.ssl": host,
+        "network.proxy.ssl_port": port
+      });
+    }
+
+    // merge objects
+    Object.assign(extraPrefsFirefox, proxyPrefs);
+  }
+
+  return extraPrefsFirefox;
+}
+
+export function getUserAgent() {
+  return webBrowserUserAgent
+}
+
+export async function testWebBrowserInstallation(): Promise<void> {
+  log.info("Testing web browser installation...")
+
+  // check user home dir. this dir will be used by Firefox
+  const homeDir = os.homedir();
+  fs.accessSync(homeDir, fs.constants.F_OK | fs.constants.R_OK | fs.constants.W_OK | fs.constants.X_OK);
+  log.debug("FlareSolverr user home directory is OK: " + homeDir)
+
+  // test web browser
+  const testUrl = process.env.TEST_URL || "https://www.google.com";
+  log.debug("Test URL: " + testUrl)
+  const session = await create(null, {
+    oneTimeSession: true
+  })
+  const page = await session.browser.newPage()
+  const pageTimeout = Number(process.env.BROWSER_TIMEOUT) || 40000
+  await page.goto(testUrl, {waitUntil: 'domcontentloaded', timeout: pageTimeout})
+  webBrowserUserAgent = await page.evaluate(() => navigator.userAgent)
+
+  // replace Linux ARM user-agent because it's detected
+  if (["arm", "aarch64"].some(arch => webBrowserUserAgent.toLocaleLowerCase().includes('linux ' + arch))) {
+    webBrowserUserAgent = webBrowserUserAgent.replace(/linux \w+;/i, 'Linux x86_64;')
+  }
+
+  log.info("FlareSolverr User-Agent: " + webBrowserUserAgent)
+  await page.close()
+  await destroy(session.sessionId)
+
+  log.info("Test successful")
+}
+
+export async function create(session: string, options: SessionCreateOptions): Promise<SessionsCacheItem> {
+  log.debug('Creating new session...')
+
+  const sessionId = session || UUIDv1()
+
+  // NOTE: cookies can't be set in the session, you need to open the page first
+
+  const puppeteerOptions: any = {
+    product: 'firefox',
+    headless: process.env.HEADLESS !== 'false',
+    timeout: Number(process.env.BROWSER_TIMEOUT) || 40000
+  }
+
+  puppeteerOptions.extraPrefsFirefox = buildExtraPrefsFirefox(options.proxy)
+
+  // if we are running inside executable binary, change browser path
+  if (typeof (process as any).pkg !== 'undefined') {
+    const exe = process.platform === "win32" ? 'firefox.exe' : 'firefox';
+    puppeteerOptions.executablePath = path.join(path.dirname(process.execPath), 'firefox', exe)
+  }
+
+  log.debug('Launching web browser...')
+  let browser: Browser = await puppeteer.launch(puppeteerOptions)
+  if (!browser) {
+    throw Error(`Failed to launch web browser.`)
+  }
+
+  sessionCache[sessionId] = {
+    sessionId: sessionId,
+    browser: browser
+  }
+
+  return sessionCache[sessionId]
+}
+
+export function list(): string[] {
+  return Object.keys(sessionCache)
+}
+
+export async function destroy(id: string): Promise<boolean>{
+  if (id && sessionCache.hasOwnProperty(id)) {
+    const { browser } = sessionCache[id]
+    if (browser) {
+      await browser.close()
+      delete sessionCache[id]
+      return true
+    }
+  }
+  return false
+}
+
+export function get(id: string): SessionsCacheItem {
+  return sessionCache[id]
+}

src/services/solver.ts

@@ -0,0 +1,206 @@
+import {Page, HTTPResponse} from 'puppeteer'
+const Timeout = require('await-timeout');
+
+import log from './log'
+import {SessionCreateOptions, SessionsCacheItem} from "./sessions";
+import {V1Request} from "../controllers/v1";
+import cloudflareProvider from '../providers/cloudflare';
+
+const sessions = require('./sessions')
+
+export interface ChallengeResolutionResultT {
+    url: string
+    status: number,
+    headers?: Record<string, string>,
+    response: string,
+    cookies: object[]
+    userAgent: string
+}
+
+export interface ChallengeResolutionT {
+    status?: string
+    message: string
+    result: ChallengeResolutionResultT
+}
+
+async function resolveChallengeWithTimeout(params: V1Request, session: SessionsCacheItem) {
+    const timer = new Timeout();
+    try {
+        const promise = resolveChallenge(params, session);
+        return await Promise.race([
+            promise,
+            timer.set(params.maxTimeout, `Maximum timeout reached. maxTimeout=${params.maxTimeout} (ms)`)
+        ]);
+    } finally {
+        timer.clear();
+    }
+}
+
+async function resolveChallenge(params: V1Request, session: SessionsCacheItem): Promise<ChallengeResolutionT | void> {
+    try {
+        let status = 'ok'
+        let message = ''
+
+        const page: Page = await session.browser.newPage()
+
+        // the Puppeter timeout should be half the maxTimeout because we reload the page and wait for challenge
+        // the user can set a really high maxTimeout if he wants to
+        await page.setDefaultNavigationTimeout(params.maxTimeout / 2)
+
+        // the user-agent is changed just for linux arm build
+        await page.setUserAgent(sessions.getUserAgent())
+
+        // set the proxy
+        if (params.proxy) {
+            log.debug(`Using proxy: ${params.proxy.url}`);
+            // todo: credentials are not working
+            // if (params.proxy.username) {
+            //     await page.authenticate({
+            //         username: params.proxy.username,
+            //         password: params.proxy.password
+            //     });
+            // }
+        }
+
+        // go to the page
+        log.debug(`Navigating to... ${params.url}`)
+        let response: HTTPResponse = await gotoPage(params, page);
+
+        // set cookies
+        if (params.cookies) {
+            for (const cookie of params.cookies) {
+                // the other fields in the cookie can cause issues
+                await page.setCookie({
+                    "name": cookie.name,
+                    "value": cookie.value
+                })
+            }
+            // reload the page
+            response = await gotoPage(params, page);
+        }
+
+        // log html in debug mode
+        log.html(await page.content())
+
+        // detect protection services and solve challenges
+        try {
+            response = await cloudflareProvider(params.url, page, response);
+
+            // is response is ok
+            // reload the page to be sure we get the real page
+            log.debug("Reloading the page")
+            try {
+                response = await gotoPage(params, page, params.method);
+            } catch (e) {
+                log.warn("Page not reloaded (do not report!): Cause: " + e.toString())
+            }
+
+        } catch (e) {
+            status = "error";
+            message = "Cloudflare " + e.toString();
+        }
+
+        const payload: ChallengeResolutionT = {
+            status,
+            message,
+            result: {
+                url: page.url(),
+                status: response.status(),
+                headers: response.headers(),
+                response: null,
+                cookies: await page.cookies(),
+                userAgent: sessions.getUserAgent()
+            }
+        }
+
+        if (params.returnOnlyCookies) {
+            payload.result.headers = null;
+            payload.result.userAgent = null;
+        } else {
+            payload.result.response = await page.content()
+        }
+
+        // make sure the page is closed because if it isn't and error will be thrown
+        // when a user uses a temporary session, the browser make be quit before
+        // the page is properly closed.
+        await page.close()
+
+        return payload
+    } catch (e) {
+        log.error("Unexpected error: " + e);
+        throw e;
+    }
+}
+
+async function gotoPage(params: V1Request, page: Page, method: string = 'GET'): Promise<HTTPResponse> {
+    let pageTimeout = params.maxTimeout / 3;
+    let response: HTTPResponse
+
+    try {
+        response = await page.goto(params.url, {waitUntil: 'domcontentloaded', timeout: pageTimeout});
+    } catch (e) {
+        // retry
+        response = await page.goto(params.url, {waitUntil: 'domcontentloaded', timeout: pageTimeout});
+    }
+
+    if (method == 'POST') {
+        // post hack, it only works with utf-8 encoding
+
+        let postForm = `<form id="hackForm" action="${params.url}" method="POST">`;
+        let queryString = params.postData;
+        let pairs = (queryString[0] === '?' ? queryString.substr(1) : queryString).split('&');
+        for (let i = 0; i < pairs.length; i++) {
+            let pair = pairs[i].split('=');
+            let name; try { name = decodeURIComponent(pair[0]) } catch { name = pair[0] }
+            if (name == 'submit') continue;
+            let value; try { value = decodeURIComponent(pair[1] || '') } catch { value = pair[1] || '' }
+            postForm += `<input type="text" name="${name}" value="${value}"><br>`;
+        }
+        postForm += `</form>`;
+
+        await page.setContent(`
+            <!DOCTYPE html>
+            <html>
+            <body>
+            ${postForm}
+            <script>document.getElementById('hackForm').submit();</script>
+            </body>
+            </html>`
+        );
+        await page.waitForTimeout(2000)
+        try {
+            await page.waitForNavigation({waitUntil: 'domcontentloaded', timeout: 2000})
+        } catch (e) {}
+
+    }
+    return response
+}
+
+export async function browserRequest(params: V1Request): Promise<ChallengeResolutionT> {
+    const oneTimeSession = params.session === undefined;
+
+    const options: SessionCreateOptions = {
+        oneTimeSession: oneTimeSession,
+        cookies: params.cookies,
+        maxTimeout: params.maxTimeout,
+        proxy: params.proxy
+    }
+
+    const session: SessionsCacheItem = oneTimeSession
+        ? await sessions.create(null, options)
+        : sessions.get(params.session)
+
+    if (!session) {
+        throw Error('This session does not exist. Use \'list_sessions\' to see all the existing sessions.')
+    }
+
+    try {
+        return  await resolveChallengeWithTimeout(params, session)
+    } catch (error) {
+        throw Error("Unable to process browser request. " + error)
+    } finally {
+        if (oneTimeSession) {
+            await sessions.destroy(session.sessionId)
+        }
+    }
+}

src/session.ts

@@ -1,150 +0,0 @@
-import * as os from 'os'
-import * as path from 'path'
-import * as fs from 'fs'
-
-import puppeteer from 'puppeteer-extra'
-import { LaunchOptions, Headers, SetCookie } from 'puppeteer'
-
-import log from './log'
-import { deleteFolderRecursive, sleep, removeEmptyFields } from './utils'
-import * as Puppeteer from "puppeteer-extra/dist/puppeteer";
-
-interface SessionPageDefaults {
-  headers?: Headers
-  userAgent?: string
-}
-
-export interface SessionsCacheItem {
-  browser: Puppeteer.Browser
-  userDataDir?: string
-  defaults: SessionPageDefaults
-}
-
-interface SessionsCache {
-  [key: string]: SessionsCacheItem
-}
-
-interface SessionCreateOptions {
-  oneTimeSession?: boolean
-  userAgent?: string
-  cookies?: SetCookie[]
-  headers?: Headers,
-  maxTimeout?: number
-  proxy?: any
-}
-
-const sessionCache: SessionsCache = {}
-
-// setting "user-agent-override" evasion is not working for us because it can't be changed
-// in each request. we set the user-agent in the browser args instead
-puppeteer.use(require('puppeteer-extra-plugin-stealth')())
-
-function userDataDirFromId(id: string): string {
-  return path.join(os.tmpdir(), `/puppeteer_chrome_profile_${id}`)
-}
-
-function prepareBrowserProfile(id: string): string {
-  // TODO: maybe pass SessionCreateOptions for loading later?
-  const userDataDir = userDataDirFromId(id)
-
-  if (!fs.existsSync(userDataDir)) {
-    fs.mkdirSync(userDataDir, { recursive: true })
-  }
-
-  return userDataDir
-}
-
-export default {
-  create: async (id: string, { cookies, oneTimeSession, userAgent, headers, maxTimeout, proxy }: SessionCreateOptions): Promise<SessionsCacheItem> => {
-    let args = [
-      '--no-sandbox',
-      '--disable-setuid-sandbox',
-      '--disable-dev-shm-usage' // issue #45
-    ];
-    if (proxy && proxy.url) {
-      args.push(`--proxy-server=${proxy.url}`);
-    }
-
-    const puppeteerOptions: LaunchOptions = {
-      product: 'chrome',
-      headless: process.env.HEADLESS !== 'false',
-      args
-    }
-
-    if (!oneTimeSession) {
-      log.debug('Creating userDataDir for session.')
-      puppeteerOptions.userDataDir = prepareBrowserProfile(id)
-    }
-
-    // if we are running inside executable binary, change chrome path
-    if (typeof (process as any).pkg !== 'undefined') {
-      const exe = process.platform === "win32" ? 'chrome.exe' : 'chrome';
-      puppeteerOptions.executablePath = path.join(path.dirname(process.execPath), 'chrome', exe)
-    }
-
-    log.debug('Launching headless browser...')
-
-    // TODO: maybe access env variable?
-    // TODO: sometimes browser instances are created and not connected to correctly.
-    //       how do we handle/quit those instances inside Docker?
-    let launchTries = 3
-    let browser: Puppeteer.Browser;
-
-    while (0 <= launchTries--) {
-      try {
-        browser = await puppeteer.launch(puppeteerOptions)
-        break
-      } catch (e) {
-        if (e.message !== 'Failed to launch the browser process!')
-          throw e
-        log.warn('Failed to open browser, trying again...')
-      }
-    }
-
-    if (!browser) { throw Error(`Failed to launch browser 3 times in a row.`) }
-
-    if (cookies) {
-      const page = await browser.newPage()
-      await page.setCookie(...cookies)
-    }
-
-    sessionCache[id] = {
-      browser: browser,
-      userDataDir: puppeteerOptions.userDataDir,
-      defaults: removeEmptyFields({
-        userAgent,
-        headers,
-        maxTimeout
-      })
-    }
-
-    return sessionCache[id]
-  },
-
-  list: (): string[] => Object.keys(sessionCache),
-
-  // TODO: create a sessions.close that doesn't rm the userDataDir
-
-  destroy: async (id: string): Promise<boolean> => {
-    const { browser, userDataDir } = sessionCache[id]
-    if (browser) {
-      await browser.close()
-      delete sessionCache[id]
-      if (userDataDir) {
-        const userDataDirPath = userDataDirFromId(id)
-        try {
-          // for some reason this keeps an error from being thrown in Windows, figures
-          await sleep(5000)
-          deleteFolderRecursive(userDataDirPath)
-        } catch (e) {
-          console.error(e)
-          throw Error(`Error deleting browser session folder. ${e.message}`)
-        }
-      }
-      return true
-    }
-    return false
-  },
-
-  get: (id: string): SessionsCacheItem | false => sessionCache[id] && sessionCache[id] || false
-}

src/tests/app.test.ts

@@ -0,0 +1,625 @@
+// noinspection DuplicatedCode
+
+import {Response} from "superagent";
+import {V1ResponseBase, V1ResponseSession, V1ResponseSessions, V1ResponseSolution} from "../controllers/v1"
+
+const request = require("supertest");
+const app = require("../app");
+const sessions = require('../services/sessions');
+const version: string = 'v' + require('../../package.json').version
+
+const proxyUrl = "http://127.0.0.1:8888"
+const proxySocksUrl = "socks5://127.0.0.1:1080"
+const googleUrl = "https://www.google.com";
+const postUrl = "https://ptsv2.com/t/qv4j3-1634496523";
+const cfUrl = "https://nowsecure.nl";
+const cfCaptchaUrl = "https://idope.se"
+const cfBlockedUrl = "https://www.torrentmafya.org/table.php"
+const ddgUrl = "https://anidex.info/";
+const ccfUrl = "https://www.muziekfabriek.org";
+
+beforeAll(async () => {
+    // Init session
+    await sessions.testWebBrowserInstallation();
+});
+
+afterEach(async () => {
+    // Clean sessions
+    const sessionList = sessions.list();
+    for (const session of sessionList) {
+        await sessions.destroy(session);
+    }
+});
+
+describe("Test '/' path", () => {
+    test("GET method should return OK ", async () => {
+        const response: Response = await request(app).get("/");
+        expect(response.statusCode).toBe(200);
+        expect(response.body.msg).toBe("FlareSolverr is ready!");
+        expect(response.body.version).toBe(version);
+        expect(response.body.userAgent).toContain("Firefox/")
+    });
+
+    test("POST method should fail", async () => {
+        const response: Response = await request(app).post("/");
+        expect(response.statusCode).toBe(404);
+        expect(response.body.error).toBe("Unknown resource or HTTP verb");
+    });
+});
+
+describe("Test '/health' path", () => {
+    test("GET method should return OK", async () => {
+        const response: Response = await request(app).get("/health");
+        expect(response.statusCode).toBe(200);
+        expect(response.body.status).toBe("ok");
+    });
+});
+
+describe("Test '/wrong' path", () => {
+    test("GET method should fail", async () => {
+        const response: Response = await request(app).get("/wrong");
+        expect(response.statusCode).toBe(404);
+        expect(response.body.error).toBe("Unknown resource or HTTP verb");
+    });
+});
+
+describe("Test '/v1' path", () => {
+    test("Cmd 'request.bad' should fail", async () => {
+        const payload = {
+            "cmd": "request.bad",
+            "url": googleUrl
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(500);
+
+        const apiResponse: V1ResponseBase = response.body;
+        expect(apiResponse.status).toBe("error");
+        expect(apiResponse.message).toBe("Error: The command 'request.bad' is invalid.");
+        expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
+        expect(apiResponse.endTimestamp).toBeGreaterThanOrEqual(apiResponse.startTimestamp);
+        expect(apiResponse.version).toBe(version);
+    });
+
+    test("Cmd 'request.get' should return OK with no Cloudflare", async () => {
+        const payload = {
+            "cmd": "request.get",
+            "url": googleUrl
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSolution = response.body;
+        expect(apiResponse.status).toBe("ok");
+        expect(apiResponse.message).toBe("");
+        expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
+        expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
+        expect(apiResponse.version).toBe(version);
+
+        const solution = apiResponse.solution;
+        expect(solution.url).toContain(googleUrl)
+        expect(solution.status).toBe(200);
+        expect(Object.keys(solution.headers).length).toBeGreaterThan(0)
+        expect(solution.response).toContain("<!DOCTYPE html>")
+        expect(Object.keys(solution.cookies).length).toBeGreaterThan(0)
+        expect(solution.userAgent).toContain("Firefox/")
+    });
+
+    test("Cmd 'request.get' should return OK with Cloudflare JS", async () => {
+        const payload = {
+            "cmd": "request.get",
+            "url": cfUrl
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSolution = response.body;
+        expect(apiResponse.status).toBe("ok");
+        expect(apiResponse.message).toBe("");
+        expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
+        expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
+        expect(apiResponse.version).toBe(version);
+
+        const solution = apiResponse.solution;
+        expect(solution.url).toContain(cfUrl)
+        expect(solution.status).toBe(200);
+        expect(Object.keys(solution.headers).length).toBeGreaterThan(0)
+        expect(solution.response).toContain("<!DOCTYPE html>")
+        expect(Object.keys(solution.cookies).length).toBeGreaterThan(0)
+        expect(solution.userAgent).toContain("Firefox/")
+
+        const cfCookie: string = (solution.cookies as any[]).filter(function(cookie) {
+            return cookie.name == "cf_clearance";
+        })[0].value
+        expect(cfCookie.length).toBeGreaterThan(30)
+    });
+
+    test("Cmd 'request.get' should return fail with Cloudflare CAPTCHA", async () => {
+        const payload = {
+            "cmd": "request.get",
+            "url": cfCaptchaUrl
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSolution = response.body;
+        expect(apiResponse.status).toBe("error");
+        expect(apiResponse.message).toBe("Cloudflare Error: FlareSolverr can not resolve CAPTCHA challenges. Since the captcha doesn't always appear, you may have better luck with the next request.");
+        expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
+        expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
+        expect(apiResponse.version).toBe(version);
+        // solution is filled but not useful
+        expect(apiResponse.solution.url).toContain(cfCaptchaUrl)
+    });
+
+    test("Cmd 'request.post' should return fail with Cloudflare Blocked", async () => {
+        const payload = {
+            "cmd": "request.post",
+            "url": cfBlockedUrl,
+            "postData": "test1=test2"
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSolution = response.body;
+        expect(apiResponse.status).toBe("error");
+        expect(apiResponse.message).toBe("Cloudflare Error: Cloudflare has blocked this request. Probably your IP is banned for this site, check in your web browser.");
+        expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
+        expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
+        expect(apiResponse.version).toBe(version);
+        // solution is filled but not useful
+        expect(apiResponse.solution.url).toContain(cfBlockedUrl)
+    });
+
+    test("Cmd 'request.get' should return OK with DDoS-GUARD JS", async () => {
+        const payload = {
+            "cmd": "request.get",
+            "url": ddgUrl
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSolution = response.body;
+        expect(apiResponse.status).toBe("ok");
+        expect(apiResponse.message).toBe("");
+        expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
+        expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
+        expect(apiResponse.version).toBe(version);
+
+        const solution = apiResponse.solution;
+        expect(solution.url).toContain(ddgUrl)
+        expect(solution.status).toBe(200);
+        expect(Object.keys(solution.headers).length).toBeGreaterThan(0)
+        expect(solution.response).toContain("<!DOCTYPE html>")
+        expect(Object.keys(solution.cookies).length).toBeGreaterThan(0)
+        expect(solution.userAgent).toContain("Firefox/")
+
+        const cfCookie: string = (solution.cookies as any[]).filter(function(cookie) {
+            return cookie.name == "__ddg1_";
+        })[0].value
+        expect(cfCookie.length).toBeGreaterThan(10)
+    });
+
+    test("Cmd 'request.get' should return OK with Custom CloudFlare JS", async () => {
+        const payload = {
+            "cmd": "request.get",
+            "url": ccfUrl
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSolution = response.body;
+        expect(apiResponse.status).toBe("ok");
+        expect(apiResponse.message).toBe("");
+        expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
+        expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
+        expect(apiResponse.version).toBe(version);
+
+        const solution = apiResponse.solution;
+        expect(solution.url).toContain(ccfUrl)
+        expect(solution.status).toBe(200);
+        expect(Object.keys(solution.headers).length).toBeGreaterThan(0)
+        expect(solution.response).toContain("<html><head>")
+        expect(Object.keys(solution.cookies).length).toBeGreaterThan(0)
+        expect(solution.userAgent).toContain("Firefox/")
+
+        const cfCookie: string = (solution.cookies as any[]).filter(function(cookie) {
+            return cookie.name == "ct_anti_ddos_key";
+        })[0].value
+        expect(cfCookie.length).toBeGreaterThan(10)
+    });
+
+    test("Cmd 'request.get' should return OK with 'cookies' param", async () => {
+        const payload = {
+            "cmd": "request.get",
+            "url": googleUrl,
+            "cookies": [
+                {
+                    "name": "testcookie1",
+                    "value": "testvalue1"
+                },
+                {
+                    "name": "testcookie2",
+                    "value": "testvalue2"
+                }
+            ]
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSolution = response.body;
+        expect(apiResponse.status).toBe("ok");
+
+        const solution = apiResponse.solution;
+        expect(solution.url).toContain(googleUrl)
+        expect(Object.keys(solution.cookies).length).toBeGreaterThan(1)
+        const cookie1: string = (solution.cookies as any[]).filter(function(cookie) {
+            return cookie.name == "testcookie1";
+        })[0].value
+        expect(cookie1).toBe("testvalue1")
+        const cookie2: string = (solution.cookies as any[]).filter(function(cookie) {
+            return cookie.name == "testcookie2";
+        })[0].value
+        expect(cookie2).toBe("testvalue2")
+    });
+
+    test("Cmd 'request.get' should return OK with 'returnOnlyCookies' param", async () => {
+        const payload = {
+            "cmd": "request.get",
+            "url": googleUrl,
+            "returnOnlyCookies": true
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSolution = response.body;
+
+        const solution = apiResponse.solution;
+        expect(solution.url).toContain(googleUrl)
+        expect(solution.status).toBe(200);
+        expect(solution.headers).toBe(null)
+        expect(solution.response).toBe(null)
+        expect(Object.keys(solution.cookies).length).toBeGreaterThan(0)
+        expect(solution.userAgent).toBe(null)
+    });
+
+    test("Cmd 'request.get' should return OK with HTTP 'proxy' param", async () => {
+        /*
+        To configure TinyProxy in local:
+           * sudo vim /etc/tinyproxy/tinyproxy.conf
+              * edit => LogFile "/tmp/tinyproxy.log"
+              * edit => Syslog Off
+           * sudo tinyproxy -d
+           * sudo tail -f /tmp/tinyproxy.log
+        */
+        const payload = {
+            "cmd": "request.get",
+            "url": googleUrl,
+            "proxy": {
+                "url": proxyUrl
+            }
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSolution = response.body;
+        expect(apiResponse.status).toBe("ok");
+
+        const solution = apiResponse.solution;
+        expect(solution.url).toContain(googleUrl)
+        expect(solution.status).toBe(200);
+    });
+
+    // todo: credentials are not working
+    test.skip("Cmd 'request.get' should return OK with HTTP 'proxy' param with credentials", async () => {
+        /*
+        To configure TinyProxy in local:
+           * sudo vim /etc/tinyproxy/tinyproxy.conf
+              * edit => LogFile "/tmp/tinyproxy.log"
+              * edit => Syslog Off
+              * add => BasicAuth testuser testpass
+           * sudo tinyproxy -d
+           * sudo tail -f /tmp/tinyproxy.log
+        */
+        const payload = {
+            "cmd": "request.get",
+            "url": googleUrl,
+            "proxy": {
+                "url": proxyUrl,
+                "username": "testuser",
+                "password": "testpass"
+            }
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSolution = response.body;
+        expect(apiResponse.status).toBe("ok");
+
+        const solution = apiResponse.solution;
+        expect(solution.url).toContain(googleUrl)
+        expect(solution.status).toContain(200)
+    });
+
+    test("Cmd 'request.get' should return OK with SOCKSv5 'proxy' param", async () => {
+        /*
+        To configure Dante in local:
+           * https://linuxhint.com/set-up-a-socks5-proxy-on-ubuntu-with-dante/
+           * sudo vim /etc/sockd.conf
+           * sudo systemctl restart sockd.service
+           * curl --socks5 socks5://127.0.0.1:1080 https://www.google.com
+        */
+        const payload = {
+            "cmd": "request.get",
+            "url": googleUrl,
+            "proxy": {
+                "url": proxySocksUrl
+            }
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSolution = response.body;
+        expect(apiResponse.status).toBe("ok");
+
+        const solution = apiResponse.solution;
+        expect(solution.url).toContain(googleUrl)
+        expect(solution.status).toBe(200);
+    });
+
+    test("Cmd 'request.get' should fail with wrong 'proxy' param", async () => {
+        const payload = {
+            "cmd": "request.get",
+            "url": googleUrl,
+            "proxy": {
+                "url": "http://127.0.0.1:43210"
+            }
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(500);
+
+        const apiResponse: V1ResponseSolution = response.body;
+        expect(apiResponse.status).toBe("error");
+        expect(apiResponse.message).toBe("Error: Unable to process browser request. Error: NS_ERROR_PROXY_CONNECTION_REFUSED at https://www.google.com");
+    });
+
+    test("Cmd 'request.get' should return fail with timeout", async () => {
+        const payload = {
+            "cmd": "request.get",
+            "url": googleUrl,
+            "maxTimeout": 10
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(500);
+
+        const apiResponse: V1ResponseBase = response.body;
+        expect(apiResponse.status).toBe("error");
+        expect(apiResponse.message).toBe("Error: Unable to process browser request. Error: Maximum timeout reached. maxTimeout=10 (ms)");
+        expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
+        expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
+        expect(apiResponse.version).toBe(version);
+    });
+
+    test("Cmd 'request.get' should return fail with bad domain", async () => {
+        const payload = {
+            "cmd": "request.get",
+            "url": "https://www.google.combad"
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(500);
+
+        const apiResponse: V1ResponseBase = response.body;
+        expect(apiResponse.status).toBe("error");
+        expect(apiResponse.message).toBe("Error: Unable to process browser request. Error: NS_ERROR_UNKNOWN_HOST at https://www.google.combad");
+    });
+
+    test("Cmd 'request.get' should accept deprecated params", async () => {
+        const payload = {
+            "cmd": "request.get",
+            "url": googleUrl,
+            "userAgent": "Test User-Agent" // was removed in v2, not used
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSolution = response.body;
+        expect(apiResponse.status).toBe("ok");
+
+        const solution = apiResponse.solution;
+        expect(solution.url).toContain(googleUrl)
+        expect(solution.status).toBe(200);
+        expect(solution.userAgent).toContain("Firefox/")
+    });
+
+    test("Cmd 'request.post' should return OK with no Cloudflare", async () => {
+        const payload = {
+            "cmd": "request.post",
+            "url": postUrl + '/post',
+            "postData": "param1=value1&param2=value2"
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSolution = response.body;
+        expect(apiResponse.status).toBe("ok");
+        expect(apiResponse.message).toBe("");
+        expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
+        expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
+        expect(apiResponse.version).toBe(version);
+
+        const solution = apiResponse.solution;
+        expect(solution.url).toContain(postUrl)
+        expect(solution.status).toBe(200);
+        expect(Object.keys(solution.headers).length).toBeGreaterThan(0)
+        expect(solution.response).toContain(" I hope you have a lovely day!")
+        expect(Object.keys(solution.cookies).length).toBe(0)
+        expect(solution.userAgent).toContain("Firefox/")
+
+        // check that we sent the date
+        const payload2 = {
+            "cmd": "request.get",
+            "url": postUrl
+        }
+        const response2: Response = await request(app).post("/v1").send(payload2);
+        expect(response2.statusCode).toBe(200);
+
+        const apiResponse2: V1ResponseSolution = response2.body;
+        expect(apiResponse2.status).toBe("ok");
+
+        const solution2 = apiResponse2.solution;
+        expect(solution2.status).toBe(200);
+        expect(solution2.response).toContain(new Date().toISOString().split(':')[0].replace('T', ' '))
+    });
+
+    test("Cmd 'request.post' should fail without 'postData' param", async () => {
+        const payload = {
+            "cmd": "request.post",
+            "url": googleUrl
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(500);
+
+        const apiResponse: V1ResponseBase = response.body;
+        expect(apiResponse.status).toBe("error");
+        expect(apiResponse.message).toBe("Error: Must send param \"postBody\" when sending a POST request.");
+        expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
+        expect(apiResponse.endTimestamp).toBeGreaterThanOrEqual(apiResponse.startTimestamp);
+        expect(apiResponse.version).toBe(version);
+    });
+
+    test("Cmd 'sessions.create' should return OK", async () => {
+        const payload = {
+            "cmd": "sessions.create"
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSession = response.body;
+        expect(apiResponse.status).toBe("ok");
+        expect(apiResponse.message).toBe("Session created successfully.");
+        expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
+        expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
+        expect(apiResponse.version).toBe(version);
+        expect(apiResponse.session.length).toBe(36);
+    });
+
+    test("Cmd 'sessions.create' should return OK with session", async () => {
+        const payload = {
+            "cmd": "sessions.create",
+            "session": "2bc6bb20-2f56-11ec-9543-test"
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSession = response.body;
+        expect(apiResponse.status).toBe("ok");
+        expect(apiResponse.message).toBe("Session created successfully.");
+        expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
+        expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
+        expect(apiResponse.version).toBe(version);
+        expect(apiResponse.session).toBe("2bc6bb20-2f56-11ec-9543-test");
+    });
+
+    test("Cmd 'sessions.list' should return OK", async () => {
+        // create one session for testing
+        const payload0 = {
+            "cmd": "sessions.create"
+        }
+        const response0: Response = await request(app).post("/v1").send(payload0);
+        expect(response0.statusCode).toBe(200);
+
+        const payload = {
+            "cmd": "sessions.list"
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSessions = response.body;
+        expect(apiResponse.status).toBe("ok");
+        expect(apiResponse.message).toBe("");
+        expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
+        expect(apiResponse.endTimestamp).toBeGreaterThanOrEqual(apiResponse.startTimestamp);
+        expect(apiResponse.version).toBe(version);
+        expect(apiResponse.sessions.length).toBeGreaterThan(0)
+    });
+
+    test("Cmd 'sessions.destroy' should return OK", async () => {
+        // create one session for testing
+        const payload0 = {
+            "cmd": "sessions.create"
+        }
+        const response0: Response = await request(app).post("/v1").send(payload0);
+        expect(response0.statusCode).toBe(200);
+        const apiResponse0: V1ResponseSession = response0.body;
+        const sessionId0 = apiResponse0.session
+
+        const payload = {
+            "cmd": "sessions.destroy",
+            "session": sessionId0
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseBase = response.body;
+        expect(apiResponse.status).toBe("ok");
+        expect(apiResponse.message).toBe("The session has been removed.");
+        expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
+        expect(apiResponse.endTimestamp).toBeGreaterThanOrEqual(apiResponse.startTimestamp);
+        expect(apiResponse.version).toBe(version);
+    });
+
+    test("Cmd 'sessions.destroy' should fail", async () => {
+        const payload = {
+            "cmd": "sessions.destroy",
+            "session": "bad-session"
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(500);
+
+        const apiResponse: V1ResponseBase = response.body;
+        expect(apiResponse.status).toBe("error");
+        expect(apiResponse.message).toBe("Error: This session does not exist.");
+        expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
+        expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
+        expect(apiResponse.version).toBe(version);
+    });
+
+    test("Cmd 'request.get' should use session", async () => {
+        // create one session for testing
+        const payload0 = {
+            "cmd": "sessions.create"
+        }
+        const response0: Response = await request(app).post("/v1").send(payload0);
+        expect(response0.statusCode).toBe(200);
+        const apiResponse0: V1ResponseSession = response0.body;
+        const sessionId0 = apiResponse0.session
+
+        // first request should solve the challenge
+        const payload = {
+            "cmd": "request.get",
+            "url": cfUrl,
+            "session": sessionId0
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSolution = response.body;
+        expect(apiResponse.status).toBe("ok");
+        const cfCookie: string = (apiResponse.solution.cookies as any[]).filter(function(cookie) {
+            return cookie.name == "cf_clearance";
+        })[0].value
+        expect(cfCookie.length).toBeGreaterThan(30)
+
+        // second request should have the same cookie
+        const response2: Response = await request(app).post("/v1").send(payload);
+        expect(response2.statusCode).toBe(200);
+
+        const apiResponse2: V1ResponseSolution = response2.body;
+        expect(apiResponse2.status).toBe("ok");
+        const cfCookie2: string = (apiResponse2.solution.cookies as any[]).filter(function(cookie) {
+            return cookie.name == "cf_clearance";
+        })[0].value
+        expect(cfCookie2.length).toBeGreaterThan(30)
+        expect(cfCookie2).toBe(cfCookie)
+    });
+
+});

src/types.ts

@@ -1,9 +0,0 @@
-import { IncomingMessage, ServerResponse } from 'http';
-
-export interface RequestContext {
-  req: IncomingMessage
-  res: ServerResponse
-  startTimestamp: number
-  errorResponse: (msg: string) => void,
-  successResponse: (msg: string, extendedProperties?: object) => void
-}

src/utils.ts

@@ -1,31 +0,0 @@
-import * as fs from 'fs'
-import * as Path from 'path'
-import { promisify } from 'util'
-
-export const sleep = promisify(setTimeout)
-
-// recursive fs.rmdir needs node version 12:
-// https://github.com/ngosang/FlareSolverr/issues/5#issuecomment-655572712
-export function deleteFolderRecursive(path: string) {
-  if (fs.existsSync(path)) {
-    fs.readdirSync(path).forEach((file) => {
-      const curPath = Path.join(path, file)
-      if (fs.lstatSync(curPath).isDirectory()) { // recurse
-        deleteFolderRecursive(curPath)
-      } else { // delete file
-        fs.unlinkSync(curPath)
-      }
-    })
-    fs.rmdirSync(path)
-  }
-}
-
-export const removeEmptyFields = (o: Record<string, any>): typeof o => {
-  const r: typeof o = {}
-  for (const k in o) {
-    if (o[k] !== undefined) {
-      r[k] = o[k]
-    }
-  }
-  return r
-}