Bump version 1.2.3

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,37 @@
+**Please use the search bar** at the top of the page and make sure you are not creating an already submitted issue.
+Check closed issues as well, because your issue may have already been fixed.
+
+### Instruction on how to enable debug and html trace
+
+[Follow the instructions from this wiki page](https://github.com/FlareSolverr/FlareSolverr/wiki/How-to-enable-debug-and-html-trace)
+
+### Environment
+
+**FlareSolverr Version**:
+
+**Docker**: [yes/no]
+
+**OS**: 
+
+**Last Working FlareSolverr Version**:
+
+**Are you using a proxy or VPN?** [yes/no]
+
+**Using Captcha Solver:** [yes/no]
+
+**If using captcha solver, which one:**
+
+### Description
+
+[List steps to reproduce the error and details on what happens and what you expected to happen]
+
+
+### Logged Error Messages
+
+[Place any relevant error messages you noticed from the logs here.]
+
+[Make sure you attach the full logs with your personal information removed in case we need more information]
+
+### Screenshots
+
+[Place any screenshots of the issue here if needed]

.github/workflows/release-docker.yml

@@ -1,4 +1,4 @@
-name: publish
+name: release-docker
 
 on:
   push:
@@ -30,13 +30,13 @@ jobs:
         uses: docker/setup-buildx-action@v1
       -
         name: Login to DockerHub
-        uses: docker/login-action@v1 
+        uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       -
         name: Login to GitHub Container Registry
-        uses: docker/login-action@v1 
+        uses: docker/login-action@v1
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}

.github/workflows/release.yml

@@ -0,0 +1,55 @@
+name: release
+
+on:
+  push:
+    tags:
+      - 'v*.*.*'
+
+jobs:
+  build:
+    name: Create release
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # get all commits, branches and tags (required for the changelog)
+
+      - name: Setup Node
+        uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+
+      - name: Build artifacts
+        run: |
+          npm install
+          npm run build
+          npm run package
+
+      - name: Build changelog
+        id: github_changelog
+        run: |
+          changelog=$(git log $(git tag | tail -2 | head -1)..HEAD --no-merges --oneline)
+          changelog="${changelog//'%'/'%25'}"
+          changelog="${changelog//$'\n'/'%0A'}"
+          changelog="${changelog//$'\r'/'%0D'}"
+          echo "##[set-output name=changelog;]${changelog}"
+
+      - name: Create release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_PAT }}
+        with:
+          tag_name: ${{ github.ref }}
+          release_name: ${{ github.ref }}
+          body: ${{ steps.github_changelog.outputs.changelog }}
+          draft: false
+          prerelease: false
+
+      - name: Upload release artifacts
+        uses: alexellis/upload-assets@0.2.2
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_PAT }}
+        with:
+          asset_paths: '["./bin/*.zip"]'

.gitignore

@@ -121,3 +121,6 @@ dist
 
 # Project Development
 testing/
+
+# Binaries
+bin/

README.md

@@ -1,33 +1,81 @@
 # FlareSolverr
 
-Proxy server to bypass Cloudflare protection
+[![Latest release](https://img.shields.io/github/v/release/FlareSolverr/FlareSolverr)](https://github.com/FlareSolverr/FlareSolverr/releases)
+[![Docker Pulls](https://img.shields.io/docker/pulls/flaresolverr/flaresolverr)](https://hub.docker.com/r/flaresolverr/flaresolverr/)
+[![GitHub issues](https://img.shields.io/github/issues/FlareSolverr/FlareSolverr)](https://github.com/FlareSolverr/FlareSolverr/issues)
+[![GitHub pull requests](https://img.shields.io/github/issues-pr/FlareSolverr/FlareSolverr)](https://github.com/FlareSolverr/FlareSolverr/pulls)
+[![Donate PayPal](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=X5NJLLX5GLTV6&source=url)
+[![Donate Buy Me A Coffee](https://img.shields.io/badge/Donate-Buy%20me%20a%20coffee-yellow.svg)](https://www.buymeacoffee.com/ngosang)
+[![Donate Bitcoin](https://img.shields.io/badge/Donate-Bitcoin-orange.svg)](https://en.cryptobadges.io/donate/13Hcv77AdnFWEUZ9qUpoPBttQsUT7q9TTh)
+
+FlareSolverr is a proxy server to bypass Cloudflare protection
 
 :warning: This project is in beta state. Some things may not work and the API can change at any time.
-See the known issues section.
 
 ## How it works
 
 FlareSolverr starts a proxy server and it waits for user requests in an idle state using few resources.
 When some request arrives, it uses [puppeteer](https://github.com/puppeteer/puppeteer) with the
 [stealth plugin](https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin-stealth)
-to create a headless browser (Chrome). It opens the URL with user parameters and waits until the
-Cloudflare challenge is solved (or timeout). The HTML code and the cookies are sent back to the
-user and those cookies can be used to bypass Cloudflare using other HTTP clients.
+to create a headless browser (Chrome). It opens the URL with user parameters and waits until the Cloudflare challenge
+is solved (or timeout). The HTML code and the cookies are sent back to the user, and those cookies can be used to
+bypass Cloudflare using other HTTP clients.
 
-**NOTE**: Web browsers consume a lot of memory. If you are running FlareSolverr on a machine with few RAM,
-do not make many requests at once. With each request a new browser is launched.
-(It is possible to use a permanent session. However, if you use sessions, you should make sure to close them as soon as you are done using them.)
+**NOTE**: Web browsers consume a lot of memory. If you are running FlareSolverr on a machine with few RAM, do not make
+many requests at once. With each request a new browser is launched.
+
+It is also possible to use a permanent session. However, if you use sessions, you should make sure to close them as
+soon as you are done using them.
 
 ## Installation
 
-It requires NodeJS.
+### Docker
 
-Run `PUPPETEER_PRODUCT=chrome npm install` to install FlareSolverr dependencies.
+It is recommended to install using a Docker container because the project depends on an external browser that is
+already included within the image.
+
+Docker images are available in:
+* GitHub Registry => https://github.com/orgs/FlareSolverr/packages/container/package/flaresolverr
+* DockerHub => https://hub.docker.com/r/flaresolverr/flaresolverr
+
+Supported architectures are:
+| Architecture | Tag |
+| :----: | --- |
+| x86-64 | linux/amd64 |
+| ARM64 | linux/arm64 |
+| ARM32 | linux/arm/v7 |
+
+We provide a `docker-compose.yml` configuration file. Clone this repository and execute `docker-compose up -d` to start
+the container.
+
+If you prefer the `docker cli` execute the following command.
+```bash
+docker run -d \
+  --name=flaresolverr \
+  -p 8191:8191 \
+  -e LOG_LEVEL=info \
+  --restart unless-stopped \
+  ghcr.io/flaresolverr/flaresolverr:latest
+```
+
+### Precompiled binaries
+
+This is the recommended way for Windows users.
+* Download the [FlareSolverr zip](https://github.com/FlareSolverr/FlareSolverr/releases) from the release's assets. It is available for Windows and Linux.
+* Extract the zip file. FlareSolverr executable and chrome folder must be in the same directory.
+* Execute FlareSolverr binary. In the environment variables section you can find how to change the configuration.
+
+### From source code
+
+This is the recommended way for MacOS users and for developers.
+* Install [NodeJS](https://nodejs.org/)
+* Clone this repository and open a shell in that path
+* Run `npm install` command to install FlareSolverr dependencies
+* Run `npm run build` command to compile TypeScript code
+* Run `npm start` command to start FlareSolverr
 
 ## Usage
 
-First run `npm run build`. Once the TypeScript is compiled, you can use `npm start` to start FlareSolverr.
-
 Example request:
 ```bash
 curl -L -X POST 'http://localhost:8191/v1' \
@@ -35,7 +83,7 @@ curl -L -X POST 'http://localhost:8191/v1' \
 --data-raw '{
   "cmd": "request.get",
   "url":"http://www.google.com/",
-  "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36",
+  "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleW...",
   "maxTimeout": 60000,
   "headers": {
     "X-Test": "Testing 123..."
@@ -47,23 +95,22 @@ curl -L -X POST 'http://localhost:8191/v1' \
 
 #### + `sessions.create`
 
-This will launch a new browser instance which will retain cookies until you destroy it
-with `sessions.destroy`. This comes in handy so you don't have to keep solving challenges
-over and over and you won't need to keep sending cookies for the browser to use.
+This will launch a new browser instance which will retain cookies until you destroy it with `sessions.destroy`.
+This comes in handy, so you don't have to keep solving challenges over and over and you won't need to keep sending
+cookies for the browser to use.
 
-This also speeds up the requests since it won't have to launch a new browser instance for
-every request.
+This also speeds up the requests since it won't have to launch a new browser instance for every request.
 
 Parameter | Notes
 |--|--|
-session | Optional. The session ID that you want to be assinged to the instance. If one isn't set a random UUID will be assigned.
+session | Optional. The session ID that you want to be assigned to the instance. If isn't set a random UUID will be assigned.
 userAgent | Optional. Will be used by the headless browser.
 
 #### + `sessions.list`
 
-Returns a list of all the active sessions. More for debuging if you are curious to see
-how many sessions are running. You should always make sure to properly close each
-session when you are done using them as too many may slow your computer down.
+Returns a list of all the active sessions. More for debugging if you are curious to see how many sessions are running.
+You should always make sure to properly close each session when you are done using them as too many may slow your
+computer down.
 
 Example response:
 
@@ -79,9 +126,8 @@ Example response:
 
 #### + `sessions.destroy`
 
-This will properly shutdown a browser instance and remove all files associaded with it
-to free up resources for a new session. Whenever you no longer need to use a session you
-should make sure to close it.
+This will properly shutdown a browser instance and remove all files associated with it to free up resources for a new
+session. When you no longer need to use a session you should make sure to close it.
 
 Parameter | Notes
 |--|--|
@@ -94,8 +140,9 @@ Parameter | Notes
 url | Mandatory
 session | Optional. Will send the request from and existing browser instance. If one is not sent it will create a temporary instance that will be destroyed immediately after the request is completed.
 headers | Optional. To specify user headers.
-maxTimeout | Optional. Max timeout to solve the challenge
-cookies | Optional. Will be used by the headless browser. Follow [this](https://github.com/puppeteer/puppeteer/blob/v3.3.0/docs/api.md#pagesetcookiecookies) format
+maxTimeout | Optional, default value 60000. Max timeout to solve the challenge in milliseconds.
+cookies | Optional. Will be used by the headless browser. Follow [this](https://github.com/puppeteer/puppeteer/blob/v3.3.0/docs/api.md#pagesetcookiecookies) format.
+returnOnlyCookies | Optional, default false. Only returns the cookies. Response data, headers and other parts of the response are removed.
 
 Example response from running the `curl` above:
 
@@ -117,14 +164,13 @@ Example response from running the `curl` above:
             "content-length": "61587",
             "x-xss-protection": "0",
             "x-frame-options": "SAMEORIGIN",
-            "set-cookie": "1P_JAR=2020-07-16-04; expires=Sat, 15-Aug-2020 04:15:49 GMT; path=/; domain=.google.com; Secure; SameSite=none\nNID=204=QE3Ocq15XalczqjuDy52HeseG3zAZuJzID3R57g_oeQHyoV5DuvDhpWc4r9IcPoeIYmkr_ZTX_MNOU8IAbtXmVO7Bmq0adb-hpIHaTBIdBk3Ofifp4gO6vZleVuFYfj7ePkHeHdzGoX-en0FvKtd9iofX4O6RiAdEIAnpL7Wge4; expires=Fri, 15-Jan-2021 04:15:49 GMT; path=/; domain=.google.com; Secure; HttpOnly; SameSite=none",
-            "alt-svc": "h3-29=\":443\"; ma=2592000,h3-27=\":443\"; ma=2592000,h3-25=\":443\"; ma=2592000,h3-T050=\":443\"; ma=2592000,h3-Q050=\":443\"; ma=2592000,h3-Q046=\":443\"; ma=2592000,h3-Q043=\":443\"; ma=2592000,quic=\":443\"; ma=2592000; v=\"46,43\""
+            "set-cookie": "1P_JAR=2020-07-16-04; expires=Sat..."
         },
         "response":"<!DOCTYPE html>...",
         "cookies": [
             {
                 "name": "NID",
-                "value": "204=QE3Ocq15XalczqjuDy52HeseG3zAZuJzID3R57g_oeQHyoV5DuvDhpWc4r9IcPoeIYmkr_ZTX_MNOU8IAbtXmVO7Bmq0adb-hpIHaTBIdBk3Ofifp4gO6vZleVuFYfj7ePkHeHdzGoX-en0FvKtd9iofX4O6RiAdEIAnpL7Wge4",
+                "value": "204=QE3Ocq15XalczqjuDy52HeseG3zAZuJzID3R57...",
                 "domain": ".google.com",
                 "path": "/",
                 "expires": 1610684149.307722,
@@ -147,7 +193,7 @@ Example response from running the `curl` above:
                 "sameSite": "None"
             }
         ],
-        "userAgent": "Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36"
+        "userAgent": "Windows NT 10.0; Win64; x64) AppleWebKit/5..."
     },
     "status": "ok",
     "message": "",
@@ -165,90 +211,73 @@ Parameter | Notes
 |--|--|
 postData | Must be a string. If you want to POST a form, don't forget to set the `Content-Type` header to `application/x-www-form-urlencoded` or the server might not understand your request.
 
-## Downloading Images and PDFs (small files)
+### Download small files
 
-If you need to access an image/pdf or small file, you should pass the `download` parameter to
-`request.get` setting it to `true`. Rather than access the html and return text it will
-return a the buffer **base64** encoded which you will be able to decode and save the image/pdf.
+If you need to access an image/pdf or small file, you should pass the `download` parameter to `request.get` setting it
+to `true`. Rather than access the html and return text it will return the buffer **base64** encoded which you will be
+able to decode and save the image/pdf.
 
-This method isn't recommended for videos or anything larger. As that should be streamed back to
-the client and at the moment there is nothing setup to do so. If this is something you need feel
-free to create an issue and/or submit a PR.
+This method isn't recommended for videos or anything larger. As that should be streamed back to the client and at the
+moment there is nothing setup to do so. If this is something you need feel free to create an issue and/or submit a PR.
 
 ## Environment variables
 
-To set the environment vars in Linux run `export LOG_LEVEL=debug` and then start FlareSolverr in the same shell.
-
 Name | Default | Notes
 |--|--|--|
-LOG_LEVEL | info | Used to change the verbosity of the logging.
-LOG_HTML | false | Used for debugging. If `true` all html that passes through the proxy will be logged to the console.
+LOG_LEVEL | info | Used to change the verbosity of the logging. Use `LOG_LEVEL=debug` for more information.
+LOG_HTML | false | Used for debugging. If `true` all HTML that passes through the proxy will be logged to the console in `debug` level.
 PORT | 8191 | Change this if you already have a process running on port `8191`.
 HOST | 0.0.0.0 | This shouldn't need to be messed with but if you insist, it's here!
-CAPTCHA_SOLVER | None | This is used to select which captcha solving method it used when a captcha is encounted.
+CAPTCHA_SOLVER | none | This is used to select which captcha solving method it used when a captcha is encountered.
 HEADLESS | true | This is used to debug the browser by not running it in headless mode.
 
+Environment variables are set differently depending on the operating system. Some examples:
+* Docker: Take a look at the Docker section in this document. Environment variables can be set in the `docker-compose.yml` file or in the Docker CLI command.
+* Linux: Run `export LOG_LEVEL=debug` and then start FlareSolverr in the same shell.
+* Windows: Open `cmd.exe`, run `set LOG_LEVEL=debug` and then start FlareSolverr in the same shell.
+
 ## Captcha Solvers
 
-Sometimes CF not only gives mathmatical computations and browser tests, sometimes they also require
-the user to solve a captcha. If this is the case, FlareSolverr will return the captcha page. But that's
-not very helpful to you is it?
+:warning: At this time none of the captcha solvers work. You can check the status in the open issues. Any help is welcome.
 
-FlareSolverr can be customized to solve the captcha's automatically by setting the environment variable
-`CAPTCHA_SOLVER` to the file name of one of the adapters inside the [/captcha](src/captcha) directory.
+Sometimes CloudFlare not only gives mathematical computations and browser tests, sometimes they also require the user to
+solve a captcha.
+If this is the case, FlareSolverr will return the error `Captcha detected but no automatic solver is configured.`
 
-### [CaptchaHarvester](https://github.com/NoahCardoza/CaptchaHarvester)
+FlareSolverr can be customized to solve the captchas automatically by setting the environment variable `CAPTCHA_SOLVER`
+to the file name of one of the adapters inside the [/captcha](src/captcha) directory.
 
-This method makes use of the [CaptchaHarvester](https://github.com/NoahCardoza/CaptchaHarvester) project which allows users to collect thier own tokens from ReCaptcha V2/V3 and hCaptcha for free.
+### hcaptcha-solver
 
-To use this method you must set these ENV variables:
+This method makes use of the [hcaptcha-solver](https://github.com/JimmyLaurent/hcaptcha-solver) project.
+
+NOTE: This solver works picking random images so it will fail in a lot of requests and it's hard to know if it is
+working or not. In a real use case with Sonarr/Radarr + Jackett it is still useful because those apps make a new request
+each 15 minutes. Eventually one of the requests is going to work and Jackett saves the cookie forever (until it stops
+working).
+
+To use this solver you must set the environment variable:
+
+```bash
+CAPTCHA_SOLVER=hcaptcha-solver
+```
+
+### CaptchaHarvester
+
+This method makes use of the [CaptchaHarvester](https://github.com/NoahCardoza/CaptchaHarvester) project which allows
+users to collect their own tokens from ReCaptcha V2/V3 and hCaptcha for free.
+
+To use this method you must set these environment variables:
 
 ```bash
 CAPTCHA_SOLVER=harvester
 HARVESTER_ENDPOINT=https://127.0.0.1:5000/token
 ```
 
-**Note**: above I set `HARVESTER_ENDPOINT` to the default configureation
-of the captcha harvester's server, but that could change if
-you customize the command line flags. Simply put, `HARVESTER_ENDPOINT`
-should be set to the URI of the route that returns a token in plain text when called.
+**Note**: above I set `HARVESTER_ENDPOINT` to the default configuration of the captcha harvester's server, but that
+could change if you customize the command line flags. Simply put, `HARVESTER_ENDPOINT` should be set to the URI of the
+route that returns a token in plain text when called.
 
-### [hcaptcha-solver](https://github.com/JimmyLaurent/hcaptcha-solver)
+## Related projects
 
-This method makes use of the [hcaptcha-solver](https://github.com/JimmyLaurent/hcaptcha-solver) project which attempts to solve hcaptcha by randomly selecting images.
-
-To use this solver you must first install it and then set it as the `CAPTCHA_SOLVER`.
-
-```bash
-npm i hcaptcha-solver
-CAPTCHA_SOLVER=hcaptcha-solver
-```
-
-## Docker
-
-You can edit environment variables in `./Dockerfile` and build your own image.
-
-```bash
-docker build -t flaresolverr:latest .
-docker run --restart=always --name flaresolverr -p 8191:8191 -d flaresolverr:latest
-```
-
-## TypeScript
-
-I'm quite new to TypeScript. If you spot any funny business or anything that is or isn't being
-used properly feel free to submit a PR or open an issue.
-
-## Known issues / Roadmap
-
-The current implementation seems to be working on the sites I have been testing them on. However, if you find it unable to access a site, open an issue and I'd be happy to investigate.
-
-That being said, the project uses the [puppeteer stealth plugin](https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin-stealth). If Cloudflare is able to detect the headless browser, it's more that projects domain to fix.
-
-TODO:
-
-* Fix remaining issues in the code (see TODOs in code)
-* Make the maxTimeout more accurate (count the time to open the first page / maybe count the captcha solve time?)
-* Hide sensitive information in logs
-* Reduce Docker image size
-* Docker image for ARM architecture
-* Install instructions for Windows
+* C# implementation => https://github.com/FlareSolverr/FlareSolverrSharp

build-binaries.js

@@ -0,0 +1,78 @@
+const fs = require('fs')
+const path = require('path')
+const {execSync} = require('child_process')
+const archiver = require('archiver')
+const puppeteer = require('puppeteer')
+const version = 'v' + require('./package.json').version;
+
+(async () => {
+  const builds = [
+    {
+      platform: 'linux',
+      version: 756035,
+      chromeFolder: 'chrome-linux',
+      fsExec: 'flaresolverr-linux',
+      fsZipExec: 'flaresolverr',
+      fsZipName: 'linux-x64'
+    },
+    {
+      platform: 'win64',
+      version: 756035,
+      chromeFolder: 'chrome-win',
+      fsExec: 'flaresolverr-win.exe',
+      fsZipExec: 'flaresolverr.exe',
+      fsZipName: 'windows-x64'
+    }
+    // TODO: this is working but changes are required in session.ts to find chrome path
+    // {
+    //   platform: 'mac',
+    //   version: 756035,
+    //   chromeFolder: 'chrome-mac',
+    //   fsExec: 'flaresolverr-macos',
+    //   fsZipExec: 'flaresolverr',
+    //   fsZipName: 'macos'
+    // }
+  ]
+
+  // generate executables
+  console.log('Generating executables...')
+  if (fs.existsSync('bin')) {
+    fs.rmdirSync('bin', {recursive: true})
+  }
+  execSync('pkg -t node14-win-x64,node14-linux-x64 --out-path bin .')
+  // execSync('pkg -t node14-win-x64,node14-mac-x64,node14-linux-x64 --out-path bin .')
+
+  // download Chrome and zip together
+  for (const os of builds) {
+    console.log('Building ' + os.fsZipName + ' artifact')
+
+    // download chrome
+    console.log('Downloading Chrome...')
+    const f = puppeteer.createBrowserFetcher({
+      platform: os.platform,
+      path: path.join(__dirname, 'bin', 'puppeteer')
+    })
+    await f.download(os.version)
+
+    // compress in zip
+    console.log('Compressing zip file...')
+    const zipName = 'bin/flaresolverr-' + version + '-' + os.fsZipName + '.zip'
+    const output = fs.createWriteStream(zipName)
+    const archive = archiver('zip')
+
+    output.on('close', function () {
+      console.log('File ' + zipName + ' created. Size: ' + archive.pointer() + ' bytes')
+    })
+
+    archive.on('error', function (err) {
+      throw err
+    })
+
+    archive.pipe(output)
+
+    archive.file('bin/' + os.fsExec, { name: 'flaresolverr/' + os.fsZipExec })
+    archive.directory('bin/puppeteer/' + os.platform + '-' + os.version + '/' + os.chromeFolder, 'flaresolverr/chrome')
+
+    archive.finalize()
+  }
+})()

docker-compose.yml

@@ -0,0 +1,14 @@
+---
+version: "2.1"
+services:
+  flaresolverr:
+    # DockerHub mirror flaresolverr/flaresolverr:latest
+    image: ghcr.io/flaresolverr/flaresolverr:latest
+    container_name: flaresolverr
+    environment:
+      - LOG_LEVEL=${LOG_LEVEL:-info}
+      - LOG_HTML=${LOG_HTML:-false}
+      - CAPTCHA_SOLVER=${CAPTCHA_SOLVER:-none}
+    ports:
+      - "${PORT:-8191}:8191"
+    restart: unless-stopped

package.json

@@ -1,11 +1,12 @@
 {
   "name": "flaresolverr",
-  "version": "1.2.0",
+  "version": "1.2.3",
   "description": "Proxy server to bypass Cloudflare protection.",
   "scripts": {
     "start": "node ./dist/index.js",
     "build": "tsc",
-    "dev": "nodemon -e ts --exec ts-node src/index.ts"
+    "dev": "nodemon -e ts --exec ts-node src/index.ts",
+    "package": "node build-binaries.js"
   },
   "author": "Diego Heras (ngosang)",
   "contributors": [
@@ -19,7 +20,16 @@
     "type": "git",
     "url": "https://github.com/ngosang/FlareSolverr"
   },
+  "pkg": {
+    "assets": [
+      "node_modules/puppeteer-extra-plugin-stealth/**/*.*"
+    ]
+  },
+  "bin": {
+    "flaresolverr": "dist/index.js"
+  },
   "dependencies": {
+    "await-timeout": "^1.1.1",
     "console-log-level": "^1.4.1",
     "got": "^11.5.1",
     "hcaptcha-solver": "^1.0.2",
@@ -29,9 +39,11 @@
     "uuid": "^8.2.0"
   },
   "devDependencies": {
+    "@types/await-timeout": "^0.3.1",
     "@types/node": "^14.0.23",
     "@types/puppeteer": "^3.0.1",
     "@types/uuid": "^8.0.0",
+    "archiver": "^5.2.0",
     "eslint": "^7.5.0",
     "eslint-config-airbnb-base": "^14.2.0",
     "eslint-config-standard": "^14.1.1",
@@ -40,6 +52,7 @@
     "eslint-plugin-promise": "^4.2.1",
     "eslint-plugin-standard": "^4.0.1",
     "nodemon": "^2.0.4",
+    "pkg": "^4.4.9",
     "ts-node": "^8.10.2",
     "typescript": "^3.9.7"
   }

src/captcha/index.ts

@@ -1,3 +1,5 @@
+import log from "../log";
+
 export enum CaptchaType {
   re = 'reCaptcha',
   h = 'hCaptcha'
@@ -16,7 +18,9 @@ const captchaSolvers: { [key: string]: Solver } = {}
 export default (): Solver => {
   const method = process.env.CAPTCHA_SOLVER
 
-  if (!method) { return null }
+  if (!method || method.toLowerCase() == 'none') {
+    return null;
+  }
 
   if (!(method in captchaSolvers)) {
     try {
@@ -31,5 +35,7 @@ export default (): Solver => {
     }
   }
 
+  log.info(`Using '${method} to solve the captcha.`);
+
   return captchaSolvers[method]
-}
+}

src/index.ts

@@ -3,7 +3,7 @@ import { createServer, IncomingMessage, ServerResponse } from 'http';
 import { RequestContext } from './types'
 import Router, { BaseAPICall } from './routes'
 
-const version: string = require('../package.json').version
+const version: string = "v" + require('../package.json').version
 const serverPort: number = Number(process.env.PORT) || 8191
 const serverHost: string = process.env.HOST || '0.0.0.0'
 
@@ -65,11 +65,19 @@ function validateIncomingRequest(ctx: RequestContext, params: BaseAPICall) {
 }
 
 createServer((req: IncomingMessage, res: ServerResponse) => {
+  const startTimestamp = Date.now()
+
   // count the request for the log prefix
   log.incRequests()
-
-  const startTimestamp = Date.now()
   log.info(`Incoming request: ${req.method} ${req.url}`)
+
+  // show welcome message
+  if (req.url == '/') {
+    successResponse("FlareSolverr is ready!", null, res, startTimestamp);
+    return;
+  }
+
+  // get request body
   const bodyParts: any[] = []
   req.on('data', chunk => {
     bodyParts.push(chunk)
@@ -102,5 +110,6 @@ createServer((req: IncomingMessage, res: ServerResponse) => {
     })
   })
 }).listen(serverPort, serverHost, () => {
-  log.info(`FlareSolverr v${version} listening on http://${serverHost}:${serverPort}`)
+  log.info(`FlareSolverr ${version} listening on http://${serverHost}:${serverPort}`);
+  log.debug('Debug log enabled');
 })

src/log.ts

@@ -10,10 +10,10 @@ export default {
   },
   ...require('console-log-level')(
     {
-      level: process.env.LOG_LEVEL || 'debug',
+      level: process.env.LOG_LEVEL || 'info',
       prefix(level: string) {
         return `${new Date().toISOString()} ${level.toUpperCase()} REQ-${requests}`
       }
     }
   )
-}
+}

src/providers/cloudflare.ts

@@ -0,0 +1,164 @@
+import {Response} from 'puppeteer'
+import {Page} from "puppeteer-extra/dist/puppeteer";
+
+import log from "../log";
+import getCaptchaSolver, {CaptchaType} from "../captcha";
+
+/**
+ *  This class contains the logic to solve protections provided by CloudFlare
+**/
+
+const CHALLENGE_SELECTORS = ['#trk_jschal_js', '.ray_id', '.attack-box'];
+const TOKEN_INPUT_NAMES = ['g-recaptcha-response', 'h-captcha-response'];
+
+export default async function resolveChallenge(url: string, page: Page, response: Response): Promise<Response> {
+
+  // look for challenge and return fast if not detected
+  if (!response.headers().server.startsWith('cloudflare')) {
+    log.info('Cloudflare not detected');
+    return response;
+  }
+  log.info('Cloudflare detected');
+
+  if (await page.$('.cf-error-code')) {
+    throw new Error('Cloudflare has blocked this request (Code 1020 Detected).')
+  }
+
+  let selectorFoundCount = 0;
+  if (response.status() > 400) {
+    // detect cloudflare wait 5s
+    for (const selector of CHALLENGE_SELECTORS) {
+      const cfChallengeElem = await page.$(selector)
+      if (cfChallengeElem) {
+        selectorFoundCount++
+        log.debug(`Javascript challenge element '${selector}' detected.`)
+        log.debug('Waiting for Cloudflare challenge...')
+
+        while (true) {
+          await page.waitFor(1000)
+          try {
+            // catch exception timeout in waitForNavigation
+            response = await page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 5000 })
+          } catch (error) { }
+
+          try {
+            // catch Execution context was destroyed
+            const cfChallengeElem = await page.$(selector)
+            if (!cfChallengeElem) { break }
+            log.debug('Found challenge element again...')
+          } catch (error)
+          { }
+
+          response = await page.reload({ waitUntil: 'domcontentloaded' })
+          log.debug('Page reloaded.')
+          log.html(await page.content())
+        }
+
+        log.debug('Validating HTML code...')
+        break
+      } else {
+        log.debug(`No '${selector}' challenge element detected.`)
+      }
+    }
+    log.debug("Javascript challenge selectors found: " + selectorFoundCount + ", total selectors: " + CHALLENGE_SELECTORS.length)
+  } else {
+    // some sites use cloudflare but there is no challenge
+    log.debug(`Javascript challenge not detected. Status code: ${response.status()}`);
+    selectorFoundCount = 1;
+  }
+
+  // it seems some captcha pages return 200 sometimes
+  if (await page.$('input[name="cf_captcha_kind"]')) {
+    log.info('Captcha challenge detected.');
+    const captchaSolver = getCaptchaSolver()
+    if (captchaSolver) {
+      const captchaStartTimestamp = Date.now()
+      const challengeForm = await page.$('#challenge-form')
+      if (challengeForm) {
+        const captchaTypeElm = await page.$('input[name="cf_captcha_kind"]')
+        const cfCaptchaType: string = await captchaTypeElm.evaluate((e: any) => e.value)
+        const captchaType: CaptchaType = (CaptchaType as any)[cfCaptchaType]
+        if (!captchaType) {
+          throw new Error('Unknown captcha type!');
+        }
+
+        let sitekey = null
+        if (captchaType != 'hCaptcha' && process.env.CAPTCHA_SOLVER != 'hcaptcha-solver') {
+          const sitekeyElem = await page.$('*[data-sitekey]')
+          if (!sitekeyElem) {
+            throw new Error('Could not find sitekey!');
+          }
+          sitekey = await sitekeyElem.evaluate((e) => e.getAttribute('data-sitekey'))
+        }
+
+        log.info('Waiting to receive captcha token to bypass challenge...')
+        const token = await captchaSolver({
+          url,
+          sitekey,
+          type: captchaType
+        })
+        log.debug(`Token received: ${token}`);
+        if (!token) {
+          throw new Error('Token solver failed to return a token.')
+        }
+
+        let responseFieldsFoundCount = 0;
+        for (const name of TOKEN_INPUT_NAMES) {
+          const input = await page.$(`textarea[name="${name}"]`)
+          if (input) {
+            responseFieldsFoundCount ++;
+            log.debug(`Challenge response field '${name}' found in challenge form.`);
+            await input.evaluate((e: HTMLTextAreaElement, token) => { e.value = token }, token);
+          }
+        }
+        if (responseFieldsFoundCount == 0) {
+          throw new Error('Challenge response field not found in challenge form.');
+        }
+
+        // ignore preset event listeners on the form
+        await page.evaluate(() => {
+          window.addEventListener('submit', (e) => { event.stopPropagation() }, true)
+        })
+
+        // it seems some sites obfuscate their challenge forms
+        // TODO: look into how they do it and come up with a more solid solution
+        try {
+          // this element is added with js and we want to wait for all the js to load before submitting
+          await page.waitForSelector('#challenge-form', { timeout: 10000 })
+        } catch (err) {
+          throw new Error("No '#challenge-form' element detected.");
+        }
+
+        // calculates the time it took to solve the captcha
+        const captchaSolveTotalTime = Date.now() - captchaStartTimestamp
+
+        // generates a random wait time
+        const randomWaitTime = (Math.floor(Math.random() * 10) + 10) * 1000
+
+        // waits, if any, time remaining to appear human but stay as fast as possible
+        const timeLeft = randomWaitTime - captchaSolveTotalTime
+        if (timeLeft > 0) {
+          log.debug(`Waiting for '${timeLeft}' milliseconds.`);
+          await page.waitFor(timeLeft);
+        }
+
+        // submit captcha response
+        challengeForm.evaluate((e: HTMLFormElement) => e.submit())
+        response = await page.waitForNavigation({ waitUntil: 'domcontentloaded' })
+
+        if (await page.$('input[name="cf_captcha_kind"]')) {
+          throw new Error('Captcha service failed to solve the challenge.');
+        }
+      }
+    } else {
+      throw new Error('Captcha detected but no automatic solver is configured.');
+    }
+  } else {
+    if (selectorFoundCount == 0)
+    {
+      throw new Error('No challenge selectors found, unable to proceed')
+    }
+  }
+
+  return response;
+}

src/routes.ts

@@ -1,11 +1,12 @@
 import { v1 as UUIDv1 } from 'uuid'
+import { SetCookie, Request, Response, Headers, HttpMethod, Overrides } from 'puppeteer'
+import { Page, Browser } from "puppeteer-extra/dist/puppeteer";
+const Timeout = require('await-timeout');
+
+import log from './log'
 import sessions, { SessionsCacheItem } from './session'
 import { RequestContext } from './types'
-import log from './log'
-import { SetCookie, Request, Headers, HttpMethod, Overrides, Cookie } from 'puppeteer'
-import { TimeoutError } from 'puppeteer/Errors'
-import getCaptchaSolver, { CaptchaType } from './captcha'
-import * as Puppeteer from "puppeteer-extra/dist/puppeteer";
+import cloudflareProvider from './providers/cloudflare';
 
 export interface BaseAPICall {
   cmd: string
@@ -68,60 +69,25 @@ type OverridesProps =
   'postData' |
   'headers'
 
-// We always set a Windows User-Agent because ARM builds are detected by CloudFlare
-const DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36"
-const CHALLENGE_SELECTORS = ['#trk_jschal_js', '.ray_id', '.attack-box']
-const TOKEN_INPUT_NAMES = ['g-recaptcha-response', 'h-captcha-response']
+// We always set a Windows User-Agent because ARM builds are detected by Cloudflare
+const DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36"
 
-async function interceptResponse(page: Puppeteer.Page, callback: (payload: ChallengeResolutionT) => any) {
-  const client = await page.target().createCDPSession();
-  await client.send('Fetch.enable', {
-    patterns: [
-      {
-        urlPattern: '*',
-        resourceType: 'Document',
-        requestStage: 'Response',
-      },
-    ],
-  });
-
-  client.on('Fetch.requestPaused', async (e) => {
-    log.debug('Fetch.requestPaused. Checking if the response has valid cookies')
-    let headers = e.responseHeaders || []
-
-    let cookies = await page.cookies();
-    log.debug(cookies)
-
-    if (cookies.filter((c: Cookie) => c.name === 'cf_clearance').length > 0) {
-      log.debug('Aborting request and return cookies. valid cookies found')
-      await client.send('Fetch.failRequest', {requestId: e.requestId, errorReason: 'Aborted'})
-
-      let status = 'ok'
-      let message = ''
-      const payload: ChallengeResolutionT = {
-        status,
-        message,
-        result: {
-          url: page.url(),
-          status: e.status,
-          headers: headers.reduce((a: any, x: { name: any; value: any }) => ({ ...a, [x.name]: x.value }), {}),
-          response: null,
-          cookies: cookies,
-          userAgent: ''
-        }
-      }
-
-      callback(payload);
-    } else {
-      log.debug('Continuing request. no valid cookies found')
-      await client.send('Fetch.continueRequest', {requestId: e.requestId})
-    }
-  });
+async function resolveChallengeWithTimeout(ctx: RequestContext, params: BaseRequestAPICall, page: Page) {
+  const maxTimeout = params.maxTimeout || 60000
+  const timer = new Timeout();
+  try {
+    const promise = resolveChallenge(ctx, params, page);
+    return await Promise.race([
+      promise,
+      timer.set(maxTimeout, `Maximum timeout reached. maxTimeout=${maxTimeout} (ms)`)
+    ]);
+  } finally {
+    timer.clear();
+  }
 }
 
-async function resolveChallenge(ctx: RequestContext, { url, maxTimeout, proxy, download, returnOnlyCookies }: BaseRequestAPICall, page: Puppeteer.Page): Promise<ChallengeResolutionT | void> {
+async function resolveChallenge(ctx: RequestContext, { url, proxy, download, returnOnlyCookies }: BaseRequestAPICall, page: Page): Promise<ChallengeResolutionT | void> {
 
-  maxTimeout = maxTimeout || 60000
   let status = 'ok'
   let message = ''
 
@@ -132,154 +98,15 @@ async function resolveChallenge(ctx: RequestContext, { url, maxTimeout, proxy, d
   }
 
   log.debug(`Navigating to... ${url}`)
-  let response = await page.goto(url, { waitUntil: 'domcontentloaded' })
+  let response: Response = await page.goto(url, { waitUntil: 'domcontentloaded' })
+  log.html(await page.content())
 
-  // look for challenge
-  if (response.headers().server.startsWith('cloudflare')) {
-    log.info('Cloudflare detected')
-
-    if (await page.$('.cf-error-code')) {
-      await page.close()
-      return ctx.errorResponse('Cloudflare has blocked this request (Code 1020 Detected).')
-    }
-
-    if (response.status() > 400) {
-      // detect cloudflare wait 5s
-      for (const selector of CHALLENGE_SELECTORS) {
-        const cfChallengeElem = await page.$(selector)
-        if (cfChallengeElem) {
-          log.html(await page.content())
-          log.debug('Waiting for Cloudflare challenge...')
-
-          let interceptingResult: ChallengeResolutionT;
-          if (returnOnlyCookies) { //If we just want to get the cookies, intercept the response before we get the content/body (just cookies and headers)
-            await interceptResponse(page, async function(payload){
-              interceptingResult = payload;
-            });
-          }
-
-          // TODO: find out why these pages hang sometimes
-          while (Date.now() - ctx.startTimestamp < maxTimeout) {
-            await page.waitFor(1000)
-            try {
-              // catch exception timeout in waitForNavigation
-              response = await page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 5000 })
-            } catch (error) { }
-
-            if (returnOnlyCookies && interceptingResult) {
-              await page.close();
-              return interceptingResult;
-            }
-
-            try {
-              // catch Execution context was destroyed
-              const cfChallengeElem = await page.$(selector)
-              if (!cfChallengeElem) { break }
-              log.debug('Found challenge element again...')
-            } catch (error)
-            { }
-
-            response = await page.reload({ waitUntil: 'domcontentloaded' })
-            log.debug('Reloaded page...')
-          }
-
-          if (Date.now() - ctx.startTimestamp >= maxTimeout) {
-            ctx.errorResponse(`Maximum timeout reached. maxTimeout=${maxTimeout} (ms)`)
-            return
-          }
-
-          log.debug('Validating HTML code...')
-          break
-        } else {
-          log.debug(`No '${selector}' challenge element detected.`)
-        }
-      }
-    }
-
-    // it seems some captcha pages return 200 sometimes
-    if (await page.$('input[name="cf_captcha_kind"]')) {
-      const captchaSolver = getCaptchaSolver()
-      if (captchaSolver) {
-        const captchaStartTimestamp = Date.now()
-        const challengeForm = await page.$('#challenge-form')
-        if (challengeForm) {
-          log.html(await page.content())
-          const captchaTypeElm = await page.$('input[name="cf_captcha_kind"]')
-          const cfCaptchaType: string = await captchaTypeElm.evaluate((e: any) => e.value)
-          const captchaType: CaptchaType = (CaptchaType as any)[cfCaptchaType]
-          if (!captchaType) { return ctx.errorResponse('Unknown captcha type!') }
-
-          let sitekey = null
-          if (captchaType != 'hCaptcha' && process.env.CAPTCHA_SOLVER != 'hcaptcha-solver') {
-            const sitekeyElem = await page.$('*[data-sitekey]')
-            if (!sitekeyElem) { return ctx.errorResponse('Could not find sitekey!') }
-            sitekey = await sitekeyElem.evaluate((e) => e.getAttribute('data-sitekey'))
-          }
-
-          log.info('Waiting to receive captcha token to bypass challenge...')
-          const token = await captchaSolver({
-            url,
-            sitekey,
-            type: captchaType
-          })
-
-          if (!token) {
-            await page.close()
-            return ctx.errorResponse('Token solver failed to return a token.')
-          }
-
-          for (const name of TOKEN_INPUT_NAMES) {
-            const input = await page.$(`textarea[name="${name}"]`)
-            if (input) { await input.evaluate((e: HTMLTextAreaElement, token) => { e.value = token }, token) }
-          }
-
-          // ignore preset event listeners on the form
-          await page.evaluate(() => {
-            window.addEventListener('submit', (e) => { event.stopPropagation() }, true)
-          })
-
-          // it seems some sites obfuscate their challenge forms
-          // TODO: look into how they do it and come up with a more solid solution
-          try {
-            // this element is added with js and we want to wait for all the js to load before submitting
-            await page.waitForSelector('#challenge-form [type=submit]', { timeout: 5000 })
-          } catch (err) {
-            if (err instanceof TimeoutError) {
-              log.debug(`No '#challenge-form [type=submit]' element detected.`)
-            }
-          }
-
-          // calculates the time it took to solve the captcha
-          const captchaSolveTotalTime = Date.now() - captchaStartTimestamp
-
-          // generates a random wait time
-          const randomWaitTime = (Math.floor(Math.random() * 20) + 10) * 1000
-
-          // waits, if any, time remaining to appear human but stay as fast as possible
-          const timeLeft = randomWaitTime - captchaSolveTotalTime
-          if (timeLeft > 0) { await page.waitFor(timeLeft) }
-
-          let interceptingResult: ChallengeResolutionT;
-          if (returnOnlyCookies) { //If we just want to get the cookies, intercept the response before we get the content/body (just cookies and headers)
-            await interceptResponse(page, async function(payload){
-              interceptingResult = payload;
-            });
-          }
-
-          // submit captcha response
-          challengeForm.evaluate((e: HTMLFormElement) => e.submit())
-          response = await page.waitForNavigation({ waitUntil: 'domcontentloaded' })
-
-          if (returnOnlyCookies && interceptingResult) {
-            await page.close();
-            return interceptingResult;
-          }
-        }
-      } else {
-        status = 'warning'
-        message = 'Captcha detected but no automatic solver is configured.'
-      }
-    }
+  // Detect protection services and solve challenges
+  try {
+    response = await cloudflareProvider(url, page, response);
+  } catch (e) {
+    status = "error";
+    message = "Cloudflare " + e.toString();
   }
 
   const payload: ChallengeResolutionT = {
@@ -295,14 +122,19 @@ async function resolveChallenge(ctx: RequestContext, { url, maxTimeout, proxy, d
     }
   }
 
-  if (download) {
-    // for some reason we get an error unless we reload the page
-    // has something to do with a stale buffer and this is the quickest
-    // fix since I am short on time
-    response = await page.goto(url, { waitUntil: 'domcontentloaded' })
-    payload.result.response = (await response.buffer()).toString('base64')
+  if (returnOnlyCookies) {
+    payload.result.headers = null;
+    payload.result.userAgent = null;
   } else {
-    payload.result.response = await page.content()
+    if (download) {
+      // for some reason we get an error unless we reload the page
+      // has something to do with a stale buffer and this is the quickest
+      // fix since I am short on time
+      response = await page.goto(url, { waitUntil: 'domcontentloaded' })
+      payload.result.response = (await response.buffer()).toString('base64')
+    } else {
+      payload.result.response = await page.content()
+    }
   }
 
   // make sure the page is closed because if it isn't and error will be thrown
@@ -322,7 +154,7 @@ function mergeSessionWithParams({ defaults }: SessionsCacheItem, params: BaseReq
   return copy
 }
 
-async function setupPage(ctx: RequestContext, params: BaseRequestAPICall, browser: Puppeteer.Browser): Promise<Puppeteer.Page> {
+async function setupPage(ctx: RequestContext, params: BaseRequestAPICall, browser: Browser): Promise<Page> {
   const page = await browser.newPage()
 
   // merge session defaults with params
@@ -406,7 +238,7 @@ const browserRequest = async (ctx: RequestContext, params: BaseRequestAPICall) =
 
   try {
     const page = await setupPage(ctx, params, session.browser)
-    const data = await resolveChallenge(ctx, params, page)
+    const data = await resolveChallengeWithTimeout(ctx, params, page)
 
     if (data) {
       const { status } = data
@@ -454,14 +286,6 @@ export const routes: Routes = {
 
     await browserRequest(ctx, params)
   },
-  'request.cookies': async (ctx, params: BaseRequestAPICall) => {
-    params.returnOnlyCookies = true
-    params.method = 'GET'
-    if (params.postData) {
-      return ctx.errorResponse('Cannot use "postBody" when sending a GET request.')
-    }
-    await browserRequest(ctx, params)
-  },
 }
 
 export default async function Router(ctx: RequestContext, params: BaseAPICall): Promise<void> {

src/session.ts

@@ -56,7 +56,11 @@ function prepareBrowserProfile(id: string): string {
 
 export default {
   create: async (id: string, { cookies, oneTimeSession, userAgent, headers, maxTimeout, proxy }: SessionCreateOptions): Promise<SessionsCacheItem> => {
-    let args = ['--no-sandbox', '--disable-setuid-sandbox'];
+    let args = [
+      '--no-sandbox',
+      '--disable-setuid-sandbox',
+      '--disable-dev-shm-usage' // issue #45
+    ];
     if (proxy && proxy.url) {
       args.push(`--proxy-server=${proxy.url}`);
     }
@@ -72,6 +76,12 @@ export default {
       puppeteerOptions.userDataDir = prepareBrowserProfile(id)
     }
 
+    // if we are running inside executable binary, change chrome path
+    if (typeof (process as any).pkg !== 'undefined') {
+      const exe = process.platform === "win32" ? 'chrome.exe' : 'chrome';
+      puppeteerOptions.executablePath = path.join(path.dirname(process.execPath), 'chrome', exe)
+    }
+
     log.debug('Launching headless browser...')
 
     // TODO: maybe access env variable?
@@ -137,4 +147,4 @@ export default {
   },
 
   get: (id: string): SessionsCacheItem | false => sessionCache[id] && sessionCache[id] || false
-}
+}