Bump version 1.2.4

.dockerignore

@@ -1,6 +1,7 @@
-node_modules
-npm-debug.log
-Dockerfile
-.dockerignore
-.git
-.gitignore
+.git/
+.github/
+.idea/
+bin/
+dist/
+node_modules/
+resources/

.github/ISSUE_TEMPLATE.md

@@ -1,31 +1,24 @@
 **Please use the search bar** at the top of the page and make sure you are not creating an already submitted issue.
 Check closed issues as well, because your issue may have already been fixed.
 
-### Instruction on how to enable debug and html trace
+### How to enable debug and html traces
 
 [Follow the instructions from this wiki page](https://github.com/FlareSolverr/FlareSolverr/wiki/How-to-enable-debug-and-html-trace)
 
 ### Environment
 
-**FlareSolverr Version**:
-
-**Docker**: [yes/no]
-
-**OS**: 
-
-**Last Working FlareSolverr Version**:
-
-**Are you using a proxy or VPN?** [yes/no]
-
-**Using Captcha Solver:** [yse/no]
-
-**If using captcha solver, which one:**
+* **FlareSolverr version**:
+* **Last working FlareSolverr version**:
+* **Operating system**:
+* **Are you using Docker**: [yes/no]
+* **Are you using a proxy or VPN?** [yes/no]
+* **Are you using Captcha Solver:** [yes/no]
+* **If using captcha solver, which one:**
 
 ### Description
 
 [List steps to reproduce the error and details on what happens and what you expected to happen]
 
-
 ### Logged Error Messages
 
 [Place any relevant error messages you noticed from the logs here.]

.github/workflows/release-docker.yml

@@ -1,4 +1,4 @@
-name: publish
+name: release-docker
 
 on:
   push:

.github/workflows/release.yml

@@ -7,19 +7,35 @@ on:
 
 jobs:
   build:
-    name: Create Release
+    name: Create release
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
+        with:
+          fetch-depth: 0 # get all commits, branches and tags (required for the changelog)
 
-      - name: Build Changelog
-        id: github_release
-        uses: mikepenz/release-changelog-builder-action@main
-        env:
-          GITHUB_TOKEN: ${{ secrets.GH_PAT }}
+      - name: Setup Node
+        uses: actions/setup-node@v2
+        with:
+          node-version: '14'
 
-      - name: Create Release
+      - name: Build artifacts
+        run: |
+          npm install
+          npm run build
+          npm run package
+
+      - name: Build changelog
+        id: github_changelog
+        run: |
+          changelog=$(git log $(git tag | tail -2 | head -1)..HEAD --no-merges --oneline)
+          changelog="${changelog//'%'/'%25'}"
+          changelog="${changelog//$'\n'/'%0A'}"
+          changelog="${changelog//$'\r'/'%0D'}"
+          echo "##[set-output name=changelog;]${changelog}"
+
+      - name: Create release
         id: create_release
         uses: actions/create-release@v1
         env:
@@ -27,6 +43,13 @@ jobs:
         with:
           tag_name: ${{ github.ref }}
           release_name: ${{ github.ref }}
-          body: ${{ steps.github_release.outputs.changelog }}
+          body: ${{ steps.github_changelog.outputs.changelog }}
           draft: false
           prerelease: false
+
+      - name: Upload release artifacts
+        uses: alexellis/upload-assets@0.2.2
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_PAT }}
+        with:
+          asset_paths: '["./bin/*.zip"]'

.gitignore

@@ -121,3 +121,6 @@ dist
 
 # Project Development
 testing/
+
+# Binaries
+bin/

README.md

@@ -1,8 +1,12 @@
 # FlareSolverr
 
-[![GitHub issues](https://img.shields.io/github/issues/FlareSolverr/FlareSolverr.svg?maxAge=60&style=flat-square)](https://github.com/FlareSolverr/FlareSolverr/issues)
-[![GitHub pull requests](https://img.shields.io/github/issues-pr/FlareSolverr/FlareSolverr.svg?maxAge=60&style=flat-square)](https://github.com/FlareSolverr/FlareSolverr/pulls)
-[![Docker Pulls](https://img.shields.io/docker/pulls/flaresolverr/flaresolverr.svg?maxAge=60&style=flat-square)](https://hub.docker.com/r/flaresolverr/flaresolverr/)
+[![Latest release](https://img.shields.io/github/v/release/FlareSolverr/FlareSolverr)](https://github.com/FlareSolverr/FlareSolverr/releases)
+[![Docker Pulls](https://img.shields.io/docker/pulls/flaresolverr/flaresolverr)](https://hub.docker.com/r/flaresolverr/flaresolverr/)
+[![GitHub issues](https://img.shields.io/github/issues/FlareSolverr/FlareSolverr)](https://github.com/FlareSolverr/FlareSolverr/issues)
+[![GitHub pull requests](https://img.shields.io/github/issues-pr/FlareSolverr/FlareSolverr)](https://github.com/FlareSolverr/FlareSolverr/pulls)
+[![Donate PayPal](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=X5NJLLX5GLTV6&source=url)
+[![Donate Buy Me A Coffee](https://img.shields.io/badge/Donate-Buy%20me%20a%20coffee-yellow.svg)](https://www.buymeacoffee.com/ngosang)
+[![Donate Bitcoin](https://img.shields.io/badge/Donate-Bitcoin-orange.svg)](https://en.cryptobadges.io/donate/13Hcv77AdnFWEUZ9qUpoPBttQsUT7q9TTh)
 
 FlareSolverr is a proxy server to bypass Cloudflare protection
 
@@ -48,14 +52,22 @@ If you prefer the `docker cli` execute the following command.
 ```bash
 docker run -d \
   --name=flaresolverr \
+  -p 8191:8191 \
   -e LOG_LEVEL=info \
   --restart unless-stopped \
   ghcr.io/flaresolverr/flaresolverr:latest
 ```
 
+### Precompiled binaries
+
+This is the recommended way for Windows users.
+* Download the [FlareSolverr zip](https://github.com/FlareSolverr/FlareSolverr/releases) from the release's assets. It is available for Windows and Linux.
+* Extract the zip file. FlareSolverr executable and chrome folder must be in the same directory.
+* Execute FlareSolverr binary. In the environment variables section you can find how to change the configuration.
+
 ### From source code
 
-This is the recommended way for Windows / MacOS users and for developers.
+This is the recommended way for MacOS users and for developers.
 * Install [NodeJS](https://nodejs.org/)
 * Clone this repository and open a shell in that path
 * Run `npm install` command to install FlareSolverr dependencies
@@ -128,8 +140,9 @@ Parameter | Notes
 url | Mandatory
 session | Optional. Will send the request from and existing browser instance. If one is not sent it will create a temporary instance that will be destroyed immediately after the request is completed.
 headers | Optional. To specify user headers.
-maxTimeout | Optional. Max timeout to solve the challenge
-cookies | Optional. Will be used by the headless browser. Follow [this](https://github.com/puppeteer/puppeteer/blob/v3.3.0/docs/api.md#pagesetcookiecookies) format
+maxTimeout | Optional, default value 60000. Max timeout to solve the challenge in milliseconds.
+cookies | Optional. Will be used by the headless browser. Follow [this](https://github.com/puppeteer/puppeteer/blob/v3.3.0/docs/api.md#pagesetcookiecookies) format.
+returnOnlyCookies | Optional, default false. Only returns the cookies. Response data, headers and other parts of the response are removed.
 
 Example response from running the `curl` above:
 
@@ -209,43 +222,52 @@ moment there is nothing setup to do so. If this is something you need feel free
 
 ## Environment variables
 
-To set the environment vars in Linux run `export LOG_LEVEL=debug` and then start FlareSolverr in the same shell.
-
 Name | Default | Notes
 |--|--|--|
-LOG_LEVEL | info | Used to change the verbosity of the logging.
-LOG_HTML | false | Used for debugging. If `true` all HTML that passes through the proxy will be logged to the console in `debug` level.
-PORT | 8191 | Change this if you already have a process running on port `8191`.
-HOST | 0.0.0.0 | This shouldn't need to be messed with but if you insist, it's here!
-CAPTCHA_SOLVER | None | This is used to select which captcha solving method it used when a captcha is encountered.
-HEADLESS | true | This is used to debug the browser by not running it in headless mode.
+LOG_LEVEL | info | Verbosity of the logging. Use `LOG_LEVEL=debug` for more information.
+LOG_HTML | false | Only for debugging. If `true` all HTML that passes through the proxy will be logged to the console in `debug` level.
+CAPTCHA_SOLVER | none | Captcha solving method. It used when a captcha is encountered. See the Captcha Solvers section.
+HEADLESS | true | Only for debugging. To run the web browser in headless mode or visible.
+PORT | 8191 | Listening port. You don't need to change this if you are running on Docker.
+HOST | 0.0.0.0 | Listening interface. You don't need to change this if you are running on Docker.
+
+Environment variables are set differently depending on the operating system. Some examples:
+* Docker: Take a look at the Docker section in this document. Environment variables can be set in the `docker-compose.yml` file or in the Docker CLI command.
+* Linux: Run `export LOG_LEVEL=debug` and then start FlareSolverr in the same shell.
+* Windows: Open `cmd.exe`, run `set LOG_LEVEL=debug` and then start FlareSolverr in the same shell.
 
 ## Captcha Solvers
 
-Sometimes CF not only gives mathematical computations and browser tests, sometimes they also require the user to solve
-a captcha. If this is the case, FlareSolverr will return the captcha page. But that's not very helpful to you is it?
+:warning: At this time none of the captcha solvers work. You can check the status in the open issues. Any help is welcome.
+
+Sometimes CloudFlare not only gives mathematical computations and browser tests, sometimes they also require the user to
+solve a captcha.
+If this is the case, FlareSolverr will return the error `Captcha detected but no automatic solver is configured.`
 
 FlareSolverr can be customized to solve the captchas automatically by setting the environment variable `CAPTCHA_SOLVER`
 to the file name of one of the adapters inside the [/captcha](src/captcha) directory.
 
 ### hcaptcha-solver
 
-This method makes use of the [hcaptcha-solver](https://github.com/JimmyLaurent/hcaptcha-solver) project which attempts
-to solve hCaptcha by randomly selecting images.
+This method makes use of the [hcaptcha-solver](https://github.com/JimmyLaurent/hcaptcha-solver) project.
 
-To use this solver you must first install it and then set it as the `CAPTCHA_SOLVER`.
+NOTE: This solver works picking random images so it will fail in a lot of requests and it's hard to know if it is
+working or not. In a real use case with Sonarr/Radarr + Jackett it is still useful because those apps make a new request
+each 15 minutes. Eventually one of the requests is going to work and Jackett saves the cookie forever (until it stops
+working).
+
+To use this solver you must set the environment variable:
 
 ```bash
-npm i hcaptcha-solver
 CAPTCHA_SOLVER=hcaptcha-solver
 ```
 
 ### CaptchaHarvester
 
 This method makes use of the [CaptchaHarvester](https://github.com/NoahCardoza/CaptchaHarvester) project which allows
-users to collect thier own tokens from ReCaptcha V2/V3 and hCaptcha for free.
+users to collect their own tokens from ReCaptcha V2/V3 and hCaptcha for free.
 
-To use this method you must set these ENV variables:
+To use this method you must set these environment variables:
 
 ```bash
 CAPTCHA_SOLVER=harvester

build-binaries.js

@@ -0,0 +1,82 @@
+const fs = require('fs')
+const path = require('path')
+const { execSync } = require('child_process')
+const archiver = require('archiver')
+const puppeteer = require('puppeteer')
+const version = 'v' + require('./package.json').version;
+
+(async () => {
+  const builds = [
+    {
+      platform: 'linux',
+      version: 756035,
+      chromeFolder: 'chrome-linux',
+      fsExec: 'flaresolverr-linux',
+      fsZipExec: 'flaresolverr',
+      fsZipName: 'linux-x64',
+      fsLicenseName: 'LICENSE'
+    },
+    {
+      platform: 'win64',
+      version: 756035,
+      chromeFolder: 'chrome-win',
+      fsExec: 'flaresolverr-win.exe',
+      fsZipExec: 'flaresolverr.exe',
+      fsZipName: 'windows-x64',
+      fsLicenseName: 'LICENSE.txt'
+    }
+    // TODO: this is working but changes are required in session.ts to find chrome path
+    // {
+    //   platform: 'mac',
+    //   version: 756035,
+    //   chromeFolder: 'chrome-mac',
+    //   fsExec: 'flaresolverr-macos',
+    //   fsZipExec: 'flaresolverr',
+    //   fsZipName: 'macos',
+    //   fsLicenseName: 'LICENSE'
+    // }
+  ]
+
+  // generate executables
+  console.log('Generating executables...')
+  if (fs.existsSync('bin')) {
+    fs.rmdirSync('bin', { recursive: true })
+  }
+  execSync('pkg -t node14-win-x64,node14-linux-x64 --out-path bin .')
+  // execSync('pkg -t node14-win-x64,node14-mac-x64,node14-linux-x64 --out-path bin .')
+
+  // download Chrome and zip together
+  for (const os of builds) {
+    console.log('Building ' + os.fsZipName + ' artifact')
+
+    // download chrome
+    console.log('Downloading Chrome...')
+    const f = puppeteer.createBrowserFetcher({
+      platform: os.platform,
+      path: path.join(__dirname, 'bin', 'puppeteer')
+    })
+    await f.download(os.version)
+
+    // compress in zip
+    console.log('Compressing zip file...')
+    const zipName = 'bin/flaresolverr-' + version + '-' + os.fsZipName + '.zip'
+    const output = fs.createWriteStream(zipName)
+    const archive = archiver('zip')
+
+    output.on('close', function () {
+      console.log('File ' + zipName + ' created. Size: ' + archive.pointer() + ' bytes')
+    })
+
+    archive.on('error', function (err) {
+      throw err
+    })
+
+    archive.pipe(output)
+
+    archive.file('LICENSE', { name: 'flaresolverr/' + os.fsLicenseName })
+    archive.file('bin/' + os.fsExec, { name: 'flaresolverr/' + os.fsZipExec })
+    archive.directory('bin/puppeteer/' + os.platform + '-' + os.version + '/' + os.chromeFolder, 'flaresolverr/chrome')
+
+    await archive.finalize()
+  }
+})()

docker-compose.yml

@@ -6,13 +6,9 @@ services:
     image: ghcr.io/flaresolverr/flaresolverr:latest
     container_name: flaresolverr
     environment:
-      # Used to change the verbosity of the logging
-      - LOG_LEVEL=info
-      # Enables hcaptcha-solver => https://github.com/JimmyLaurent/hcaptcha-solver
-      #- CAPTCHA_SOLVER=hcaptcha-solver
-      # Enables CaptchaHarvester => https://github.com/NoahCardoza/CaptchaHarvester
-      #- CAPTCHA_SOLVER=harvester
-      #- HARVESTER_ENDPOINT=https://127.0.0.1:5000/token
+      - LOG_LEVEL=${LOG_LEVEL:-info}
+      - LOG_HTML=${LOG_HTML:-false}
+      - CAPTCHA_SOLVER=${CAPTCHA_SOLVER:-none}
     ports:
-      - 8191:8191
+      - "${PORT:-8191}:8191"
     restart: unless-stopped

package.json

@@ -1,11 +1,12 @@
 {
   "name": "flaresolverr",
-  "version": "1.2.1",
+  "version": "1.2.4",
   "description": "Proxy server to bypass Cloudflare protection.",
   "scripts": {
     "start": "node ./dist/index.js",
     "build": "tsc",
-    "dev": "nodemon -e ts --exec ts-node src/index.ts"
+    "dev": "nodemon -e ts --exec ts-node src/index.ts",
+    "package": "node build-binaries.js"
   },
   "author": "Diego Heras (ngosang)",
   "contributors": [
@@ -19,6 +20,14 @@
     "type": "git",
     "url": "https://github.com/ngosang/FlareSolverr"
   },
+  "pkg": {
+    "assets": [
+      "node_modules/puppeteer-extra-plugin-stealth/**/*.*"
+    ]
+  },
+  "bin": {
+    "flaresolverr": "dist/index.js"
+  },
   "dependencies": {
     "await-timeout": "^1.1.1",
     "console-log-level": "^1.4.1",
@@ -34,6 +43,7 @@
     "@types/node": "^14.0.23",
     "@types/puppeteer": "^3.0.1",
     "@types/uuid": "^8.0.0",
+    "archiver": "^5.2.0",
     "eslint": "^7.5.0",
     "eslint-config-airbnb-base": "^14.2.0",
     "eslint-config-standard": "^14.1.1",
@@ -42,6 +52,7 @@
     "eslint-plugin-promise": "^4.2.1",
     "eslint-plugin-standard": "^4.0.1",
     "nodemon": "^2.0.4",
+    "pkg": "^4.4.9",
     "ts-node": "^8.10.2",
     "typescript": "^3.9.7"
   }

resources/flaresolverr_logo.svg

@@ -0,0 +1,180 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="256"
+   height="256"
+   viewBox="0 0 256 256"
+   id="svg2"
+   version="1.1"
+   inkscape:version="0.91 r13725"
+   sodipodi:docname="flaresolverr_logo.svg"
+   inkscape:export-filename="C:\Users\Diego\Desktop\flaresolverr_logo.png"
+   inkscape:export-xdpi="90"
+   inkscape:export-ydpi="90">
+  <defs
+     id="defs4" />
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="0.64"
+     inkscape:cx="-88.263072"
+     inkscape:cy="-93.571587"
+     inkscape:document-units="px"
+     inkscape:current-layer="layer1"
+     showgrid="false"
+     inkscape:window-width="2560"
+     inkscape:window-height="1377"
+     inkscape:window-x="-8"
+     inkscape:window-y="-8"
+     inkscape:window-maximized="1"
+     units="px"
+     showborder="true" />
+  <metadata
+     id="metadata7">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Capa 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(0,-796.36219)">
+    <g
+       id="g4177"
+       transform="matrix(0.51436047,0,0,0.59495735,-334.60687,650.43877)">
+      <g
+         id="g4141" />
+      <g
+         id="g4143" />
+      <g
+         id="g4145" />
+      <g
+         id="g4147" />
+      <g
+         id="g4149" />
+      <g
+         id="g4151" />
+      <g
+         id="g4153" />
+      <g
+         id="g4155" />
+      <g
+         id="g4157" />
+      <g
+         id="g4159" />
+      <g
+         id="g4161" />
+      <g
+         id="g4163" />
+      <g
+         id="g4165" />
+      <g
+         id="g4167" />
+      <g
+         id="g4169" />
+      <g
+         id="g4263"
+         transform="matrix(0.94954959,0,0,0.94954959,-111.49858,393.65111)">
+        <g
+           id="g4269" />
+        <g
+           id="g4342"
+           transform="translate(736.24631,-345.97247)">
+          <path
+             style="fill:#9dc6fb"
+             inkscape:connector-curvature="0"
+             d="m 584.32729,454.42324 c -0.995,-51.995 -44.49,-93.257 -96.488,-91.376 -7.616,0.273 -14.792,-3.862 -18.446,-10.55 -22.605,-41.376 -66.519,-69.441 -116.989,-69.441 -51.757,0 -96.596,29.528 -118.647,72.648 -6.423,12.56 -19.224,10.9 -24.689,10.9 -40.126,0 -74.199,25.852 -86.512,61.804 -2.046,5.973 -6.938,10.463 -12.894,12.556 -22.389998,7.87 -38.250998,29.605 -37.275998,54.902 1.163,30.174 26.849,53.631 57.044998,53.631 l 359.817,0 c 52.291,0 96.08,-42.793 95.079,-95.074 z"
+             id="path4285" />
+          <path
+             style="fill:#80b4fb"
+             inkscape:connector-curvature="0"
+             d="m 190.59629,495.86724 c -0.975,-25.298 14.885,-47.033 37.276,-54.902 5.956,-2.094 10.848,-6.584 12.894,-12.556 12.313,-35.952 46.385,-61.804 86.512,-61.804 5.465,0 18.265,1.66 24.688,-10.9 13.005,-25.43 33.94,-46.125 59.541,-58.832 -17.812,-8.834 -37.873,-13.816 -59.103,-13.816 -51.757,0 -96.596,29.528 -118.647,72.648 -6.423,12.56 -19.224,10.9 -24.689,10.9 -40.126,0 -74.199,25.852 -86.512,61.804 -2.046,5.973 -6.938,10.463 -12.894,12.556 -22.389998,7.87 -38.250998,29.605 -37.275998,54.902 1.163,30.174 26.849,53.63 57.044998,53.63 l 118.21,0 c -30.196,0 -55.881,-23.457 -57.045,-53.63 z"
+             id="path4287" />
+        </g>
+      </g>
+    </g>
+    <g
+       id="g4241"
+       transform="matrix(0.1453379,0,0,0.1453379,47.012211,854.83732)"
+       style="fill:#4d4d4d">
+      <g
+         id="g4197"
+         style="fill:#4d4d4d;fill-opacity:1">
+        <path
+           id="path4201"
+           d="m 867.699,356.238 -31.5,-26.6 c -9.699,-8.2 -24,-7.8 -33.199,0.9 l -17.4,16.3 c -14.699,-7.1 -30.299,-12.1 -46.4,-15 l -4.898,-24 c -2.5,-12.4 -14,-21 -26.602,-20 l -41.1,3.5 c -12.6,1.1 -22.5,11.4 -22.9,24.1 l -0.799,24.4 c -15.801,5.7 -30.701,13.5 -44.301,23.3 l -20.799,-13.8 c -10.602,-7 -24.701,-5 -32.9,4.7 l -26.6,31.7 c -8.201,9.7 -7.801,24 0.898,33.2 l 18.201,19.399 c -6.301,14.2 -10.801,29.101 -13.4,44.4 l -26,5.3 c -12.4,2.5 -21,14 -20,26.601 l 3.5,41.1 c 1.1,12.6 11.4,22.5 24.1,22.9 l 28.1,0.899 c 5.102,13.4 11.801,26.101 19.9,38 l -15.699,23.7 c -7,10.6 -5,24.7 4.699,32.9 l 31.5,26.6 c 9.701,8.2 24,7.8 33.201,-0.9 l 20.6,-19.3 c 13.5,6.3 27.699,11 42.299,13.8 l 5.701,28.2 c 2.5,12.4 14,21 26.6,20 l 41.1,-3.5 c 12.6,-1.1 22.5,-11.399 22.9,-24.1 l 0.9,-27.601 c 15,-5.3 29.199,-12.5 42.299,-21.399 l 22.701,15 c 10.6,7 24.699,5 32.9,-4.7 l 26.6,-31.5 c 8.199,-9.7 7.799,-24 -0.9,-33.2 L 872.7,592.138 c 6.701,-14.2 11.602,-29.2 14.4,-44.601 l 25,-5.1 c 12.4,-2.5 21,-14 20,-26.601 l -3.5,-41.1 c -1.1,-12.6 -11.4,-22.5 -24.1,-22.9 l -25.1,-0.8 c -5.201,-14.6 -12.201,-28.399 -20.9,-41.2 l 13.699,-20.6 c 7.201,-10.598 5.201,-24.798 -4.5,-32.998 z M 712.801,593.837 c -44.4,3.801 -83.602,-29.3 -87.301,-73.699 -3.801,-44.4 29.301,-83.601 73.699,-87.301 44.4,-3.8 83.602,29.301 87.301,73.7 3.801,44.401 -29.301,83.601 -73.699,87.3 z"
+           inkscape:connector-curvature="0"
+           style="fill:#4d4d4d;fill-opacity:1" />
+        <path
+           id="path4203"
+           d="m 205,704.438 c -12.6,1.3 -22.3,11.899 -22.4,24.6 l -0.3,25.3 c -0.2,12.7 9.2,23.5 21.8,25.101 l 18.6,2.399 c 3.1,11.301 7.5,22.101 13.2,32.301 l -12,14.8 c -8,9.899 -7.4,24.1 1.5,33.2 l 17.7,18.1 c 8.9,9.1 23.1,10.1 33.2,2.3 l 14.899,-11.5 c 10.5,6.2 21.601,11.101 33.2,14.5 l 2,19.2 c 1.3,12.6 11.9,22.3 24.6,22.4 l 25.301,0.3 c 12.699,0.2 23.5,-9.2 25.1,-21.8 l 2.3,-18.2 c 12.601,-3.101 24.601,-7.8 36,-14 l 14,11.3 c 9.9,8 24.101,7.4 33.201,-1.5 l 18.1,-17.7 c 9.1,-8.899 10.1,-23.1 2.301,-33.2 L 496.6,818.438 c 6.6,-11 11.701,-22.7 15.201,-35 l 16.6,-1.7 c 12.6,-1.3 22.299,-11.9 22.4,-24.6 l 0.299,-25.301 c 0.201,-12.699 -9.199,-23.5 -21.799,-25.1 l -16.201,-2.1 c -3.1,-12.2 -7.699,-24 -13.699,-35 l 10.1,-12.4 c 8,-9.9 7.4,-24.1 -1.5,-33.2 l -17.699,-18.1 c -8.9,-9.101 -23.102,-10.101 -33.201,-2.3 l -12.101,9.3 c -11.399,-6.9 -23.6,-12.2 -36.399,-15.8 L 407,581.437 c -1.3,-12.601 -11.899,-22.3 -24.6,-22.4 l -25.3,-0.3 c -12.7,-0.2 -23.5,9.2 -25.101,21.8 l -2,15.601 c -13.199,3.399 -25.899,8.6 -37.699,15.399 l -12.5,-10.2 c -9.9,-8 -24.101,-7.399 -33.201,1.5 l -18.2,17.801 c -9.1,8.899 -10.1,23.1 -2.3,33.199 l 10.7,13.801 c -6.2,11 -11.1,22.699 -14.3,35 l -17.499,1.8 z m 163.3,-28.601 c 36.3,0.4 65.399,30.301 65,66.601 -0.4,36.3 -30.301,65.399 -66.601,65 -36.3,-0.4 -65.399,-30.3 -65,-66.601 0.401,-36.299 30.301,-65.399 66.601,-65 z"
+           inkscape:connector-curvature="0"
+           style="fill:#4d4d4d;fill-opacity:1" />
+      </g>
+      <g
+         id="g4205"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4207"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4209"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4211"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4213"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4215"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4217"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4219"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4221"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4223"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4225"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4227"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4229"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4231"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4233"
+         style="fill:#4d4d4d" />
+    </g>
+  </g>
+</svg>

src/captcha/hcaptcha-solver.ts

@@ -12,8 +12,7 @@ import { SolverOptions } from '.'
 
 export default async function solve({ url }: SolverOptions): Promise<string> {
   try {
-    const token = await solveCaptcha(url)
-    return token
+    return await solveCaptcha(url)
   } catch (e) {
     console.error(e)
     return null

src/captcha/index.ts

@@ -1,3 +1,5 @@
+import log from "../log";
+
 export enum CaptchaType {
   re = 'reCaptcha',
   h = 'hCaptcha'
@@ -16,7 +18,9 @@ const captchaSolvers: { [key: string]: Solver } = {}
 export default (): Solver => {
   const method = process.env.CAPTCHA_SOLVER
 
-  if (!method) { return null }
+  if (!method || method.toLowerCase() == 'none') {
+    return null;
+  }
 
   if (!(method in captchaSolvers)) {
     try {
@@ -26,10 +30,12 @@ export default (): Solver => {
         throw Error(`The solver '${method}' is not a valid captcha solving method.`)
       } else {
         console.error(e)
-        throw Error(`An error occured loading the solver '${method}'.`)
+        throw Error(`An error occurred loading the solver '${method}'.`)
       }
     }
   }
 
+  log.info(`Using '${method}' to solve the captcha.`);
+
   return captchaSolvers[method]
 }

src/index.ts

@@ -1,12 +1,55 @@
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
 import log from './log'
 import { createServer, IncomingMessage, ServerResponse } from 'http';
 import { RequestContext } from './types'
 import Router, { BaseAPICall } from './routes'
+import getCaptchaSolver from "./captcha";
+import sessions from "./session";
+import {v1 as UUIDv1} from "uuid";
 
 const version: string = "v" + require('../package.json').version
 const serverPort: number = Number(process.env.PORT) || 8191
 const serverHost: string = process.env.HOST || '0.0.0.0'
 
+function validateEnvironmentVariables() {
+  // ip and port variables are validated by nodejs
+  if (process.env.LOG_LEVEL && ['error', 'warn', 'info', 'verbose', 'debug'].indexOf(process.env.LOG_LEVEL) == -1) {
+    log.error(`The environment variable 'LOG_LEVEL' is wrong. Check the documentation.`);
+    process.exit(1);
+  }
+  if (process.env.LOG_HTML && ['true', 'false'].indexOf(process.env.LOG_HTML) == -1) {
+    log.error(`The environment variable 'LOG_HTML' is wrong. Check the documentation.`);
+    process.exit(1);
+  }
+  if (process.env.HEADLESS && ['true', 'false'].indexOf(process.env.HEADLESS) == -1) {
+    log.error(`The environment variable 'HEADLESS' is wrong. Check the documentation.`);
+    process.exit(1);
+  }
+  try {
+    getCaptchaSolver();
+  } catch (e) {
+    log.error(`The environment variable 'CAPTCHA_SOLVER' is wrong. ${e.message}`);
+    process.exit(1);
+  }
+}
+
+async function testChromeInstallation() {
+  log.debug("Testing Chrome installation...")
+  // create a temporary file for testing
+  const filePath = path.join(os.tmpdir(), 'flaresolverr.txt')
+  fs.writeFileSync(filePath, 'flaresolverr');
+  // launch the browser
+  const url = `file://${filePath}`;
+  const session = await sessions.create(UUIDv1(), {
+    userAgent: null,
+    oneTimeSession: true
+  })
+  const page = await session.browser.newPage()
+  await page.goto(url, { waitUntil: 'domcontentloaded' })
+  log.debug("Test successful.")
+}
 
 function errorResponse(errorMsg: string, res: ServerResponse, startTimestamp: number) {
   log.error(errorMsg)
@@ -64,51 +107,67 @@ function validateIncomingRequest(ctx: RequestContext, params: BaseAPICall) {
   return true
 }
 
-createServer((req: IncomingMessage, res: ServerResponse) => {
-  const startTimestamp = Date.now()
+// init
+log.info(`FlareSolverr ${version}`);
+log.debug('Debug log enabled');
+validateEnvironmentVariables();
+testChromeInstallation().then(r =>
+  createServer((req: IncomingMessage, res: ServerResponse) => {
+    const startTimestamp = Date.now()
 
-  // count the request for the log prefix
-  log.incRequests()
-  log.info(`Incoming request: ${req.method} ${req.url}`)
-
-  // show welcome message
-  if (req.url == '/') {
-    successResponse("FlareSolverr is ready!", null, res, startTimestamp);
-    return;
-  }
-
-  // get request body
-  const bodyParts: any[] = []
-  req.on('data', chunk => {
-    bodyParts.push(chunk)
-  }).on('end', () => {
-    // parse params
-    const body = Buffer.concat(bodyParts).toString()
-    let params: BaseAPICall = null
-    try {
-      params = JSON.parse(body)
-    } catch (err) {
-      errorResponse('Body must be in JSON format', res, startTimestamp)
-      return
+    // health endpoint. this endpoint is special because it doesn't print traces
+    if (req.url == '/health') {
+      res.writeHead(200, {
+        'Content-Type': 'application/json'
+      })
+      res.write(JSON.stringify({"status": "ok"}))
+      res.end()
+      return;
     }
 
-    const ctx: RequestContext = {
-      req,
-      res,
-      startTimestamp,
-      errorResponse: (msg) => errorResponse(msg, res, startTimestamp),
-      successResponse: (msg, extendedProperties) => successResponse(msg, extendedProperties, res, startTimestamp)
+    // count the request for the log prefix
+    log.incRequests()
+    log.info(`Incoming request: ${req.method} ${req.url}`)
+
+    // show welcome message
+    if (req.url == '/') {
+      successResponse("FlareSolverr is ready!", null, res, startTimestamp);
+      return;
     }
 
-    // validate params
-    if (!validateIncomingRequest(ctx, params)) { return }
+    // get request body
+    const bodyParts: any[] = []
+    req.on('data', chunk => {
+      bodyParts.push(chunk)
+    }).on('end', () => {
+      // parse params
+      const body = Buffer.concat(bodyParts).toString()
+      let params: BaseAPICall = null
+      try {
+        params = JSON.parse(body)
+      } catch (err) {
+        errorResponse('Body must be in JSON format', res, startTimestamp)
+        return
+      }
 
-    // process request
-    Router(ctx, params).catch(e => {
-      console.error(e)
-      ctx.errorResponse(e.message)
+      const ctx: RequestContext = {
+        req,
+        res,
+        startTimestamp,
+        errorResponse: (msg) => errorResponse(msg, res, startTimestamp),
+        successResponse: (msg, extendedProperties) => successResponse(msg, extendedProperties, res, startTimestamp)
+      }
+
+      // validate params
+      if (!validateIncomingRequest(ctx, params)) { return }
+
+      // process request
+      Router(ctx, params).catch(e => {
+        console.error(e)
+        ctx.errorResponse(e.message)
+      })
     })
+  }).listen(serverPort, serverHost, () => {
+    log.info(`Listening on http://${serverHost}:${serverPort}`);
   })
-}).listen(serverPort, serverHost, () => {
-  log.info(`FlareSolverr ${version} listening on http://${serverHost}:${serverPort}`)
-})
+)

src/log.ts

@@ -1,6 +1,6 @@
 let requests = 0
 
-const LOG_HTML: boolean = Boolean(process.env.LOG_HTML) || false
+const LOG_HTML: boolean = process.env.LOG_HTML == 'true';
 
 export default {
   incRequests: () => { requests++ },
@@ -9,10 +9,10 @@ export default {
       this.debug(html)
   },
   ...require('console-log-level')(
-    {
-      level: process.env.LOG_LEVEL || 'info',
+    {level: process.env.LOG_LEVEL || 'info',
       prefix(level: string) {
-        return `${new Date().toISOString()} ${level.toUpperCase()} REQ-${requests}`
+        const req = (requests > 0) ? ` REQ-${requests}` : '';
+        return `${new Date().toISOString()} ${level.toUpperCase()}${req}`
       }
     }
   )

src/providers/cloudflare.ts

@@ -0,0 +1,164 @@
+import {Response} from 'puppeteer'
+import {Page} from "puppeteer-extra/dist/puppeteer";
+
+import log from "../log";
+import getCaptchaSolver, {CaptchaType} from "../captcha";
+
+/**
+ *  This class contains the logic to solve protections provided by CloudFlare
+**/
+
+const CHALLENGE_SELECTORS = ['#trk_jschal_js', '.ray_id', '.attack-box'];
+const TOKEN_INPUT_NAMES = ['g-recaptcha-response', 'h-captcha-response'];
+
+export default async function resolveChallenge(url: string, page: Page, response: Response): Promise<Response> {
+
+  // look for challenge and return fast if not detected
+  if (!response.headers().server.startsWith('cloudflare')) {
+    log.info('Cloudflare not detected');
+    return response;
+  }
+  log.info('Cloudflare detected');
+
+  if (await page.$('.cf-error-code')) {
+    throw new Error('Cloudflare has blocked this request (Code 1020 Detected).')
+  }
+
+  let selectorFoundCount = 0;
+  if (response.status() > 400) {
+    // detect cloudflare wait 5s
+    for (const selector of CHALLENGE_SELECTORS) {
+      const cfChallengeElem = await page.$(selector)
+      if (cfChallengeElem) {
+        selectorFoundCount++
+        log.debug(`Javascript challenge element '${selector}' detected.`)
+        log.debug('Waiting for Cloudflare challenge...')
+
+        while (true) {
+          await page.waitFor(1000)
+          try {
+            // catch exception timeout in waitForNavigation
+            response = await page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 5000 })
+          } catch (error) { }
+
+          try {
+            // catch Execution context was destroyed
+            const cfChallengeElem = await page.$(selector)
+            if (!cfChallengeElem) { break }
+            log.debug('Found challenge element again...')
+          } catch (error)
+          { }
+
+          response = await page.reload({ waitUntil: 'domcontentloaded' })
+          log.debug('Page reloaded.')
+          log.html(await page.content())
+        }
+
+        log.debug('Validating HTML code...')
+        break
+      } else {
+        log.debug(`No '${selector}' challenge element detected.`)
+      }
+    }
+    log.debug("Javascript challenge selectors found: " + selectorFoundCount + ", total selectors: " + CHALLENGE_SELECTORS.length)
+  } else {
+    // some sites use cloudflare but there is no challenge
+    log.debug(`Javascript challenge not detected. Status code: ${response.status()}`);
+    selectorFoundCount = 1;
+  }
+
+  // it seems some captcha pages return 200 sometimes
+  if (await page.$('input[name="cf_captcha_kind"]')) {
+    log.info('Captcha challenge detected.');
+    const captchaSolver = getCaptchaSolver()
+    if (captchaSolver) {
+      const captchaStartTimestamp = Date.now()
+      const challengeForm = await page.$('#challenge-form')
+      if (challengeForm) {
+        const captchaTypeElm = await page.$('input[name="cf_captcha_kind"]')
+        const cfCaptchaType: string = await captchaTypeElm.evaluate((e: any) => e.value)
+        const captchaType: CaptchaType = (CaptchaType as any)[cfCaptchaType]
+        if (!captchaType) {
+          throw new Error('Unknown captcha type!');
+        }
+
+        let sitekey = null
+        if (captchaType != 'hCaptcha' && process.env.CAPTCHA_SOLVER != 'hcaptcha-solver') {
+          const sitekeyElem = await page.$('*[data-sitekey]')
+          if (!sitekeyElem) {
+            throw new Error('Could not find sitekey!');
+          }
+          sitekey = await sitekeyElem.evaluate((e) => e.getAttribute('data-sitekey'))
+        }
+
+        log.info('Waiting to receive captcha token to bypass challenge...')
+        const token = await captchaSolver({
+          url,
+          sitekey,
+          type: captchaType
+        })
+        log.debug(`Token received: ${token}`);
+        if (!token) {
+          throw new Error('Token solver failed to return a token.')
+        }
+
+        let responseFieldsFoundCount = 0;
+        for (const name of TOKEN_INPUT_NAMES) {
+          const input = await page.$(`textarea[name="${name}"]`)
+          if (input) {
+            responseFieldsFoundCount ++;
+            log.debug(`Challenge response field '${name}' found in challenge form.`);
+            await input.evaluate((e: HTMLTextAreaElement, token) => { e.value = token }, token);
+          }
+        }
+        if (responseFieldsFoundCount == 0) {
+          throw new Error('Challenge response field not found in challenge form.');
+        }
+
+        // ignore preset event listeners on the form
+        await page.evaluate(() => {
+          window.addEventListener('submit', (e) => { e.stopPropagation() }, true)
+        })
+
+        // it seems some sites obfuscate their challenge forms
+        // TODO: look into how they do it and come up with a more solid solution
+        try {
+          // this element is added with js and we want to wait for all the js to load before submitting
+          await page.waitForSelector('#challenge-form', { timeout: 10000 })
+        } catch (err) {
+          throw new Error("No '#challenge-form' element detected.");
+        }
+
+        // calculates the time it took to solve the captcha
+        const captchaSolveTotalTime = Date.now() - captchaStartTimestamp
+
+        // generates a random wait time
+        const randomWaitTime = (Math.floor(Math.random() * 10) + 10) * 1000
+
+        // waits, if any, time remaining to appear human but stay as fast as possible
+        const timeLeft = randomWaitTime - captchaSolveTotalTime
+        if (timeLeft > 0) {
+          log.debug(`Waiting for '${timeLeft}' milliseconds.`);
+          await page.waitFor(timeLeft);
+        }
+
+        // submit captcha response
+        challengeForm.evaluate((e: HTMLFormElement) => e.submit())
+        response = await page.waitForNavigation({ waitUntil: 'domcontentloaded' })
+
+        if (await page.$('input[name="cf_captcha_kind"]')) {
+          throw new Error('Captcha service failed to solve the challenge.');
+        }
+      }
+    } else {
+      throw new Error('Captcha detected but no automatic solver is configured.');
+    }
+  } else {
+    if (selectorFoundCount == 0)
+    {
+      throw new Error('No challenge selectors found, unable to proceed')
+    }
+  }
+
+  return response;
+}

src/routes.ts

@@ -1,12 +1,12 @@
 import { v1 as UUIDv1 } from 'uuid'
+import { SetCookie, Request, Response, Headers, HttpMethod, Overrides } from 'puppeteer'
+import { Page, Browser } from "puppeteer-extra/dist/puppeteer";
+const Timeout = require('await-timeout');
+
+import log from './log'
 import sessions, { SessionsCacheItem } from './session'
 import { RequestContext } from './types'
-import log from './log'
-import { SetCookie, Request, Headers, HttpMethod, Overrides, Cookie } from 'puppeteer'
-import { TimeoutError } from 'puppeteer/Errors'
-import getCaptchaSolver, { CaptchaType } from './captcha'
-import * as Puppeteer from "puppeteer-extra/dist/puppeteer";
-const Timeout = require('await-timeout');
+import cloudflareProvider from './providers/cloudflare';
 
 export interface BaseAPICall {
   cmd: string
@@ -69,58 +69,10 @@ type OverridesProps =
   'postData' |
   'headers'
 
-// We always set a Windows User-Agent because ARM builds are detected by CloudFlare
+// We always set a Windows User-Agent because ARM builds are detected by Cloudflare
 const DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36"
-const CHALLENGE_SELECTORS = ['#trk_jschal_js', '.ray_id', '.attack-box']
-const TOKEN_INPUT_NAMES = ['g-recaptcha-response', 'h-captcha-response']
 
-async function interceptResponse(page: Puppeteer.Page, callback: (payload: ChallengeResolutionT) => any) {
-  const client = await page.target().createCDPSession();
-  await client.send('Fetch.enable', {
-    patterns: [
-      {
-        urlPattern: '*',
-        resourceType: 'Document',
-        requestStage: 'Response',
-      },
-    ],
-  });
-
-  client.on('Fetch.requestPaused', async (e) => {
-    log.debug('Fetch.requestPaused. Checking if the response has valid cookies')
-    let headers = e.responseHeaders || []
-
-    let cookies = await page.cookies();
-    log.debug(cookies)
-
-    if (cookies.filter((c: Cookie) => c.name === 'cf_clearance').length > 0) {
-      log.debug('Aborting request and return cookies. valid cookies found')
-      await client.send('Fetch.failRequest', {requestId: e.requestId, errorReason: 'Aborted'})
-
-      let status = 'ok'
-      let message = ''
-      const payload: ChallengeResolutionT = {
-        status,
-        message,
-        result: {
-          url: page.url(),
-          status: e.status,
-          headers: headers.reduce((a: any, x: { name: any; value: any }) => ({ ...a, [x.name]: x.value }), {}),
-          response: null,
-          cookies: cookies,
-          userAgent: ''
-        }
-      }
-
-      callback(payload);
-    } else {
-      log.debug('Continuing request. no valid cookies found')
-      await client.send('Fetch.continueRequest', {requestId: e.requestId})
-    }
-  });
-}
-
-async function resolveChallengeWithTimeout(ctx: RequestContext, params: BaseRequestAPICall, page: Puppeteer.Page) {
+async function resolveChallengeWithTimeout(ctx: RequestContext, params: BaseRequestAPICall, page: Page) {
   const maxTimeout = params.maxTimeout || 60000
   const timer = new Timeout();
   try {
@@ -134,7 +86,7 @@ async function resolveChallengeWithTimeout(ctx: RequestContext, params: BaseRequ
   }
 }
 
-async function resolveChallenge(ctx: RequestContext, { url, proxy, download, returnOnlyCookies }: BaseRequestAPICall, page: Puppeteer.Page): Promise<ChallengeResolutionT | void> {
+async function resolveChallenge(ctx: RequestContext, { url, proxy, download, returnOnlyCookies }: BaseRequestAPICall, page: Page): Promise<ChallengeResolutionT | void> {
 
   let status = 'ok'
   let message = ''
@@ -146,160 +98,15 @@ async function resolveChallenge(ctx: RequestContext, { url, proxy, download, ret
   }
 
   log.debug(`Navigating to... ${url}`)
-  let response = await page.goto(url, { waitUntil: 'domcontentloaded' })
-
+  let response: Response = await page.goto(url, { waitUntil: 'domcontentloaded' })
   log.html(await page.content())
 
-  // look for challenge
-  if (response.headers().server.startsWith('cloudflare')) {
-    log.info('Cloudflare detected')
-
-    if (await page.$('.cf-error-code')) {
-      await page.close()
-      return ctx.errorResponse('Cloudflare has blocked this request (Code 1020 Detected).')
-    }
-
-    if (response.status() > 400) {
-      // detect cloudflare wait 5s
-      let selectorFoundCount = 0
-      for (const selector of CHALLENGE_SELECTORS) {
-        const cfChallengeElem = await page.$(selector)
-        if (cfChallengeElem) {
-          selectorFoundCount++
-          log.debug(`'${selector}' challenge element detected.`)
-          log.debug('Waiting for Cloudflare challenge...')
-
-          let interceptingResult: ChallengeResolutionT;
-          if (returnOnlyCookies) { //If we just want to get the cookies, intercept the response before we get the content/body (just cookies and headers)
-            await interceptResponse(page, async function(payload){
-              interceptingResult = payload;
-            });
-          }
-
-          while (true) {
-            await page.waitFor(1000)
-            try {
-              // catch exception timeout in waitForNavigation
-              response = await page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 5000 })
-            } catch (error) { }
-
-            if (returnOnlyCookies && interceptingResult) {
-              await page.close();
-              return interceptingResult;
-            }
-
-            try {
-              // catch Execution context was destroyed
-              const cfChallengeElem = await page.$(selector)
-              if (!cfChallengeElem) { break }
-              log.debug('Found challenge element again...')
-            } catch (error)
-            { }
-
-            response = await page.reload({ waitUntil: 'domcontentloaded' })
-            log.debug('Reloaded page...')
-            log.html(await page.content())
-          }
-
-          log.debug('Validating HTML code...')
-          break
-        } else {
-          log.debug(`No '${selector}' challenge element detected.`)
-        }
-      }
-      log.debug("Number of selector found: " + selectorFoundCount + ", total selector: " + CHALLENGE_SELECTORS.length)
-      if (selectorFoundCount == 0)
-      {
-        await page.close()
-        return ctx.errorResponse('No challenge selectors found, unable to proceed')
-      }
-    }
-
-    // it seems some captcha pages return 200 sometimes
-    if (await page.$('input[name="cf_captcha_kind"]')) {
-      const captchaSolver = getCaptchaSolver()
-      if (captchaSolver) {
-        const captchaStartTimestamp = Date.now()
-        const challengeForm = await page.$('#challenge-form')
-        if (challengeForm) {
-          const captchaTypeElm = await page.$('input[name="cf_captcha_kind"]')
-          const cfCaptchaType: string = await captchaTypeElm.evaluate((e: any) => e.value)
-          const captchaType: CaptchaType = (CaptchaType as any)[cfCaptchaType]
-          if (!captchaType) { return ctx.errorResponse('Unknown captcha type!') }
-
-          let sitekey = null
-          if (captchaType != 'hCaptcha' && process.env.CAPTCHA_SOLVER != 'hcaptcha-solver') {
-            const sitekeyElem = await page.$('*[data-sitekey]')
-            if (!sitekeyElem) { return ctx.errorResponse('Could not find sitekey!') }
-            sitekey = await sitekeyElem.evaluate((e) => e.getAttribute('data-sitekey'))
-          }
-
-          log.info('Waiting to receive captcha token to bypass challenge...')
-          const token = await captchaSolver({
-            url,
-            sitekey,
-            type: captchaType
-          })
-
-          if (!token) {
-            await page.close()
-            return ctx.errorResponse('Token solver failed to return a token.')
-          }
-
-          for (const name of TOKEN_INPUT_NAMES) {
-            const input = await page.$(`textarea[name="${name}"]`)
-            if (input) { await input.evaluate((e: HTMLTextAreaElement, token) => { e.value = token }, token) }
-          }
-
-          // ignore preset event listeners on the form
-          await page.evaluate(() => {
-            window.addEventListener('submit', (e) => { event.stopPropagation() }, true)
-          })
-
-          // it seems some sites obfuscate their challenge forms
-          // TODO: look into how they do it and come up with a more solid solution
-          try {
-            // this element is added with js and we want to wait for all the js to load before submitting
-            await page.waitForSelector('#challenge-form [type=submit]', { timeout: 5000 })
-          } catch (err) {
-            if (err instanceof TimeoutError) {
-              log.debug(`No '#challenge-form [type=submit]' element detected.`)
-            }
-          }
-
-          // calculates the time it took to solve the captcha
-          const captchaSolveTotalTime = Date.now() - captchaStartTimestamp
-
-          // generates a random wait time
-          const randomWaitTime = (Math.floor(Math.random() * 20) + 10) * 1000
-
-          // waits, if any, time remaining to appear human but stay as fast as possible
-          const timeLeft = randomWaitTime - captchaSolveTotalTime
-          if (timeLeft > 0) { await page.waitFor(timeLeft) }
-
-          let interceptingResult: ChallengeResolutionT;
-          if (returnOnlyCookies) { //If we just want to get the cookies, intercept the response before we get the content/body (just cookies and headers)
-            await interceptResponse(page, async function(payload){
-              interceptingResult = payload;
-            });
-          }
-
-          // submit captcha response
-          challengeForm.evaluate((e: HTMLFormElement) => e.submit())
-          response = await page.waitForNavigation({ waitUntil: 'domcontentloaded' })
-
-          if (returnOnlyCookies && interceptingResult) {
-            await page.close();
-            return interceptingResult;
-          }
-        }
-      } else {
-        status = 'warning'
-        message = 'Captcha detected but no automatic solver is configured.'
-      }
-    }
-
-    log.debug("Response is: " + response.status())
+  // Detect protection services and solve challenges
+  try {
+    response = await cloudflareProvider(url, page, response);
+  } catch (e) {
+    status = "error";
+    message = "Cloudflare " + e.toString();
   }
 
   const payload: ChallengeResolutionT = {
@@ -315,14 +122,19 @@ async function resolveChallenge(ctx: RequestContext, { url, proxy, download, ret
     }
   }
 
-  if (download) {
-    // for some reason we get an error unless we reload the page
-    // has something to do with a stale buffer and this is the quickest
-    // fix since I am short on time
-    response = await page.goto(url, { waitUntil: 'domcontentloaded' })
-    payload.result.response = (await response.buffer()).toString('base64')
+  if (returnOnlyCookies) {
+    payload.result.headers = null;
+    payload.result.userAgent = null;
   } else {
-    payload.result.response = await page.content()
+    if (download) {
+      // for some reason we get an error unless we reload the page
+      // has something to do with a stale buffer and this is the quickest
+      // fix since I am short on time
+      response = await page.goto(url, { waitUntil: 'domcontentloaded' })
+      payload.result.response = (await response.buffer()).toString('base64')
+    } else {
+      payload.result.response = await page.content()
+    }
   }
 
   // make sure the page is closed because if it isn't and error will be thrown
@@ -342,7 +154,7 @@ function mergeSessionWithParams({ defaults }: SessionsCacheItem, params: BaseReq
   return copy
 }
 
-async function setupPage(ctx: RequestContext, params: BaseRequestAPICall, browser: Puppeteer.Browser): Promise<Puppeteer.Page> {
+async function setupPage(ctx: RequestContext, params: BaseRequestAPICall, browser: Browser): Promise<Page> {
   const page = await browser.newPage()
 
   // merge session defaults with params
@@ -368,18 +180,17 @@ async function setupPage(ctx: RequestContext, params: BaseRequestAPICall, browse
   }
 
   if (headers) {
-    log.debug(`Adding custom headers: ${JSON.stringify(headers, null, 2)}`,)
+    log.debug(`Adding custom headers: ${JSON.stringify(headers)}`)
     overrideResolvers.headers = request => Object.assign(request.headers(), headers)
   }
 
   if (cookies) {
-    log.debug(`Setting custom cookies: ${JSON.stringify(cookies, null, 2)}`,)
+    log.debug(`Setting custom cookies: ${JSON.stringify(cookies)}`)
     await page.setCookie(...cookies)
   }
 
   // if any keys have been set on the object
   if (Object.keys(overrideResolvers).length > 0) {
-    log.debug(overrideResolvers)
     let callbackRunOnce = false
     const callback = (request: Request) => {
 
@@ -396,8 +207,7 @@ async function setupPage(ctx: RequestContext, params: BaseRequestAPICall, browse
         overrides[key] = overrideResolvers[key](request)
       });
 
-      log.debug(overrides)
-
+      log.debug(`Overrides: ${JSON.stringify(overrides)}`)
       request.continue(overrides)
     }
 
@@ -441,7 +251,9 @@ const browserRequest = async (ctx: RequestContext, params: BaseRequestAPICall) =
     log.error(error)
     return ctx.errorResponse("Unable to process browser request. Error: " + error)
   } finally {
-    if (oneTimeSession) { sessions.destroy(sessionId) }
+    if (oneTimeSession) {
+      await sessions.destroy(sessionId)
+    }
   }
 }
 
@@ -474,14 +286,6 @@ export const routes: Routes = {
 
     await browserRequest(ctx, params)
   },
-  'request.cookies': async (ctx, params: BaseRequestAPICall) => {
-    params.returnOnlyCookies = true
-    params.method = 'GET'
-    if (params.postData) {
-      return ctx.errorResponse('Cannot use "postBody" when sending a GET request.')
-    }
-    await browserRequest(ctx, params)
-  },
 }
 
 export default async function Router(ctx: RequestContext, params: BaseAPICall): Promise<void> {

src/session.ts

@@ -56,7 +56,11 @@ function prepareBrowserProfile(id: string): string {
 
 export default {
   create: async (id: string, { cookies, oneTimeSession, userAgent, headers, maxTimeout, proxy }: SessionCreateOptions): Promise<SessionsCacheItem> => {
-    let args = ['--no-sandbox', '--disable-setuid-sandbox'];
+    let args = [
+      '--no-sandbox',
+      '--disable-setuid-sandbox',
+      '--disable-dev-shm-usage' // issue #45
+    ];
     if (proxy && proxy.url) {
       args.push(`--proxy-server=${proxy.url}`);
     }
@@ -72,7 +76,13 @@ export default {
       puppeteerOptions.userDataDir = prepareBrowserProfile(id)
     }
 
-    log.debug('Launching headless browser...')
+    // if we are running inside executable binary, change chrome path
+    if (typeof (process as any).pkg !== 'undefined') {
+      const exe = process.platform === "win32" ? 'chrome.exe' : 'chrome';
+      puppeteerOptions.executablePath = path.join(path.dirname(process.execPath), 'chrome', exe)
+    }
+
+    log.debug('Launching browser...')
 
     // TODO: maybe access env variable?
     // TODO: sometimes browser instances are created and not connected to correctly.