Bump version 1.2.9

Bumps [ws](https://github.com/websockets/ws) from 7.4.1 to 7.4.6.
- [Release notes](https://github.com/websockets/ws/releases)
- [Commits](https://github.com/websockets/ws/compare/7.4.1...7.4.6)

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.dockerignore

@@ -1,6 +1,7 @@
-node_modules
+.git/
-npm-debug.log
+.github/
-Dockerfile
+.idea/
-.dockerignore
+bin/
-.git
+dist/
-.gitignore
+node_modules/
+resources/

.github/ISSUE_TEMPLATE.md

@@ -1,31 +1,25 @@
 **Please use the search bar** at the top of the page and make sure you are not creating an already submitted issue.
 Check closed issues as well, because your issue may have already been fixed.
 
-### Instruction on how to enable debug and html trace
+### How to enable debug and html traces
 
 [Follow the instructions from this wiki page](https://github.com/FlareSolverr/FlareSolverr/wiki/How-to-enable-debug-and-html-trace)
 
 ### Environment
 
-**FlareSolverr Version**:
+* **FlareSolverr version**:
-
+* **Last working FlareSolverr version**:
-**Docker**: [yes/no]
+* **Operating system**:
-
+* **Are you using Docker**: [yes/no]
-**OS**: 
+* **Are you using a proxy or VPN?** [yes/no]
-
+* **Are you using Captcha Solver:** [yes/no]
-**Last Working FlareSolverr Version**:
+* **If using captcha solver, which one:**
-
+* **URL to test this issue:**
-**Are you using a proxy or VPN?** [yes/no]
-
-**Using Captcha Solver:** [yes/no]
-
-**If using captcha solver, which one:**
 
 ### Description
 
 [List steps to reproduce the error and details on what happens and what you expected to happen]
 
-
 ### Logged Error Messages
 
 [Place any relevant error messages you noticed from the logs here.]

.github/workflows/release-docker.yml

@@ -1,4 +1,4 @@
-name: publish
+name: release-docker
 
 on:
   push:
@@ -24,7 +24,7 @@ jobs:
           tag-sha: false
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v1.0.1
       -
         name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1

.github/workflows/release.yml

@@ -29,7 +29,7 @@ jobs:
       - name: Build changelog
         id: github_changelog
         run: |
-          changelog=$(git log $(git describe --tags --abbrev=0)..HEAD --no-merges --oneline)
+          changelog=$(git log $(git tag | tail -2 | head -1)..HEAD --no-merges --oneline)
           changelog="${changelog//'%'/'%25'}"
           changelog="${changelog//$'\n'/'%0A'}"
           changelog="${changelog//$'\r'/'%0D'}"

Dockerfile

@@ -22,8 +22,8 @@ ENV PUPPETEER_PRODUCT=chrome \
     PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
 RUN npm install && \
     npm run build && \
-    rm -rf src tsconfig.json && \
+    npm prune --production && \
-    npm prune --production
+    rm -rf /home/node/.npm
 
 EXPOSE 8191
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]

README.md

@@ -4,10 +4,11 @@
 [![Docker Pulls](https://img.shields.io/docker/pulls/flaresolverr/flaresolverr)](https://hub.docker.com/r/flaresolverr/flaresolverr/)
 [![GitHub issues](https://img.shields.io/github/issues/FlareSolverr/FlareSolverr)](https://github.com/FlareSolverr/FlareSolverr/issues)
 [![GitHub pull requests](https://img.shields.io/github/issues-pr/FlareSolverr/FlareSolverr)](https://github.com/FlareSolverr/FlareSolverr/pulls)
+[![Donate PayPal](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=X5NJLLX5GLTV6&source=url)
+[![Donate Buy Me A Coffee](https://img.shields.io/badge/Donate-Buy%20me%20a%20coffee-yellow.svg)](https://www.buymeacoffee.com/ngosang)
+[![Donate Bitcoin](https://img.shields.io/badge/Donate-Bitcoin-orange.svg)](https://en.cryptobadges.io/donate/13Hcv77AdnFWEUZ9qUpoPBttQsUT7q9TTh)
 
-FlareSolverr is a proxy server to bypass Cloudflare protection
+FlareSolverr is a proxy server to bypass Cloudflare protection.
-
-:warning: This project is in beta state. Some things may not work and the API can change at any time.
 
 ## How it works
 
@@ -64,12 +65,17 @@ This is the recommended way for Windows users.
 
 ### From source code
 
-This is the recommended way for MacOS users and for developers.
+This is the recommended way for macOS users and for developers.
-* Install [NodeJS](https://nodejs.org/)
+* Install [NodeJS](https://nodejs.org/).
-* Clone this repository and open a shell in that path
+* Clone this repository and open a shell in that path.
-* Run `npm install` command to install FlareSolverr dependencies
+* Run `npm install` command to install FlareSolverr dependencies.
-* Run `npm run build` command to compile TypeScript code
+* Run `node node_modules/puppeteer/install.js` to install Chromium.
-* Run `npm start` command to start FlareSolverr
+* Run `npm run build` command to compile TypeScript code.
+* Run `npm start` command to start FlareSolverr.
+
+### Systemd service
+
+We provide an example Systemd unit file `flaresolverr.service` as reference. You have to modify the file to suit your needs: paths, user and environment variables.
 
 ## Usage
 
@@ -140,6 +146,7 @@ headers | Optional. To specify user headers.
 maxTimeout | Optional, default value 60000. Max timeout to solve the challenge in milliseconds.
 cookies | Optional. Will be used by the headless browser. Follow [this](https://github.com/puppeteer/puppeteer/blob/v3.3.0/docs/api.md#pagesetcookiecookies) format.
 returnOnlyCookies | Optional, default false. Only returns the cookies. Response data, headers and other parts of the response are removed.
+returnRawHtml | Optional, default false. The response data will be returned without JS processing. This is useful for JSON or plain text content.
 
 Example response from running the `curl` above:
 
@@ -221,12 +228,13 @@ moment there is nothing setup to do so. If this is something you need feel free
 
 Name | Default | Notes
 |--|--|--|
-LOG_LEVEL | info | Used to change the verbosity of the logging. Use `LOG_LEVEL=debug` for more information.
+LOG_LEVEL | info | Verbosity of the logging. Use `LOG_LEVEL=debug` for more information.
-LOG_HTML | false | Used for debugging. If `true` all HTML that passes through the proxy will be logged to the console in `debug` level.
+LOG_HTML | false | Only for debugging. If `true` all HTML that passes through the proxy will be logged to the console in `debug` level.
-PORT | 8191 | Change this if you already have a process running on port `8191`.
+CAPTCHA_SOLVER | none | Captcha solving method. It is used when a captcha is encountered. See the Captcha Solvers section.
-HOST | 0.0.0.0 | This shouldn't need to be messed with but if you insist, it's here!
+TZ | UTC | Timezone used in the logs and the web browser. Example: `TZ=Europe/London`.
-CAPTCHA_SOLVER | None | This is used to select which captcha solving method it used when a captcha is encountered.
+HEADLESS | true | Only for debugging. To run the web browser in headless mode or visible.
-HEADLESS | true | This is used to debug the browser by not running it in headless mode.
+PORT | 8191 | Listening port. You don't need to change this if you are running on Docker.
+HOST | 0.0.0.0 | Listening interface. You don't need to change this if you are running on Docker.
 
 Environment variables are set differently depending on the operating system. Some examples:
 * Docker: Take a look at the Docker section in this document. Environment variables can be set in the `docker-compose.yml` file or in the Docker CLI command.

build-binaries.js

@@ -13,7 +13,8 @@ const version = 'v' + require('./package.json').version;
       chromeFolder: 'chrome-linux',
       fsExec: 'flaresolverr-linux',
       fsZipExec: 'flaresolverr',
-      fsZipName: 'linux-x64'
+      fsZipName: 'linux-x64',
+      fsLicenseName: 'LICENSE'
     },
     {
       platform: 'win64',
@@ -21,7 +22,8 @@ const version = 'v' + require('./package.json').version;
       chromeFolder: 'chrome-win',
       fsExec: 'flaresolverr-win.exe',
       fsZipExec: 'flaresolverr.exe',
-      fsZipName: 'windows-x64'
+      fsZipName: 'windows-x64',
+      fsLicenseName: 'LICENSE.txt'
     }
     // TODO: this is working but changes are required in session.ts to find chrome path
     // {
@@ -30,14 +32,15 @@ const version = 'v' + require('./package.json').version;
     //   chromeFolder: 'chrome-mac',
     //   fsExec: 'flaresolverr-macos',
     //   fsZipExec: 'flaresolverr',
-    //   fsZipName: 'macos'
+    //   fsZipName: 'macos',
+    //   fsLicenseName: 'LICENSE'
     // }
   ]
 
   // generate executables
   console.log('Generating executables...')
   if (fs.existsSync('bin')) {
-    fs.rmdirSync('bin', {recursive: true})
+    fs.rmSync('bin', { recursive: true })
   }
   execSync('pkg -t node14-win-x64,node14-linux-x64 --out-path bin .')
   // execSync('pkg -t node14-win-x64,node14-mac-x64,node14-linux-x64 --out-path bin .')
@@ -70,9 +73,13 @@ const version = 'v' + require('./package.json').version;
 
     archive.pipe(output)
 
+    archive.file('LICENSE', { name: 'flaresolverr/' + os.fsLicenseName })
     archive.file('bin/' + os.fsExec, { name: 'flaresolverr/' + os.fsZipExec })
     archive.directory('bin/puppeteer/' + os.platform + '-' + os.version + '/' + os.chromeFolder, 'flaresolverr/chrome')
+    if (os.platform === 'linux') {
+      archive.file('flaresolverr.service', { name: 'flaresolverr/flaresolverr.service' })
+    }
 
-    archive.finalize()
+    await archive.finalize()
   }
 })()

docker-compose.yml

@@ -6,13 +6,10 @@ services:
     image: ghcr.io/flaresolverr/flaresolverr:latest
     container_name: flaresolverr
     environment:
-      # Used to change the verbosity of the logging
+      - LOG_LEVEL=${LOG_LEVEL:-info}
-      - LOG_LEVEL=info
+      - LOG_HTML=${LOG_HTML:-false}
-      # Enables hcaptcha-solver => https://github.com/JimmyLaurent/hcaptcha-solver
+      - CAPTCHA_SOLVER=${CAPTCHA_SOLVER:-none}
-      #- CAPTCHA_SOLVER=hcaptcha-solver
+      - TZ=Europe/London
-      # Enables CaptchaHarvester => https://github.com/NoahCardoza/CaptchaHarvester
-      #- CAPTCHA_SOLVER=harvester
-      #- HARVESTER_ENDPOINT=https://127.0.0.1:5000/token
     ports:
-      - 8191:8191
+      - "${PORT:-8191}:8191"
     restart: unless-stopped

flaresolverr.service

@@ -0,0 +1,19 @@
+[Unit]
+Description=FlareSolverr
+After=network.target
+
+[Service]
+SyslogIdentifier=flaresolverr
+Restart=always
+RestartSec=5
+Type=simple
+User=flaresolverr
+Group=flaresolverr
+Environment="LOG_LEVEL=info"
+Environment="CAPTCHA_SOLVER=none"
+WorkingDirectory=/opt/flaresolverr
+ExecStart=/opt/flaresolverr/flaresolverr
+TimeoutStopSec=30
+
+[Install]
+WantedBy=multi-user.target

package.json

@@ -1,6 +1,6 @@
 {
   "name": "flaresolverr",
-  "version": "1.2.2",
+  "version": "1.2.9",
   "description": "Proxy server to bypass Cloudflare protection.",
   "scripts": {
     "start": "node ./dist/index.js",

resources/flaresolverr_logo.svg

@@ -0,0 +1,180 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="256"
+   height="256"
+   viewBox="0 0 256 256"
+   id="svg2"
+   version="1.1"
+   inkscape:version="0.91 r13725"
+   sodipodi:docname="flaresolverr_logo.svg"
+   inkscape:export-filename="C:\Users\Diego\Desktop\flaresolverr_logo.png"
+   inkscape:export-xdpi="90"
+   inkscape:export-ydpi="90">
+  <defs
+     id="defs4" />
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="0.64"
+     inkscape:cx="-88.263072"
+     inkscape:cy="-93.571587"
+     inkscape:document-units="px"
+     inkscape:current-layer="layer1"
+     showgrid="false"
+     inkscape:window-width="2560"
+     inkscape:window-height="1377"
+     inkscape:window-x="-8"
+     inkscape:window-y="-8"
+     inkscape:window-maximized="1"
+     units="px"
+     showborder="true" />
+  <metadata
+     id="metadata7">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Capa 1"
+     inkscape:groupmode="layer"
+     id="layer1"
+     transform="translate(0,-796.36219)">
+    <g
+       id="g4177"
+       transform="matrix(0.51436047,0,0,0.59495735,-334.60687,650.43877)">
+      <g
+         id="g4141" />
+      <g
+         id="g4143" />
+      <g
+         id="g4145" />
+      <g
+         id="g4147" />
+      <g
+         id="g4149" />
+      <g
+         id="g4151" />
+      <g
+         id="g4153" />
+      <g
+         id="g4155" />
+      <g
+         id="g4157" />
+      <g
+         id="g4159" />
+      <g
+         id="g4161" />
+      <g
+         id="g4163" />
+      <g
+         id="g4165" />
+      <g
+         id="g4167" />
+      <g
+         id="g4169" />
+      <g
+         id="g4263"
+         transform="matrix(0.94954959,0,0,0.94954959,-111.49858,393.65111)">
+        <g
+           id="g4269" />
+        <g
+           id="g4342"
+           transform="translate(736.24631,-345.97247)">
+          <path
+             style="fill:#9dc6fb"
+             inkscape:connector-curvature="0"
+             d="m 584.32729,454.42324 c -0.995,-51.995 -44.49,-93.257 -96.488,-91.376 -7.616,0.273 -14.792,-3.862 -18.446,-10.55 -22.605,-41.376 -66.519,-69.441 -116.989,-69.441 -51.757,0 -96.596,29.528 -118.647,72.648 -6.423,12.56 -19.224,10.9 -24.689,10.9 -40.126,0 -74.199,25.852 -86.512,61.804 -2.046,5.973 -6.938,10.463 -12.894,12.556 -22.389998,7.87 -38.250998,29.605 -37.275998,54.902 1.163,30.174 26.849,53.631 57.044998,53.631 l 359.817,0 c 52.291,0 96.08,-42.793 95.079,-95.074 z"
+             id="path4285" />
+          <path
+             style="fill:#80b4fb"
+             inkscape:connector-curvature="0"
+             d="m 190.59629,495.86724 c -0.975,-25.298 14.885,-47.033 37.276,-54.902 5.956,-2.094 10.848,-6.584 12.894,-12.556 12.313,-35.952 46.385,-61.804 86.512,-61.804 5.465,0 18.265,1.66 24.688,-10.9 13.005,-25.43 33.94,-46.125 59.541,-58.832 -17.812,-8.834 -37.873,-13.816 -59.103,-13.816 -51.757,0 -96.596,29.528 -118.647,72.648 -6.423,12.56 -19.224,10.9 -24.689,10.9 -40.126,0 -74.199,25.852 -86.512,61.804 -2.046,5.973 -6.938,10.463 -12.894,12.556 -22.389998,7.87 -38.250998,29.605 -37.275998,54.902 1.163,30.174 26.849,53.63 57.044998,53.63 l 118.21,0 c -30.196,0 -55.881,-23.457 -57.045,-53.63 z"
+             id="path4287" />
+        </g>
+      </g>
+    </g>
+    <g
+       id="g4241"
+       transform="matrix(0.1453379,0,0,0.1453379,47.012211,854.83732)"
+       style="fill:#4d4d4d">
+      <g
+         id="g4197"
+         style="fill:#4d4d4d;fill-opacity:1">
+        <path
+           id="path4201"
+           d="m 867.699,356.238 -31.5,-26.6 c -9.699,-8.2 -24,-7.8 -33.199,0.9 l -17.4,16.3 c -14.699,-7.1 -30.299,-12.1 -46.4,-15 l -4.898,-24 c -2.5,-12.4 -14,-21 -26.602,-20 l -41.1,3.5 c -12.6,1.1 -22.5,11.4 -22.9,24.1 l -0.799,24.4 c -15.801,5.7 -30.701,13.5 -44.301,23.3 l -20.799,-13.8 c -10.602,-7 -24.701,-5 -32.9,4.7 l -26.6,31.7 c -8.201,9.7 -7.801,24 0.898,33.2 l 18.201,19.399 c -6.301,14.2 -10.801,29.101 -13.4,44.4 l -26,5.3 c -12.4,2.5 -21,14 -20,26.601 l 3.5,41.1 c 1.1,12.6 11.4,22.5 24.1,22.9 l 28.1,0.899 c 5.102,13.4 11.801,26.101 19.9,38 l -15.699,23.7 c -7,10.6 -5,24.7 4.699,32.9 l 31.5,26.6 c 9.701,8.2 24,7.8 33.201,-0.9 l 20.6,-19.3 c 13.5,6.3 27.699,11 42.299,13.8 l 5.701,28.2 c 2.5,12.4 14,21 26.6,20 l 41.1,-3.5 c 12.6,-1.1 22.5,-11.399 22.9,-24.1 l 0.9,-27.601 c 15,-5.3 29.199,-12.5 42.299,-21.399 l 22.701,15 c 10.6,7 24.699,5 32.9,-4.7 l 26.6,-31.5 c 8.199,-9.7 7.799,-24 -0.9,-33.2 L 872.7,592.138 c 6.701,-14.2 11.602,-29.2 14.4,-44.601 l 25,-5.1 c 12.4,-2.5 21,-14 20,-26.601 l -3.5,-41.1 c -1.1,-12.6 -11.4,-22.5 -24.1,-22.9 l -25.1,-0.8 c -5.201,-14.6 -12.201,-28.399 -20.9,-41.2 l 13.699,-20.6 c 7.201,-10.598 5.201,-24.798 -4.5,-32.998 z M 712.801,593.837 c -44.4,3.801 -83.602,-29.3 -87.301,-73.699 -3.801,-44.4 29.301,-83.601 73.699,-87.301 44.4,-3.8 83.602,29.301 87.301,73.7 3.801,44.401 -29.301,83.601 -73.699,87.3 z"
+           inkscape:connector-curvature="0"
+           style="fill:#4d4d4d;fill-opacity:1" />
+        <path
+           id="path4203"
+           d="m 205,704.438 c -12.6,1.3 -22.3,11.899 -22.4,24.6 l -0.3,25.3 c -0.2,12.7 9.2,23.5 21.8,25.101 l 18.6,2.399 c 3.1,11.301 7.5,22.101 13.2,32.301 l -12,14.8 c -8,9.899 -7.4,24.1 1.5,33.2 l 17.7,18.1 c 8.9,9.1 23.1,10.1 33.2,2.3 l 14.899,-11.5 c 10.5,6.2 21.601,11.101 33.2,14.5 l 2,19.2 c 1.3,12.6 11.9,22.3 24.6,22.4 l 25.301,0.3 c 12.699,0.2 23.5,-9.2 25.1,-21.8 l 2.3,-18.2 c 12.601,-3.101 24.601,-7.8 36,-14 l 14,11.3 c 9.9,8 24.101,7.4 33.201,-1.5 l 18.1,-17.7 c 9.1,-8.899 10.1,-23.1 2.301,-33.2 L 496.6,818.438 c 6.6,-11 11.701,-22.7 15.201,-35 l 16.6,-1.7 c 12.6,-1.3 22.299,-11.9 22.4,-24.6 l 0.299,-25.301 c 0.201,-12.699 -9.199,-23.5 -21.799,-25.1 l -16.201,-2.1 c -3.1,-12.2 -7.699,-24 -13.699,-35 l 10.1,-12.4 c 8,-9.9 7.4,-24.1 -1.5,-33.2 l -17.699,-18.1 c -8.9,-9.101 -23.102,-10.101 -33.201,-2.3 l -12.101,9.3 c -11.399,-6.9 -23.6,-12.2 -36.399,-15.8 L 407,581.437 c -1.3,-12.601 -11.899,-22.3 -24.6,-22.4 l -25.3,-0.3 c -12.7,-0.2 -23.5,9.2 -25.101,21.8 l -2,15.601 c -13.199,3.399 -25.899,8.6 -37.699,15.399 l -12.5,-10.2 c -9.9,-8 -24.101,-7.399 -33.201,1.5 l -18.2,17.801 c -9.1,8.899 -10.1,23.1 -2.3,33.199 l 10.7,13.801 c -6.2,11 -11.1,22.699 -14.3,35 l -17.499,1.8 z m 163.3,-28.601 c 36.3,0.4 65.399,30.301 65,66.601 -0.4,36.3 -30.301,65.399 -66.601,65 -36.3,-0.4 -65.399,-30.3 -65,-66.601 0.401,-36.299 30.301,-65.399 66.601,-65 z"
+           inkscape:connector-curvature="0"
+           style="fill:#4d4d4d;fill-opacity:1" />
+      </g>
+      <g
+         id="g4205"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4207"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4209"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4211"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4213"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4215"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4217"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4219"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4221"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4223"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4225"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4227"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4229"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4231"
+         style="fill:#4d4d4d" />
+      <g
+         id="g4233"
+         style="fill:#4d4d4d" />
+    </g>
+  </g>
+</svg>

src/captcha/hcaptcha-solver.ts

@@ -11,11 +11,15 @@ import { SolverOptions } from '.'
 */
 
 export default async function solve({ url }: SolverOptions): Promise<string> {
+  throw new Error("hcaptcha-solver is not able to solve the new hCaptcha challenge. This issue is already reported #31.");
+
+/*
   try {
-    const token = await solveCaptcha(url)
+    return await solveCaptcha(url)
-    return token
   } catch (e) {
     console.error(e)
     return null
   }
+*/
+
 }

src/captcha/index.ts

@@ -18,7 +18,9 @@ const captchaSolvers: { [key: string]: Solver } = {}
 export default (): Solver => {
   const method = process.env.CAPTCHA_SOLVER
 
-  if (!method) { return null }
+  if (!method || method.toLowerCase() == 'none') {
+    return null;
+  }
 
   if (!(method in captchaSolvers)) {
     try {
@@ -28,12 +30,12 @@ export default (): Solver => {
         throw Error(`The solver '${method}' is not a valid captcha solving method.`)
       } else {
         console.error(e)
-        throw Error(`An error occured loading the solver '${method}'.`)
+        throw Error(`An error occurred loading the solver '${method}'.`)
       }
     }
   }
 
-  log.info(`Using '${method} to solve the captcha.`);
+  log.info(`Using '${method}' to solve the captcha.`);
 
   return captchaSolvers[method]
 }

src/index.ts

@@ -1,12 +1,64 @@
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const process = require('process')
 import log from './log'
 import { createServer, IncomingMessage, ServerResponse } from 'http';
 import { RequestContext } from './types'
 import Router, { BaseAPICall } from './routes'
+import getCaptchaSolver from "./captcha";
+import sessions from "./session";
+import {v1 as UUIDv1} from "uuid";
 
 const version: string = "v" + require('../package.json').version
 const serverPort: number = Number(process.env.PORT) || 8191
 const serverHost: string = process.env.HOST || '0.0.0.0'
 
+function validateEnvironmentVariables() {
+  // ip and port variables are validated by nodejs
+  if (process.env.LOG_LEVEL && ['error', 'warn', 'info', 'verbose', 'debug'].indexOf(process.env.LOG_LEVEL) == -1) {
+    log.error(`The environment variable 'LOG_LEVEL' is wrong. Check the documentation.`);
+    process.exit(1);
+  }
+  if (process.env.LOG_HTML && ['true', 'false'].indexOf(process.env.LOG_HTML) == -1) {
+    log.error(`The environment variable 'LOG_HTML' is wrong. Check the documentation.`);
+    process.exit(1);
+  }
+  if (process.env.HEADLESS && ['true', 'false'].indexOf(process.env.HEADLESS) == -1) {
+    log.error(`The environment variable 'HEADLESS' is wrong. Check the documentation.`);
+    process.exit(1);
+  }
+  try {
+    getCaptchaSolver();
+  } catch (e) {
+    log.error(`The environment variable 'CAPTCHA_SOLVER' is wrong. ${e.message}`);
+    process.exit(1);
+  }
+}
+
+async function testChromeInstallation() {
+  const sessionId = UUIDv1()
+  // create a temporary file for testing
+  log.debug("Testing Chrome installation...")
+  const fileContent = `flaresolverr_${version}`
+  const filePath = path.join(os.tmpdir(), `flaresolverr_${sessionId}.txt`)
+  const fileUrl = `file://${filePath}`
+  fs.writeFileSync(filePath, fileContent)
+  // launch the browser
+  const session = await sessions.create(sessionId, {
+    userAgent: null,
+    oneTimeSession: true
+  })
+  const page = await session.browser.newPage()
+  const response = await page.goto(fileUrl, { waitUntil: 'domcontentloaded' })
+  const responseBody = (await response.buffer()).toString().trim()
+  if (responseBody != fileContent) {
+    throw new Error("The response body does not match!")
+  }
+  await page.close()
+  await sessions.destroy(sessionId)
+  log.debug("Test successful")
+}
 
 function errorResponse(errorMsg: string, res: ServerResponse, startTimestamp: number) {
   log.error(errorMsg)
@@ -26,7 +78,7 @@ function errorResponse(errorMsg: string, res: ServerResponse, startTimestamp: nu
 
 function successResponse(successMsg: string, extendedProperties: object, res: ServerResponse, startTimestamp: number) {
   const endTimestamp = Date.now()
-  log.info(`Successful response in ${(endTimestamp - startTimestamp) / 1000} s`)
+  log.info(`Response in ${(endTimestamp - startTimestamp) / 1000} s`)
   if (successMsg) { log.info(successMsg) }
 
   const response = Object.assign({
@@ -64,9 +116,37 @@ function validateIncomingRequest(ctx: RequestContext, params: BaseAPICall) {
   return true
 }
 
+// init
+log.info(`FlareSolverr ${version}`);
+log.debug('Debug log enabled');
+
+process.on('SIGTERM', () => {
+  // Capture signal on Docker Stop #158
+  log.info("Process interrupted")
+  process.exit(0)
+})
+
+validateEnvironmentVariables();
+
+testChromeInstallation()
+.catch(e => {
+  log.error("Error starting Chrome browser.", e);
+  process.exit(1);
+})
+.then(r =>
   createServer((req: IncomingMessage, res: ServerResponse) => {
     const startTimestamp = Date.now()
 
+    // health endpoint. this endpoint is special because it doesn't print traces
+    if (req.url == '/health') {
+      res.writeHead(200, {
+        'Content-Type': 'application/json'
+      })
+      res.write(JSON.stringify({"status": "ok"}))
+      res.end()
+      return;
+    }
+
     // count the request for the log prefix
     log.incRequests()
     log.info(`Incoming request: ${req.method} ${req.url}`)
@@ -110,6 +190,6 @@ createServer((req: IncomingMessage, res: ServerResponse) => {
       })
     })
   }).listen(serverPort, serverHost, () => {
-  log.info(`FlareSolverr ${version} listening on http://${serverHost}:${serverPort}`);
+    log.info(`Listening on http://${serverHost}:${serverPort}`);
-  log.debug('Debug log enabled');
   })
+)

src/log.ts

@@ -1,6 +1,25 @@
 let requests = 0
 
-const LOG_HTML: boolean = Boolean(process.env.LOG_HTML) || false
+const LOG_HTML: boolean = process.env.LOG_HTML == 'true';
+
+function toIsoString(date: Date) {
+  // this function fixes Date.toISOString() adding timezone
+  let tzo = -date.getTimezoneOffset(),
+      dif = tzo >= 0 ? '+' : '-',
+      pad = function(num: number) {
+        let norm = Math.floor(Math.abs(num));
+        return (norm < 10 ? '0' : '') + norm;
+      };
+
+  return date.getFullYear() +
+      '-' + pad(date.getMonth() + 1) +
+      '-' + pad(date.getDate()) +
+      'T' + pad(date.getHours()) +
+      ':' + pad(date.getMinutes()) +
+      ':' + pad(date.getSeconds()) +
+      dif + pad(tzo / 60) +
+      ':' + pad(tzo % 60);
+}
 
 export default {
   incRequests: () => { requests++ },
@@ -9,10 +28,10 @@ export default {
       this.debug(html)
   },
   ...require('console-log-level')(
-    {
+    {level: process.env.LOG_LEVEL || 'info',
-      level: process.env.LOG_LEVEL || 'info',
       prefix(level: string) {
-        return `${new Date().toISOString()} ${level.toUpperCase()} REQ-${requests}`
+        const req = (requests > 0) ? ` REQ-${requests}` : '';
+        return `${toIsoString(new Date())} ${level.toUpperCase()}${req}`
       }
     }
   )

src/providers/cloudflare.ts

@@ -8,7 +8,7 @@ import getCaptchaSolver, {CaptchaType} from "../captcha";
  *  This class contains the logic to solve protections provided by CloudFlare
 **/
 
-const CHALLENGE_SELECTORS = ['#trk_jschal_js', '.ray_id', '.attack-box'];
+const CHALLENGE_SELECTORS = ['#trk_jschal_js', '.ray_id', '.attack-box', '#cf-please-wait'];
 const TOKEN_INPUT_NAMES = ['g-recaptcha-response', 'h-captcha-response'];
 
 export default async function resolveChallenge(url: string, page: Page, response: Response): Promise<Response> {
@@ -20,8 +20,8 @@ export default async function resolveChallenge(url: string, page: Page, response
   }
   log.info('Cloudflare detected');
 
-  if (await page.$('.cf-error-code')) {
+  if (await page.$('span[data-translate="error"]') || (await page.content()).includes('error code: 1020')) {
-    throw new Error('Cloudflare has blocked this request (Code 1020 Detected).')
+    throw new Error('Cloudflare has blocked this request. Probably your IP is banned for this site, check in your web browser.')
   }
 
   let selectorFoundCount = 0;
@@ -35,23 +35,51 @@ export default async function resolveChallenge(url: string, page: Page, response
         log.debug('Waiting for Cloudflare challenge...')
 
         while (true) {
-          await page.waitFor(1000)
-          try {
-            // catch exception timeout in waitForNavigation
-            response = await page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 5000 })
-          } catch (error) { }
-
           try {
             // catch Execution context was destroyed
             const cfChallengeElem = await page.$(selector)
-            if (!cfChallengeElem) { break }
+            if (!cfChallengeElem) {
-            log.debug('Found challenge element again...')
+              // solved!
+              log.debug('Challenge element not found.')
+              break
+            } else {
+              // new Cloudflare Challenge #cf-please-wait
+              const displayStyle = await page.evaluate((selector) => {
+                return getComputedStyle(document.querySelector(selector)).getPropertyValue("display");
+              }, selector);
+              if (displayStyle == "none") {
+                // spinner is hidden, could be a captcha or not
+                log.debug('Challenge element is hidden.')
+                // wait until redirecting disappears
+                while (true) {
+                  try {
+                    await page.waitFor(1000)
+                    const displayStyle2 = await page.evaluate(() => {
+                      return getComputedStyle(document.querySelector('#cf-spinner-redirecting')).getPropertyValue("display");
+                    });
+                    if (displayStyle2 == "none") {
+                      break // hCaptcha detected
+                    }
+                  } catch (error) {
+                    break // redirection completed
+                  }
+                }
+                break
+              } else {
+                log.debug('Challenge element is visible.')
+              }
+            }
+            log.debug('Found challenge element again.')
           } catch (error)
-          { }
+          {
+            log.debug("Unexpected error: " + error);
+            if (!error.toString().includes("Execution context was destroyed")) {
+              break
+            }
+          }
 
-          response = await page.reload({ waitUntil: 'domcontentloaded' })
+          log.debug('Waiting for Cloudflare challenge...')
-          log.debug('Page reloaded.')
+          await page.waitFor(1000)
-          log.html(await page.content())
         }
 
         log.debug('Validating HTML code...')
@@ -117,7 +145,7 @@ export default async function resolveChallenge(url: string, page: Page, response
 
         // ignore preset event listeners on the form
         await page.evaluate(() => {
-          window.addEventListener('submit', (e) => { event.stopPropagation() }, true)
+          window.addEventListener('submit', (e) => { e.stopPropagation() }, true)
         })
 
         // it seems some sites obfuscate their challenge forms
@@ -143,7 +171,7 @@ export default async function resolveChallenge(url: string, page: Page, response
         }
 
         // submit captcha response
-        challengeForm.evaluate((e: HTMLFormElement) => e.submit())
+        await challengeForm.evaluate((e: HTMLFormElement) => e.submit())
         response = await page.waitForNavigation({ waitUntil: 'domcontentloaded' })
 
         if (await page.$('input[name="cf_captcha_kind"]')) {
@@ -157,6 +185,12 @@ export default async function resolveChallenge(url: string, page: Page, response
     if (selectorFoundCount == 0)
     {
       throw new Error('No challenge selectors found, unable to proceed')
+    } else {
+      // reload the page to make sure we get the real response
+      // do not use page.reload() to avoid #162 #143
+      response = await page.goto(url, { waitUntil: 'domcontentloaded' })
+      await page.content()
+      log.info('Challenge solved.');
     }
   }
 

src/routes.ts

@@ -36,9 +36,9 @@ interface BaseRequestAPICall extends BaseAPICall {
   proxy?: any, // TODO: use interface not any
   download?: boolean
   returnOnlyCookies?: boolean
+  returnRawHtml?: boolean
 }
 
-
 interface Routes {
   [key: string]: (ctx: RequestContext, params: BaseAPICall) => void | Promise<void>
 }
@@ -86,7 +86,9 @@ async function resolveChallengeWithTimeout(ctx: RequestContext, params: BaseRequ
   }
 }
 
-async function resolveChallenge(ctx: RequestContext, { url, proxy, download, returnOnlyCookies }: BaseRequestAPICall, page: Page): Promise<ChallengeResolutionT | void> {
+async function resolveChallenge(ctx: RequestContext,
+                                { url, proxy, download, returnOnlyCookies, returnRawHtml }: BaseRequestAPICall,
+                                page: Page): Promise<ChallengeResolutionT | void> {
 
   let status = 'ok'
   let message = ''
@@ -132,11 +134,16 @@ async function resolveChallenge(ctx: RequestContext, { url, proxy, download, ret
       // fix since I am short on time
       response = await page.goto(url, { waitUntil: 'domcontentloaded' })
       payload.result.response = (await response.buffer()).toString('base64')
+    } else if (returnRawHtml) {
+      payload.result.response = await response.text()
     } else {
       payload.result.response = await page.content()
     }
   }
 
+  // Add final url in result
+  payload.result.url = page.url();
+
   // make sure the page is closed because if it isn't and error will be thrown
   // when a user uses a temporary session, the browser make be quit before
   // the page is properly closed.
@@ -180,21 +187,26 @@ async function setupPage(ctx: RequestContext, params: BaseRequestAPICall, browse
   }
 
   if (headers) {
-    log.debug(`Adding custom headers: ${JSON.stringify(headers, null, 2)}`,)
+    log.debug(`Adding custom headers: ${JSON.stringify(headers)}`)
     overrideResolvers.headers = request => Object.assign(request.headers(), headers)
   }
 
   if (cookies) {
-    log.debug(`Setting custom cookies: ${JSON.stringify(cookies, null, 2)}`,)
+    log.debug(`Setting custom cookies: ${JSON.stringify(cookies)}`)
     await page.setCookie(...cookies)
   }
 
   // if any keys have been set on the object
   if (Object.keys(overrideResolvers).length > 0) {
-    log.debug(overrideResolvers)
     let callbackRunOnce = false
     const callback = (request: Request) => {
 
+      // avoid loading resources to speed up page load
+      if(request.resourceType() == 'stylesheet' || request.resourceType() == 'font' || request.resourceType() == 'image') {
+        request.abort()
+        return
+      }
+
       if (callbackRunOnce || !request.isNavigationRequest()) {
         request.continue()
         return
@@ -208,8 +220,7 @@ async function setupPage(ctx: RequestContext, params: BaseRequestAPICall, browse
         overrides[key] = overrideResolvers[key](request)
       });
 
-      log.debug(overrides)
+      log.debug(`Overrides: ${JSON.stringify(overrides)}`)
-
       request.continue(overrides)
     }
 
@@ -253,7 +264,9 @@ const browserRequest = async (ctx: RequestContext, params: BaseRequestAPICall) =
     log.error(error)
     return ctx.errorResponse("Unable to process browser request. Error: " + error)
   } finally {
-    if (oneTimeSession) { sessions.destroy(sessionId) }
+    if (oneTimeSession) {
+      await sessions.destroy(sessionId)
+    }
   }
 }
 

src/session.ts

@@ -82,7 +82,7 @@ export default {
       puppeteerOptions.executablePath = path.join(path.dirname(process.execPath), 'chrome', exe)
     }
 
-    log.debug('Launching headless browser...')
+    log.debug('Launching browser...')
 
     // TODO: maybe access env variable?
     // TODO: sometimes browser instances are created and not connected to correctly.