Bump version 1.2.9

Bumps [ws](https://github.com/websockets/ws) from 7.4.1 to 7.4.6.
- [Release notes](https://github.com/websockets/ws/releases)
- [Commits](https://github.com/websockets/ws/compare/7.4.1...7.4.6)

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.github/ISSUE_TEMPLATE.md

@@ -14,6 +14,7 @@ Check closed issues as well, because your issue may have already been fixed.
 * **Are you using a proxy or VPN?** [yes/no]
 * **Are you using Captcha Solver:** [yes/no]
 * **If using captcha solver, which one:**
+* **URL to test this issue:**
 
 ### Description
 

Dockerfile

@@ -22,8 +22,8 @@ ENV PUPPETEER_PRODUCT=chrome \
     PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
 RUN npm install && \
     npm run build && \
-    rm -rf src tsconfig.json && \
-    npm prune --production
+    npm prune --production && \
+    rm -rf /home/node/.npm
 
 EXPOSE 8191
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]

README.md

@@ -8,9 +8,7 @@
 [![Donate Buy Me A Coffee](https://img.shields.io/badge/Donate-Buy%20me%20a%20coffee-yellow.svg)](https://www.buymeacoffee.com/ngosang)
 [![Donate Bitcoin](https://img.shields.io/badge/Donate-Bitcoin-orange.svg)](https://en.cryptobadges.io/donate/13Hcv77AdnFWEUZ9qUpoPBttQsUT7q9TTh)
 
-FlareSolverr is a proxy server to bypass Cloudflare protection
-
-:warning: This project is in beta state. Some things may not work and the API can change at any time.
+FlareSolverr is a proxy server to bypass Cloudflare protection.
 
 ## How it works
 
@@ -67,12 +65,17 @@ This is the recommended way for Windows users.
 
 ### From source code
 
-This is the recommended way for MacOS users and for developers.
-* Install [NodeJS](https://nodejs.org/)
-* Clone this repository and open a shell in that path
-* Run `npm install` command to install FlareSolverr dependencies
-* Run `npm run build` command to compile TypeScript code
-* Run `npm start` command to start FlareSolverr
+This is the recommended way for macOS users and for developers.
+* Install [NodeJS](https://nodejs.org/).
+* Clone this repository and open a shell in that path.
+* Run `npm install` command to install FlareSolverr dependencies.
+* Run `node node_modules/puppeteer/install.js` to install Chromium.
+* Run `npm run build` command to compile TypeScript code.
+* Run `npm start` command to start FlareSolverr.
+
+### Systemd service
+
+We provide an example Systemd unit file `flaresolverr.service` as reference. You have to modify the file to suit your needs: paths, user and environment variables.
 
 ## Usage
 
@@ -143,6 +146,7 @@ headers | Optional. To specify user headers.
 maxTimeout | Optional, default value 60000. Max timeout to solve the challenge in milliseconds.
 cookies | Optional. Will be used by the headless browser. Follow [this](https://github.com/puppeteer/puppeteer/blob/v3.3.0/docs/api.md#pagesetcookiecookies) format.
 returnOnlyCookies | Optional, default false. Only returns the cookies. Response data, headers and other parts of the response are removed.
+returnRawHtml | Optional, default false. The response data will be returned without JS processing. This is useful for JSON or plain text content.
 
 Example response from running the `curl` above:
 
@@ -226,7 +230,8 @@ Name | Default | Notes
 |--|--|--|
 LOG_LEVEL | info | Verbosity of the logging. Use `LOG_LEVEL=debug` for more information.
 LOG_HTML | false | Only for debugging. If `true` all HTML that passes through the proxy will be logged to the console in `debug` level.
-CAPTCHA_SOLVER | none | Captcha solving method. It used when a captcha is encountered. See the Captcha Solvers section.
+CAPTCHA_SOLVER | none | Captcha solving method. It is used when a captcha is encountered. See the Captcha Solvers section.
+TZ | UTC | Timezone used in the logs and the web browser. Example: `TZ=Europe/London`.
 HEADLESS | true | Only for debugging. To run the web browser in headless mode or visible.
 PORT | 8191 | Listening port. You don't need to change this if you are running on Docker.
 HOST | 0.0.0.0 | Listening interface. You don't need to change this if you are running on Docker.

build-binaries.js

@@ -40,7 +40,7 @@ const version = 'v' + require('./package.json').version;
   // generate executables
   console.log('Generating executables...')
   if (fs.existsSync('bin')) {
-    fs.rmdirSync('bin', { recursive: true })
+    fs.rmSync('bin', { recursive: true })
   }
   execSync('pkg -t node14-win-x64,node14-linux-x64 --out-path bin .')
   // execSync('pkg -t node14-win-x64,node14-mac-x64,node14-linux-x64 --out-path bin .')
@@ -76,6 +76,9 @@ const version = 'v' + require('./package.json').version;
     archive.file('LICENSE', { name: 'flaresolverr/' + os.fsLicenseName })
     archive.file('bin/' + os.fsExec, { name: 'flaresolverr/' + os.fsZipExec })
     archive.directory('bin/puppeteer/' + os.platform + '-' + os.version + '/' + os.chromeFolder, 'flaresolverr/chrome')
+    if (os.platform === 'linux') {
+      archive.file('flaresolverr.service', { name: 'flaresolverr/flaresolverr.service' })
+    }
 
     await archive.finalize()
   }

docker-compose.yml

@@ -9,6 +9,7 @@ services:
       - LOG_LEVEL=${LOG_LEVEL:-info}
       - LOG_HTML=${LOG_HTML:-false}
       - CAPTCHA_SOLVER=${CAPTCHA_SOLVER:-none}
+      - TZ=Europe/London
     ports:
       - "${PORT:-8191}:8191"
     restart: unless-stopped

flaresolverr.service

@@ -0,0 +1,19 @@
+[Unit]
+Description=FlareSolverr
+After=network.target
+
+[Service]
+SyslogIdentifier=flaresolverr
+Restart=always
+RestartSec=5
+Type=simple
+User=flaresolverr
+Group=flaresolverr
+Environment="LOG_LEVEL=info"
+Environment="CAPTCHA_SOLVER=none"
+WorkingDirectory=/opt/flaresolverr
+ExecStart=/opt/flaresolverr/flaresolverr
+TimeoutStopSec=30
+
+[Install]
+WantedBy=multi-user.target

package.json

@@ -1,6 +1,6 @@
 {
   "name": "flaresolverr",
-  "version": "1.2.5",
+  "version": "1.2.9",
   "description": "Proxy server to bypass Cloudflare protection.",
   "scripts": {
     "start": "node ./dist/index.js",

src/captcha/hcaptcha-solver.ts

@@ -11,10 +11,15 @@ import { SolverOptions } from '.'
 */
 
 export default async function solve({ url }: SolverOptions): Promise<string> {
+  throw new Error("hcaptcha-solver is not able to solve the new hCaptcha challenge. This issue is already reported #31.");
+
+/*
   try {
     return await solveCaptcha(url)
   } catch (e) {
     console.error(e)
     return null
   }
+*/
+
 }

src/index.ts

@@ -1,6 +1,7 @@
 const fs = require('fs');
 const os = require('os');
 const path = require('path');
+const process = require('process')
 import log from './log'
 import { createServer, IncomingMessage, ServerResponse } from 'http';
 import { RequestContext } from './types'
@@ -36,14 +37,14 @@ function validateEnvironmentVariables() {
 }
 
 async function testChromeInstallation() {
-  log.debug("Testing Chrome installation...")
+  const sessionId = UUIDv1()
   // create a temporary file for testing
+  log.debug("Testing Chrome installation...")
   const fileContent = `flaresolverr_${version}`
-  const filePath = path.join(os.tmpdir(), 'flaresolverr.txt')
+  const filePath = path.join(os.tmpdir(), `flaresolverr_${sessionId}.txt`)
   const fileUrl = `file://${filePath}`
   fs.writeFileSync(filePath, fileContent)
   // launch the browser
-  const sessionId = UUIDv1()
   const session = await sessions.create(sessionId, {
     userAgent: null,
     oneTimeSession: true
@@ -77,7 +78,7 @@ function errorResponse(errorMsg: string, res: ServerResponse, startTimestamp: nu
 
 function successResponse(successMsg: string, extendedProperties: object, res: ServerResponse, startTimestamp: number) {
   const endTimestamp = Date.now()
-  log.info(`Successful response in ${(endTimestamp - startTimestamp) / 1000} s`)
+  log.info(`Response in ${(endTimestamp - startTimestamp) / 1000} s`)
   if (successMsg) { log.info(successMsg) }
 
   const response = Object.assign({
@@ -118,7 +119,15 @@ function validateIncomingRequest(ctx: RequestContext, params: BaseAPICall) {
 // init
 log.info(`FlareSolverr ${version}`);
 log.debug('Debug log enabled');
+
+process.on('SIGTERM', () => {
+  // Capture signal on Docker Stop #158
+  log.info("Process interrupted")
+  process.exit(0)
+})
+
 validateEnvironmentVariables();
+
 testChromeInstallation()
 .catch(e => {
   log.error("Error starting Chrome browser.", e);

src/log.ts

@@ -2,6 +2,25 @@ let requests = 0
 
 const LOG_HTML: boolean = process.env.LOG_HTML == 'true';
 
+function toIsoString(date: Date) {
+  // this function fixes Date.toISOString() adding timezone
+  let tzo = -date.getTimezoneOffset(),
+      dif = tzo >= 0 ? '+' : '-',
+      pad = function(num: number) {
+        let norm = Math.floor(Math.abs(num));
+        return (norm < 10 ? '0' : '') + norm;
+      };
+
+  return date.getFullYear() +
+      '-' + pad(date.getMonth() + 1) +
+      '-' + pad(date.getDate()) +
+      'T' + pad(date.getHours()) +
+      ':' + pad(date.getMinutes()) +
+      ':' + pad(date.getSeconds()) +
+      dif + pad(tzo / 60) +
+      ':' + pad(tzo % 60);
+}
+
 export default {
   incRequests: () => { requests++ },
   html(html: string) {
@@ -12,7 +31,7 @@ export default {
     {level: process.env.LOG_LEVEL || 'info',
       prefix(level: string) {
         const req = (requests > 0) ? ` REQ-${requests}` : '';
-        return `${new Date().toISOString()} ${level.toUpperCase()}${req}`
+        return `${toIsoString(new Date())} ${level.toUpperCase()}${req}`
       }
     }
   )

src/providers/cloudflare.ts

@@ -8,7 +8,7 @@ import getCaptchaSolver, {CaptchaType} from "../captcha";
  *  This class contains the logic to solve protections provided by CloudFlare
 **/
 
-const CHALLENGE_SELECTORS = ['#trk_jschal_js', '.ray_id', '.attack-box'];
+const CHALLENGE_SELECTORS = ['#trk_jschal_js', '.ray_id', '.attack-box', '#cf-please-wait'];
 const TOKEN_INPUT_NAMES = ['g-recaptcha-response', 'h-captcha-response'];
 
 export default async function resolveChallenge(url: string, page: Page, response: Response): Promise<Response> {
@@ -20,8 +20,8 @@ export default async function resolveChallenge(url: string, page: Page, response
   }
   log.info('Cloudflare detected');
 
-  if (await page.$('.cf-error-code')) {
-    throw new Error('Cloudflare has blocked this request (Code 1020 Detected).')
+  if (await page.$('span[data-translate="error"]') || (await page.content()).includes('error code: 1020')) {
+    throw new Error('Cloudflare has blocked this request. Probably your IP is banned for this site, check in your web browser.')
   }
 
   let selectorFoundCount = 0;
@@ -35,23 +35,51 @@ export default async function resolveChallenge(url: string, page: Page, response
         log.debug('Waiting for Cloudflare challenge...')
 
         while (true) {
-          await page.waitFor(1000)
-          try {
-            // catch exception timeout in waitForNavigation
-            response = await page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 5000 })
-          } catch (error) { }
-
           try {
             // catch Execution context was destroyed
             const cfChallengeElem = await page.$(selector)
-            if (!cfChallengeElem) { break }
-            log.debug('Found challenge element again...')
+            if (!cfChallengeElem) {
+              // solved!
+              log.debug('Challenge element not found.')
+              break
+            } else {
+              // new Cloudflare Challenge #cf-please-wait
+              const displayStyle = await page.evaluate((selector) => {
+                return getComputedStyle(document.querySelector(selector)).getPropertyValue("display");
+              }, selector);
+              if (displayStyle == "none") {
+                // spinner is hidden, could be a captcha or not
+                log.debug('Challenge element is hidden.')
+                // wait until redirecting disappears
+                while (true) {
+                  try {
+                    await page.waitFor(1000)
+                    const displayStyle2 = await page.evaluate(() => {
+                      return getComputedStyle(document.querySelector('#cf-spinner-redirecting')).getPropertyValue("display");
+                    });
+                    if (displayStyle2 == "none") {
+                      break // hCaptcha detected
+                    }
+                  } catch (error) {
+                    break // redirection completed
+                  }
+                }
+                break
+              } else {
+                log.debug('Challenge element is visible.')
+              }
+            }
+            log.debug('Found challenge element again.')
           } catch (error)
-          { }
+          {
+            log.debug("Unexpected error: " + error);
+            if (!error.toString().includes("Execution context was destroyed")) {
+              break
+            }
+          }
 
-          response = await page.reload({ waitUntil: 'domcontentloaded' })
-          log.debug('Page reloaded.')
-          log.html(await page.content())
+          log.debug('Waiting for Cloudflare challenge...')
+          await page.waitFor(1000)
         }
 
         log.debug('Validating HTML code...')
@@ -143,7 +171,7 @@ export default async function resolveChallenge(url: string, page: Page, response
         }
 
         // submit captcha response
-        challengeForm.evaluate((e: HTMLFormElement) => e.submit())
+        await challengeForm.evaluate((e: HTMLFormElement) => e.submit())
         response = await page.waitForNavigation({ waitUntil: 'domcontentloaded' })
 
         if (await page.$('input[name="cf_captcha_kind"]')) {
@@ -157,6 +185,12 @@ export default async function resolveChallenge(url: string, page: Page, response
     if (selectorFoundCount == 0)
     {
       throw new Error('No challenge selectors found, unable to proceed')
+    } else {
+      // reload the page to make sure we get the real response
+      // do not use page.reload() to avoid #162 #143
+      response = await page.goto(url, { waitUntil: 'domcontentloaded' })
+      await page.content()
+      log.info('Challenge solved.');
     }
   }
 

src/routes.ts

@@ -36,9 +36,9 @@ interface BaseRequestAPICall extends BaseAPICall {
   proxy?: any, // TODO: use interface not any
   download?: boolean
   returnOnlyCookies?: boolean
+  returnRawHtml?: boolean
 }
 
-
 interface Routes {
   [key: string]: (ctx: RequestContext, params: BaseAPICall) => void | Promise<void>
 }
@@ -86,7 +86,9 @@ async function resolveChallengeWithTimeout(ctx: RequestContext, params: BaseRequ
   }
 }
 
-async function resolveChallenge(ctx: RequestContext, { url, proxy, download, returnOnlyCookies }: BaseRequestAPICall, page: Page): Promise<ChallengeResolutionT | void> {
+async function resolveChallenge(ctx: RequestContext,
+                                { url, proxy, download, returnOnlyCookies, returnRawHtml }: BaseRequestAPICall,
+                                page: Page): Promise<ChallengeResolutionT | void> {
 
   let status = 'ok'
   let message = ''
@@ -132,11 +134,16 @@ async function resolveChallenge(ctx: RequestContext, { url, proxy, download, ret
       // fix since I am short on time
       response = await page.goto(url, { waitUntil: 'domcontentloaded' })
       payload.result.response = (await response.buffer()).toString('base64')
+    } else if (returnRawHtml) {
+      payload.result.response = await response.text()
     } else {
       payload.result.response = await page.content()
     }
   }
 
+  // Add final url in result
+  payload.result.url = page.url();
+
   // make sure the page is closed because if it isn't and error will be thrown
   // when a user uses a temporary session, the browser make be quit before
   // the page is properly closed.
@@ -194,6 +201,12 @@ async function setupPage(ctx: RequestContext, params: BaseRequestAPICall, browse
     let callbackRunOnce = false
     const callback = (request: Request) => {
 
+      // avoid loading resources to speed up page load
+      if(request.resourceType() == 'stylesheet' || request.resourceType() == 'font' || request.resourceType() == 'image') {
+        request.abort()
+        return
+      }
+
       if (callbackRunOnce || !request.isNavigationRequest()) {
         request.continue()
         return