Bump version 1.2.8

Bumps [ws](https://github.com/websockets/ws) from 7.4.1 to 7.4.6.
- [Release notes](https://github.com/websockets/ws/releases)
- [Commits](https://github.com/websockets/ws/compare/7.4.1...7.4.6)

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.github/ISSUE_TEMPLATE.md

@@ -14,6 +14,7 @@ Check closed issues as well, because your issue may have already been fixed.
 * **Are you using a proxy or VPN?** [yes/no]
 * **Are you using Captcha Solver:** [yes/no]
 * **If using captcha solver, which one:**
+* **URL to test this issue:**
 
 ### Description
 

.github/workflows/release-docker.yml

@@ -24,7 +24,7 @@ jobs:
           tag-sha: false
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v1.0.1
       -
         name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1

README.md

@@ -8,9 +8,7 @@
 [![Donate Buy Me A Coffee](https://img.shields.io/badge/Donate-Buy%20me%20a%20coffee-yellow.svg)](https://www.buymeacoffee.com/ngosang)
 [![Donate Bitcoin](https://img.shields.io/badge/Donate-Bitcoin-orange.svg)](https://en.cryptobadges.io/donate/13Hcv77AdnFWEUZ9qUpoPBttQsUT7q9TTh)
 
-FlareSolverr is a proxy server to bypass Cloudflare protection
+FlareSolverr is a proxy server to bypass Cloudflare protection.
-
-:warning: This project is in beta state. Some things may not work and the API can change at any time.
 
 ## How it works
 
@@ -67,12 +65,17 @@ This is the recommended way for Windows users.
 
 ### From source code
 
-This is the recommended way for MacOS users and for developers.
+This is the recommended way for macOS users and for developers.
-* Install [NodeJS](https://nodejs.org/)
+* Install [NodeJS](https://nodejs.org/).
-* Clone this repository and open a shell in that path
+* Clone this repository and open a shell in that path.
-* Run `npm install` command to install FlareSolverr dependencies
+* Run `npm install` command to install FlareSolverr dependencies.
-* Run `npm run build` command to compile TypeScript code
+* Run `node node_modules/puppeteer/install.js` to install Chromium.
-* Run `npm start` command to start FlareSolverr
+* Run `npm run build` command to compile TypeScript code.
+* Run `npm start` command to start FlareSolverr.
+
+### Systemd service
+
+We provide an example Systemd unit file `flaresolverr.service` as reference. You have to modify the file to suit your needs: paths, user and environment variables.
 
 ## Usage
 
@@ -226,7 +229,8 @@ Name | Default | Notes
 |--|--|--|
 LOG_LEVEL | info | Verbosity of the logging. Use `LOG_LEVEL=debug` for more information.
 LOG_HTML | false | Only for debugging. If `true` all HTML that passes through the proxy will be logged to the console in `debug` level.
-CAPTCHA_SOLVER | none | Captcha solving method. It used when a captcha is encountered. See the Captcha Solvers section.
+CAPTCHA_SOLVER | none | Captcha solving method. It is used when a captcha is encountered. See the Captcha Solvers section.
+TZ | UTC | Timezone used in the logs and the web browser. Example: `TZ=Europe/London`.
 HEADLESS | true | Only for debugging. To run the web browser in headless mode or visible.
 PORT | 8191 | Listening port. You don't need to change this if you are running on Docker.
 HOST | 0.0.0.0 | Listening interface. You don't need to change this if you are running on Docker.

build-binaries.js

@@ -40,7 +40,7 @@ const version = 'v' + require('./package.json').version;
   // generate executables
   console.log('Generating executables...')
   if (fs.existsSync('bin')) {
-    fs.rmdirSync('bin', { recursive: true })
+    fs.rmSync('bin', { recursive: true })
   }
   execSync('pkg -t node14-win-x64,node14-linux-x64 --out-path bin .')
   // execSync('pkg -t node14-win-x64,node14-mac-x64,node14-linux-x64 --out-path bin .')
@@ -76,6 +76,9 @@ const version = 'v' + require('./package.json').version;
     archive.file('LICENSE', { name: 'flaresolverr/' + os.fsLicenseName })
     archive.file('bin/' + os.fsExec, { name: 'flaresolverr/' + os.fsZipExec })
     archive.directory('bin/puppeteer/' + os.platform + '-' + os.version + '/' + os.chromeFolder, 'flaresolverr/chrome')
+    if (os.platform === 'linux') {
+      archive.file('flaresolverr.service', { name: 'flaresolverr/flaresolverr.service' })
+    }
 
     await archive.finalize()
   }

docker-compose.yml

@@ -9,6 +9,7 @@ services:
       - LOG_LEVEL=${LOG_LEVEL:-info}
       - LOG_HTML=${LOG_HTML:-false}
       - CAPTCHA_SOLVER=${CAPTCHA_SOLVER:-none}
+      - TZ=Europe/London
     ports:
       - "${PORT:-8191}:8191"
     restart: unless-stopped

flaresolverr.service

@@ -0,0 +1,19 @@
+[Unit]
+Description=FlareSolverr
+After=network.target
+
+[Service]
+SyslogIdentifier=flaresolverr
+Restart=always
+RestartSec=5
+Type=simple
+User=flaresolverr
+Group=flaresolverr
+Environment="LOG_LEVEL=info"
+Environment="CAPTCHA_SOLVER=none"
+WorkingDirectory=/opt/flaresolverr
+ExecStart=/opt/flaresolverr/flaresolverr
+TimeoutStopSec=30
+
+[Install]
+WantedBy=multi-user.target

package.json

@@ -1,6 +1,6 @@
 {
   "name": "flaresolverr",
-  "version": "1.2.4",
+  "version": "1.2.8",
   "description": "Proxy server to bypass Cloudflare protection.",
   "scripts": {
     "start": "node ./dist/index.js",

src/captcha/hcaptcha-solver.ts

@@ -11,10 +11,15 @@ import { SolverOptions } from '.'
 */
 
 export default async function solve({ url }: SolverOptions): Promise<string> {
+  throw new Error("hcaptcha-solver is not able to solve the new hCaptcha challenge. This issue is already reported #31.");
+
+/*
   try {
     return await solveCaptcha(url)
   } catch (e) {
     console.error(e)
     return null
   }
+*/
+
 }

src/index.ts

@@ -36,19 +36,27 @@ function validateEnvironmentVariables() {
 }
 
 async function testChromeInstallation() {
-  log.debug("Testing Chrome installation...")
+  const sessionId = UUIDv1()
   // create a temporary file for testing
-  const filePath = path.join(os.tmpdir(), 'flaresolverr.txt')
+  log.debug("Testing Chrome installation...")
-  fs.writeFileSync(filePath, 'flaresolverr');
+  const fileContent = `flaresolverr_${version}`
+  const filePath = path.join(os.tmpdir(), `flaresolverr_${sessionId}.txt`)
+  const fileUrl = `file://${filePath}`
+  fs.writeFileSync(filePath, fileContent)
   // launch the browser
-  const url = `file://${filePath}`;
+  const session = await sessions.create(sessionId, {
-  const session = await sessions.create(UUIDv1(), {
     userAgent: null,
     oneTimeSession: true
   })
   const page = await session.browser.newPage()
-  await page.goto(url, { waitUntil: 'domcontentloaded' })
+  const response = await page.goto(fileUrl, { waitUntil: 'domcontentloaded' })
-  log.debug("Test successful.")
+  const responseBody = (await response.buffer()).toString().trim()
+  if (responseBody != fileContent) {
+    throw new Error("The response body does not match!")
+  }
+  await page.close()
+  await sessions.destroy(sessionId)
+  log.debug("Test successful")
 }
 
 function errorResponse(errorMsg: string, res: ServerResponse, startTimestamp: number) {
@@ -69,7 +77,7 @@ function errorResponse(errorMsg: string, res: ServerResponse, startTimestamp: nu
 
 function successResponse(successMsg: string, extendedProperties: object, res: ServerResponse, startTimestamp: number) {
   const endTimestamp = Date.now()
-  log.info(`Successful response in ${(endTimestamp - startTimestamp) / 1000} s`)
+  log.info(`Response in ${(endTimestamp - startTimestamp) / 1000} s`)
   if (successMsg) { log.info(successMsg) }
 
   const response = Object.assign({
@@ -111,7 +119,12 @@ function validateIncomingRequest(ctx: RequestContext, params: BaseAPICall) {
 log.info(`FlareSolverr ${version}`);
 log.debug('Debug log enabled');
 validateEnvironmentVariables();
-testChromeInstallation().then(r =>
+testChromeInstallation()
+.catch(e => {
+  log.error("Error starting Chrome browser.", e);
+  process.exit(1);
+})
+.then(r =>
   createServer((req: IncomingMessage, res: ServerResponse) => {
     const startTimestamp = Date.now()
 

src/log.ts

@@ -2,6 +2,25 @@ let requests = 0
 
 const LOG_HTML: boolean = process.env.LOG_HTML == 'true';
 
+function toIsoString(date: Date) {
+  // this function fixes Date.toISOString() adding timezone
+  let tzo = -date.getTimezoneOffset(),
+      dif = tzo >= 0 ? '+' : '-',
+      pad = function(num: number) {
+        let norm = Math.floor(Math.abs(num));
+        return (norm < 10 ? '0' : '') + norm;
+      };
+
+  return date.getFullYear() +
+      '-' + pad(date.getMonth() + 1) +
+      '-' + pad(date.getDate()) +
+      'T' + pad(date.getHours()) +
+      ':' + pad(date.getMinutes()) +
+      ':' + pad(date.getSeconds()) +
+      dif + pad(tzo / 60) +
+      ':' + pad(tzo % 60);
+}
+
 export default {
   incRequests: () => { requests++ },
   html(html: string) {
@@ -12,7 +31,7 @@ export default {
     {level: process.env.LOG_LEVEL || 'info',
       prefix(level: string) {
         const req = (requests > 0) ? ` REQ-${requests}` : '';
-        return `${new Date().toISOString()} ${level.toUpperCase()}${req}`
+        return `${toIsoString(new Date())} ${level.toUpperCase()}${req}`
       }
     }
   )

src/providers/cloudflare.ts

@@ -8,7 +8,7 @@ import getCaptchaSolver, {CaptchaType} from "../captcha";
  *  This class contains the logic to solve protections provided by CloudFlare
 **/
 
-const CHALLENGE_SELECTORS = ['#trk_jschal_js', '.ray_id', '.attack-box'];
+const CHALLENGE_SELECTORS = ['#trk_jschal_js', '.ray_id', '.attack-box', '#cf-please-wait'];
 const TOKEN_INPUT_NAMES = ['g-recaptcha-response', 'h-captcha-response'];
 
 export default async function resolveChallenge(url: string, page: Page, response: Response): Promise<Response> {
@@ -35,23 +35,49 @@ export default async function resolveChallenge(url: string, page: Page, response
         log.debug('Waiting for Cloudflare challenge...')
 
         while (true) {
-          await page.waitFor(1000)
-          try {
-            // catch exception timeout in waitForNavigation
-            response = await page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 5000 })
-          } catch (error) { }
-
           try {
             // catch Execution context was destroyed
             const cfChallengeElem = await page.$(selector)
-            if (!cfChallengeElem) { break }
+            if (!cfChallengeElem) {
-            log.debug('Found challenge element again...')
+              // solved!
+              log.debug('Challenge element not found.')
+              break
+            } else {
+              // new Cloudflare Challenge #cf-please-wait
+              const displayStyle = await page.evaluate((selector) => {
+                return getComputedStyle(document.querySelector(selector)).getPropertyValue("display");
+              }, selector);
+              if (displayStyle == "none") {
+                // spinner is hidden, could be a captcha or not
+                log.debug('Challenge element is hidden.')
+                // wait until redirecting disappears
+                while (true) {
+                  try {
+                    await page.waitFor(1000)
+                    const displayStyle2 = await page.evaluate(() => {
+                      return getComputedStyle(document.querySelector('#cf-spinner-redirecting')).getPropertyValue("display");
+                    });
+                    if (displayStyle2 == "none") {
+                      break // hCaptcha detected
+                    }
+                  } catch (error) {
+                    break // redirection completed
+                  }
+                }
+                break
+              } else {
+                log.debug('Challenge element is visible.')
+              }
+            }
+            log.debug('Found challenge element again.')
           } catch (error)
-          { }
+          {
+            log.debug("Unexpected error: " + error);
+            break
+          }
 
-          response = await page.reload({ waitUntil: 'domcontentloaded' })
+          log.debug('Waiting for Cloudflare challenge...')
-          log.debug('Page reloaded.')
+          await page.waitFor(1000)
-          log.html(await page.content())
         }
 
         log.debug('Validating HTML code...')
@@ -157,6 +183,11 @@ export default async function resolveChallenge(url: string, page: Page, response
     if (selectorFoundCount == 0)
     {
       throw new Error('No challenge selectors found, unable to proceed')
+    } else {
+      // reload the page to make sure we get the real response
+      response = await page.reload()
+      await page.content()
+      log.info('Challenge solved.');
     }
   }
 

src/routes.ts

@@ -137,6 +137,9 @@ async function resolveChallenge(ctx: RequestContext, { url, proxy, download, ret
     }
   }
 
+  // Add final url in result
+  payload.result.url = page.url();
+
   // make sure the page is closed because if it isn't and error will be thrown
   // when a user uses a temporary session, the browser make be quit before
   // the page is properly closed.