Bump version 1.2.9

Dockerfile

@@ -22,8 +22,8 @@ ENV PUPPETEER_PRODUCT=chrome \
     PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
 RUN npm install && \
     npm run build && \
-    rm -rf src tsconfig.json && \
-    npm prune --production
+    npm prune --production && \
+    rm -rf /home/node/.npm
 
 EXPOSE 8191
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]

README.md

@@ -8,9 +8,7 @@
 [![Donate Buy Me A Coffee](https://img.shields.io/badge/Donate-Buy%20me%20a%20coffee-yellow.svg)](https://www.buymeacoffee.com/ngosang)
 [![Donate Bitcoin](https://img.shields.io/badge/Donate-Bitcoin-orange.svg)](https://en.cryptobadges.io/donate/13Hcv77AdnFWEUZ9qUpoPBttQsUT7q9TTh)
 
-FlareSolverr is a proxy server to bypass Cloudflare protection
-
-:warning: This project is in beta state. Some things may not work and the API can change at any time.
+FlareSolverr is a proxy server to bypass Cloudflare protection.
 
 ## How it works
 
@@ -68,11 +66,12 @@ This is the recommended way for Windows users.
 ### From source code
 
 This is the recommended way for macOS users and for developers.
-* Install [NodeJS](https://nodejs.org/)
-* Clone this repository and open a shell in that path
-* Run `npm install` command to install FlareSolverr dependencies
-* Run `npm run build` command to compile TypeScript code
-* Run `npm start` command to start FlareSolverr
+* Install [NodeJS](https://nodejs.org/).
+* Clone this repository and open a shell in that path.
+* Run `npm install` command to install FlareSolverr dependencies.
+* Run `node node_modules/puppeteer/install.js` to install Chromium.
+* Run `npm run build` command to compile TypeScript code.
+* Run `npm start` command to start FlareSolverr.
 
 ### Systemd service
 
@@ -147,6 +146,7 @@ headers | Optional. To specify user headers.
 maxTimeout | Optional, default value 60000. Max timeout to solve the challenge in milliseconds.
 cookies | Optional. Will be used by the headless browser. Follow [this](https://github.com/puppeteer/puppeteer/blob/v3.3.0/docs/api.md#pagesetcookiecookies) format.
 returnOnlyCookies | Optional, default false. Only returns the cookies. Response data, headers and other parts of the response are removed.
+returnRawHtml | Optional, default false. The response data will be returned without JS processing. This is useful for JSON or plain text content.
 
 Example response from running the `curl` above:
 

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "flaresolverr",
-  "version": "1.2.6",
+  "version": "1.2.9",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "flaresolverr",
-      "version": "1.2.5",
+      "version": "1.2.8",
       "license": "MIT",
       "dependencies": {
         "await-timeout": "^1.1.1",
@@ -2096,9 +2096,9 @@
       }
     },
     "node_modules/glob-parent": {
-      "version": "5.1.1",
-      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz",
-      "integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==",
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz",
+      "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==",
       "dev": true,
       "dependencies": {
         "is-glob": "^4.0.1"
@@ -2913,9 +2913,9 @@
       "dev": true
     },
     "node_modules/merge-deep": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/merge-deep/-/merge-deep-3.0.2.tgz",
-      "integrity": "sha512-T7qC8kg4Zoti1cFd8Cr0M+qaZfOwjlPDEdZIIPPB2JZctjaPM4fX+i7HOId69tAti2fvO6X5ldfYUONDODsrkA==",
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/merge-deep/-/merge-deep-3.0.3.tgz",
+      "integrity": "sha512-qtmzAS6t6grwEkNrunqTBdn0qKwFgNWvlxUbAV8es9M7Ot1EbyApytCnvE0jALPa46ZpKDUo527kKiaWplmlFA==",
       "dependencies": {
         "arr-union": "^3.1.0",
         "clone-deep": "^0.2.4",
@@ -3178,9 +3178,9 @@
       }
     },
     "node_modules/normalize-url": {
-      "version": "4.5.0",
-      "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.0.tgz",
-      "integrity": "sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ==",
+      "version": "4.5.1",
+      "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.1.tgz",
+      "integrity": "sha512-9UZCFRHQdNrfTpGg8+1INIg93B6zE0aXMVFkw1WFwvO4SlZywU6aLg5Of0Ap/PgcbSw4LNxvMWXMeugwMCX0AA==",
       "engines": {
         "node": ">=8"
       }
@@ -6860,9 +6860,9 @@
       }
     },
     "glob-parent": {
-      "version": "5.1.1",
-      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz",
-      "integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==",
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz",
+      "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==",
       "dev": true,
       "requires": {
         "is-glob": "^4.0.1"
@@ -7523,9 +7523,9 @@
       "dev": true
     },
     "merge-deep": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/merge-deep/-/merge-deep-3.0.2.tgz",
-      "integrity": "sha512-T7qC8kg4Zoti1cFd8Cr0M+qaZfOwjlPDEdZIIPPB2JZctjaPM4fX+i7HOId69tAti2fvO6X5ldfYUONDODsrkA==",
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/merge-deep/-/merge-deep-3.0.3.tgz",
+      "integrity": "sha512-qtmzAS6t6grwEkNrunqTBdn0qKwFgNWvlxUbAV8es9M7Ot1EbyApytCnvE0jALPa46ZpKDUo527kKiaWplmlFA==",
       "requires": {
         "arr-union": "^3.1.0",
         "clone-deep": "^0.2.4",
@@ -7739,9 +7739,9 @@
       "dev": true
     },
     "normalize-url": {
-      "version": "4.5.0",
-      "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.0.tgz",
-      "integrity": "sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ=="
+      "version": "4.5.1",
+      "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.1.tgz",
+      "integrity": "sha512-9UZCFRHQdNrfTpGg8+1INIg93B6zE0aXMVFkw1WFwvO4SlZywU6aLg5Of0Ap/PgcbSw4LNxvMWXMeugwMCX0AA=="
     },
     "oauth-sign": {
       "version": "0.9.0",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "flaresolverr",
-  "version": "1.2.6",
+  "version": "1.2.9",
   "description": "Proxy server to bypass Cloudflare protection.",
   "scripts": {
     "start": "node ./dist/index.js",

src/index.ts

@@ -1,6 +1,7 @@
 const fs = require('fs');
 const os = require('os');
 const path = require('path');
+const process = require('process')
 import log from './log'
 import { createServer, IncomingMessage, ServerResponse } from 'http';
 import { RequestContext } from './types'
@@ -118,7 +119,15 @@ function validateIncomingRequest(ctx: RequestContext, params: BaseAPICall) {
 // init
 log.info(`FlareSolverr ${version}`);
 log.debug('Debug log enabled');
+
+process.on('SIGTERM', () => {
+  // Capture signal on Docker Stop #158
+  log.info("Process interrupted")
+  process.exit(0)
+})
+
 validateEnvironmentVariables();
+
 testChromeInstallation()
 .catch(e => {
   log.error("Error starting Chrome browser.", e);

src/providers/cloudflare.ts

@@ -20,8 +20,8 @@ export default async function resolveChallenge(url: string, page: Page, response
   }
   log.info('Cloudflare detected');
 
-  if (await page.$('.cf-error-code')) {
-    throw new Error('Cloudflare has blocked this request (Code 1020 Detected).')
+  if (await page.$('span[data-translate="error"]') || (await page.content()).includes('error code: 1020')) {
+    throw new Error('Cloudflare has blocked this request. Probably your IP is banned for this site, check in your web browser.')
   }
 
   let selectorFoundCount = 0;
@@ -35,37 +35,51 @@ export default async function resolveChallenge(url: string, page: Page, response
         log.debug('Waiting for Cloudflare challenge...')
 
         while (true) {
-          await page.waitFor(1000)
-          try {
-            // catch exception timeout in waitForNavigation
-            response = await page.waitForNavigation({ waitUntil: 'domcontentloaded', timeout: 9000 })
-          } catch (error) { }
-
           try {
             // catch Execution context was destroyed
             const cfChallengeElem = await page.$(selector)
             if (!cfChallengeElem) {
               // solved!
+              log.debug('Challenge element not found.')
               break
             } else {
+              // new Cloudflare Challenge #cf-please-wait
               const displayStyle = await page.evaluate((selector) => {
                 return getComputedStyle(document.querySelector(selector)).getPropertyValue("display");
               }, selector);
               if (displayStyle == "none") {
                 // spinner is hidden, could be a captcha or not
-                await page.waitFor(1000)
+                log.debug('Challenge element is hidden.')
+                // wait until redirecting disappears
+                while (true) {
+                  try {
+                    await page.waitFor(1000)
+                    const displayStyle2 = await page.evaluate(() => {
+                      return getComputedStyle(document.querySelector('#cf-spinner-redirecting')).getPropertyValue("display");
+                    });
+                    if (displayStyle2 == "none") {
+                      break // hCaptcha detected
+                    }
+                  } catch (error) {
+                    break // redirection completed
+                  }
+                }
                 break
+              } else {
+                log.debug('Challenge element is visible.')
               }
             }
-            log.debug('Found challenge element again...')
+            log.debug('Found challenge element again.')
           } catch (error)
           {
             log.debug("Unexpected error: " + error);
+            if (!error.toString().includes("Execution context was destroyed")) {
+              break
+            }
           }
 
-          response = await page.reload({ waitUntil: 'domcontentloaded' })
-          log.debug('Page reloaded.')
-          log.html(await page.content())
+          log.debug('Waiting for Cloudflare challenge...')
+          await page.waitFor(1000)
         }
 
         log.debug('Validating HTML code...')
@@ -157,7 +171,7 @@ export default async function resolveChallenge(url: string, page: Page, response
         }
 
         // submit captcha response
-        challengeForm.evaluate((e: HTMLFormElement) => e.submit())
+        await challengeForm.evaluate((e: HTMLFormElement) => e.submit())
         response = await page.waitForNavigation({ waitUntil: 'domcontentloaded' })
 
         if (await page.$('input[name="cf_captcha_kind"]')) {
@@ -171,6 +185,12 @@ export default async function resolveChallenge(url: string, page: Page, response
     if (selectorFoundCount == 0)
     {
       throw new Error('No challenge selectors found, unable to proceed')
+    } else {
+      // reload the page to make sure we get the real response
+      // do not use page.reload() to avoid #162 #143
+      response = await page.goto(url, { waitUntil: 'domcontentloaded' })
+      await page.content()
+      log.info('Challenge solved.');
     }
   }
 

src/routes.ts

@@ -36,9 +36,9 @@ interface BaseRequestAPICall extends BaseAPICall {
   proxy?: any, // TODO: use interface not any
   download?: boolean
   returnOnlyCookies?: boolean
+  returnRawHtml?: boolean
 }
 
-
 interface Routes {
   [key: string]: (ctx: RequestContext, params: BaseAPICall) => void | Promise<void>
 }
@@ -86,7 +86,9 @@ async function resolveChallengeWithTimeout(ctx: RequestContext, params: BaseRequ
   }
 }
 
-async function resolveChallenge(ctx: RequestContext, { url, proxy, download, returnOnlyCookies }: BaseRequestAPICall, page: Page): Promise<ChallengeResolutionT | void> {
+async function resolveChallenge(ctx: RequestContext,
+                                { url, proxy, download, returnOnlyCookies, returnRawHtml }: BaseRequestAPICall,
+                                page: Page): Promise<ChallengeResolutionT | void> {
 
   let status = 'ok'
   let message = ''
@@ -132,6 +134,8 @@ async function resolveChallenge(ctx: RequestContext, { url, proxy, download, ret
       // fix since I am short on time
       response = await page.goto(url, { waitUntil: 'domcontentloaded' })
       payload.result.response = (await response.buffer()).toString('base64')
+    } else if (returnRawHtml) {
+      payload.result.response = await response.text()
     } else {
       payload.result.response = await page.content()
     }
@@ -197,6 +201,12 @@ async function setupPage(ctx: RequestContext, params: BaseRequestAPICall, browse
     let callbackRunOnce = false
     const callback = (request: Request) => {
 
+      // avoid loading resources to speed up page load
+      if(request.resourceType() == 'stylesheet' || request.resourceType() == 'font' || request.resourceType() == 'image') {
+        request.abort()
+        return
+      }
+
       if (callbackRunOnce || !request.isNavigationRequest()) {
         request.continue()
         return