Bump version 1.2.9

Dockerfile

@@ -22,8 +22,8 @@ ENV PUPPETEER_PRODUCT=chrome \
     PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
 RUN npm install && \
     npm run build && \
-    rm -rf src tsconfig.json && \
-    npm prune --production
+    npm prune --production && \
+    rm -rf /home/node/.npm
 
 EXPOSE 8191
 ENTRYPOINT ["/usr/bin/dumb-init", "--"]

README.md

@@ -146,6 +146,7 @@ headers | Optional. To specify user headers.
 maxTimeout | Optional, default value 60000. Max timeout to solve the challenge in milliseconds.
 cookies | Optional. Will be used by the headless browser. Follow [this](https://github.com/puppeteer/puppeteer/blob/v3.3.0/docs/api.md#pagesetcookiecookies) format.
 returnOnlyCookies | Optional, default false. Only returns the cookies. Response data, headers and other parts of the response are removed.
+returnRawHtml | Optional, default false. The response data will be returned without JS processing. This is useful for JSON or plain text content.
 
 Example response from running the `curl` above:
 

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "flaresolverr",
-  "version": "1.2.7",
+  "version": "1.2.9",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "flaresolverr",
-      "version": "1.2.5",
+      "version": "1.2.8",
       "license": "MIT",
       "dependencies": {
         "await-timeout": "^1.1.1",
@@ -2096,9 +2096,9 @@
       }
     },
     "node_modules/glob-parent": {
-      "version": "5.1.1",
-      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz",
-      "integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==",
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz",
+      "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==",
       "dev": true,
       "dependencies": {
         "is-glob": "^4.0.1"
@@ -2913,9 +2913,9 @@
       "dev": true
     },
     "node_modules/merge-deep": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/merge-deep/-/merge-deep-3.0.2.tgz",
-      "integrity": "sha512-T7qC8kg4Zoti1cFd8Cr0M+qaZfOwjlPDEdZIIPPB2JZctjaPM4fX+i7HOId69tAti2fvO6X5ldfYUONDODsrkA==",
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/merge-deep/-/merge-deep-3.0.3.tgz",
+      "integrity": "sha512-qtmzAS6t6grwEkNrunqTBdn0qKwFgNWvlxUbAV8es9M7Ot1EbyApytCnvE0jALPa46ZpKDUo527kKiaWplmlFA==",
       "dependencies": {
         "arr-union": "^3.1.0",
         "clone-deep": "^0.2.4",
@@ -3178,9 +3178,9 @@
       }
     },
     "node_modules/normalize-url": {
-      "version": "4.5.0",
-      "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.0.tgz",
-      "integrity": "sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ==",
+      "version": "4.5.1",
+      "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.1.tgz",
+      "integrity": "sha512-9UZCFRHQdNrfTpGg8+1INIg93B6zE0aXMVFkw1WFwvO4SlZywU6aLg5Of0Ap/PgcbSw4LNxvMWXMeugwMCX0AA==",
       "engines": {
         "node": ">=8"
       }
@@ -6860,9 +6860,9 @@
       }
     },
     "glob-parent": {
-      "version": "5.1.1",
-      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz",
-      "integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==",
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz",
+      "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==",
       "dev": true,
       "requires": {
         "is-glob": "^4.0.1"
@@ -7523,9 +7523,9 @@
       "dev": true
     },
     "merge-deep": {
-      "version": "3.0.2",
-      "resolved": "https://registry.npmjs.org/merge-deep/-/merge-deep-3.0.2.tgz",
-      "integrity": "sha512-T7qC8kg4Zoti1cFd8Cr0M+qaZfOwjlPDEdZIIPPB2JZctjaPM4fX+i7HOId69tAti2fvO6X5ldfYUONDODsrkA==",
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/merge-deep/-/merge-deep-3.0.3.tgz",
+      "integrity": "sha512-qtmzAS6t6grwEkNrunqTBdn0qKwFgNWvlxUbAV8es9M7Ot1EbyApytCnvE0jALPa46ZpKDUo527kKiaWplmlFA==",
       "requires": {
         "arr-union": "^3.1.0",
         "clone-deep": "^0.2.4",
@@ -7739,9 +7739,9 @@
       "dev": true
     },
     "normalize-url": {
-      "version": "4.5.0",
-      "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.0.tgz",
-      "integrity": "sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ=="
+      "version": "4.5.1",
+      "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.1.tgz",
+      "integrity": "sha512-9UZCFRHQdNrfTpGg8+1INIg93B6zE0aXMVFkw1WFwvO4SlZywU6aLg5Of0Ap/PgcbSw4LNxvMWXMeugwMCX0AA=="
     },
     "oauth-sign": {
       "version": "0.9.0",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "flaresolverr",
-  "version": "1.2.7",
+  "version": "1.2.9",
   "description": "Proxy server to bypass Cloudflare protection.",
   "scripts": {
     "start": "node ./dist/index.js",

src/index.ts

@@ -1,6 +1,7 @@
 const fs = require('fs');
 const os = require('os');
 const path = require('path');
+const process = require('process')
 import log from './log'
 import { createServer, IncomingMessage, ServerResponse } from 'http';
 import { RequestContext } from './types'
@@ -118,7 +119,15 @@ function validateIncomingRequest(ctx: RequestContext, params: BaseAPICall) {
 // init
 log.info(`FlareSolverr ${version}`);
 log.debug('Debug log enabled');
+
+process.on('SIGTERM', () => {
+  // Capture signal on Docker Stop #158
+  log.info("Process interrupted")
+  process.exit(0)
+})
+
 validateEnvironmentVariables();
+
 testChromeInstallation()
 .catch(e => {
   log.error("Error starting Chrome browser.", e);

src/providers/cloudflare.ts

@@ -20,8 +20,8 @@ export default async function resolveChallenge(url: string, page: Page, response
   }
   log.info('Cloudflare detected');
 
-  if (await page.$('.cf-error-code')) {
-    throw new Error('Cloudflare has blocked this request (Code 1020 Detected).')
+  if (await page.$('span[data-translate="error"]') || (await page.content()).includes('error code: 1020')) {
+    throw new Error('Cloudflare has blocked this request. Probably your IP is banned for this site, check in your web browser.')
   }
 
   let selectorFoundCount = 0;
@@ -73,7 +73,9 @@ export default async function resolveChallenge(url: string, page: Page, response
           } catch (error)
           {
             log.debug("Unexpected error: " + error);
-            break
+            if (!error.toString().includes("Execution context was destroyed")) {
+              break
+            }
           }
 
           log.debug('Waiting for Cloudflare challenge...')
@@ -169,7 +171,7 @@ export default async function resolveChallenge(url: string, page: Page, response
         }
 
         // submit captcha response
-        challengeForm.evaluate((e: HTMLFormElement) => e.submit())
+        await challengeForm.evaluate((e: HTMLFormElement) => e.submit())
         response = await page.waitForNavigation({ waitUntil: 'domcontentloaded' })
 
         if (await page.$('input[name="cf_captcha_kind"]')) {
@@ -185,7 +187,9 @@ export default async function resolveChallenge(url: string, page: Page, response
       throw new Error('No challenge selectors found, unable to proceed')
     } else {
       // reload the page to make sure we get the real response
-      response = await page.reload()
+      // do not use page.reload() to avoid #162 #143
+      response = await page.goto(url, { waitUntil: 'domcontentloaded' })
+      await page.content()
       log.info('Challenge solved.');
     }
   }

src/routes.ts

@@ -36,9 +36,9 @@ interface BaseRequestAPICall extends BaseAPICall {
   proxy?: any, // TODO: use interface not any
   download?: boolean
   returnOnlyCookies?: boolean
+  returnRawHtml?: boolean
 }
 
-
 interface Routes {
   [key: string]: (ctx: RequestContext, params: BaseAPICall) => void | Promise<void>
 }
@@ -86,7 +86,9 @@ async function resolveChallengeWithTimeout(ctx: RequestContext, params: BaseRequ
   }
 }
 
-async function resolveChallenge(ctx: RequestContext, { url, proxy, download, returnOnlyCookies }: BaseRequestAPICall, page: Page): Promise<ChallengeResolutionT | void> {
+async function resolveChallenge(ctx: RequestContext,
+                                { url, proxy, download, returnOnlyCookies, returnRawHtml }: BaseRequestAPICall,
+                                page: Page): Promise<ChallengeResolutionT | void> {
 
   let status = 'ok'
   let message = ''
@@ -132,6 +134,8 @@ async function resolveChallenge(ctx: RequestContext, { url, proxy, download, ret
       // fix since I am short on time
       response = await page.goto(url, { waitUntil: 'domcontentloaded' })
       payload.result.response = (await response.buffer()).toString('base64')
+    } else if (returnRawHtml) {
+      payload.result.response = await response.text()
     } else {
       payload.result.response = await page.content()
     }
@@ -197,6 +201,12 @@ async function setupPage(ctx: RequestContext, params: BaseRequestAPICall, browse
     let callbackRunOnce = false
     const callback = (request: Request) => {
 
+      // avoid loading resources to speed up page load
+      if(request.resourceType() == 'stylesheet' || request.resourceType() == 'font' || request.resourceType() == 'image') {
+        request.abort()
+        return
+      }
+
       if (callbackRunOnce || !request.isNavigationRequest()) {
         request.continue()
         return