Bump version 2.2.3

Clarify that `request.get` will not use the provided proxy when a session is set.

Dockerfile

@@ -1,4 +1,4 @@
-FROM --platform=${TARGETPLATFORM:-linux/amd64} node:16-alpine3.14
+FROM --platform=${TARGETPLATFORM:-linux/amd64} node:16-alpine3.15
 
 # Print build information
 ARG TARGETPLATFORM

README.md

@@ -5,8 +5,8 @@
 [![GitHub issues](https://img.shields.io/github/issues/FlareSolverr/FlareSolverr)](https://github.com/FlareSolverr/FlareSolverr/issues)
 [![GitHub pull requests](https://img.shields.io/github/issues-pr/FlareSolverr/FlareSolverr)](https://github.com/FlareSolverr/FlareSolverr/pulls)
 [![Donate PayPal](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=X5NJLLX5GLTV6&source=url)
-[![Donate Buy Me A Coffee](https://img.shields.io/badge/Donate-Buy%20me%20a%20coffee-yellow.svg)](https://www.buymeacoffee.com/ngosang)
+[![Donate Bitcoin](https://en.cryptobadges.io/badge/micro/13Hcv77AdnFWEUZ9qUpoPBttQsUT7q9TTh)](https://en.cryptobadges.io/donate/13Hcv77AdnFWEUZ9qUpoPBttQsUT7q9TTh)
-[![Donate Bitcoin](https://img.shields.io/badge/Donate-Bitcoin-orange.svg)](https://en.cryptobadges.io/donate/13Hcv77AdnFWEUZ9qUpoPBttQsUT7q9TTh)
+[![Donate Ethereum](https://en.cryptobadges.io/badge/micro/0x0D1549BbB00926BF3D92c1A8A58695e982f1BE2E)](https://en.cryptobadges.io/donate/0x0D1549BbB00926BF3D92c1A8A58695e982f1BE2E)
 
 FlareSolverr is a proxy server to bypass Cloudflare protection.
 
@@ -56,6 +56,10 @@ docker run -d \
   ghcr.io/flaresolverr/flaresolverr:latest
 ```
 
+If your host OS is Debian, make sure `libseccomp2` version is 2.5.x. You can check the version with `sudo apt-cache policy libseccomp2` 
+and update the package with `sudo apt install libseccomp2=2.5.1-1~bpo10+1` or `sudo apt install libseccomp2=2.5.1-1+deb11u1`.
+Remember to restart the Docker daemon and the container after the update.
+
 ### Precompiled binaries
 
 This is the recommended way for Windows users.
@@ -66,13 +70,13 @@ This is the recommended way for Windows users.
 ### From source code
 
 This is the recommended way for macOS users and for developers.
-* Install [NodeJS](https://nodejs.org/).
+* Install [NodeJS](https://nodejs.org/) 16.
 * Clone this repository and open a shell in that path.
 * Run `export PUPPETEER_PRODUCT=firefox` (Linux/macOS) or `set PUPPETEER_PRODUCT=firefox` (Windows).
 * Run `npm install` command to install FlareSolverr dependencies.
-* Run `node node_modules/puppeteer/install.js` to install Firefox.
+* Run `npm start` command to compile TypeScript code and start FlareSolverr.
-* Run `npm run build` command to compile TypeScript code.
+
-* Run `npm start` command to start FlareSolverr.
+If you get errors related to firefox not installed try running `node node_modules/puppeteer/install.js` to install Firefox.
 
 ### Systemd service
 
@@ -104,6 +108,7 @@ This also speeds up the requests since it won't have to launch a new browser ins
 Parameter | Notes
 |--|--|
 session | Optional. The session ID that you want to be assigned to the instance. If isn't set a random UUID will be assigned.
+proxy | Optional, default disabled. Eg: `"proxy": {"url": "http://127.0.0.1:8888"}`. You must include the proxy schema in the URL: `http://`, `socks4://` or `socks5://`. Authorization (username/password) is not supported.
 
 #### + `sessions.list`
 
@@ -141,7 +146,7 @@ session | Optional. Will send the request from and existing browser instance. If
 maxTimeout | Optional, default value 60000. Max timeout to solve the challenge in milliseconds.
 cookies | Optional. Will be used by the headless browser. Follow [this](https://github.com/puppeteer/puppeteer/blob/v3.3.0/docs/api.md#pagesetcookiecookies) format.
 returnOnlyCookies | Optional, default false. Only returns the cookies. Response data, headers and other parts of the response are removed.
-proxy | Optional, default disabled. Eg: `"proxy": {"url": "http://127.0.0.1:8888"}`. Authorization (username/password) is not supported.
+proxy | Optional, default disabled. Eg: `"proxy": {"url": "http://127.0.0.1:8888"}`. You must include the proxy schema in the URL: `http://`, `socks4://` or `socks5://`. Authorization (username/password) is not supported. (When the `session` parameter is set, the proxy is ignored; a session specific proxy can be set in `sessions.create`.)
 
 :warning: If you want to use Cloudflare clearance cookie in your scripts, make sure you use the FlareSolverr User-Agent too. If they don't match you will see the challenge.
 
@@ -210,7 +215,7 @@ This is the same as `request.get` but it takes one more param:
 
 Parameter | Notes
 |--|--|
-postData | Must be a string with `application/x-www-form-urlencoded`. Eg: `postData": "a=b&c=d"`
+postData | Must be a string with `application/x-www-form-urlencoded`. Eg: `a=b&c=d`
 
 ## Environment variables
 
@@ -221,7 +226,8 @@ LOG_HTML | false | Only for debugging. If `true` all HTML that passes through th
 CAPTCHA_SOLVER | none | Captcha solving method. It is used when a captcha is encountered. See the Captcha Solvers section.
 TZ | UTC | Timezone used in the logs and the web browser. Example: `TZ=Europe/London`.
 HEADLESS | true | Only for debugging. To run the web browser in headless mode or visible.
-BROWSER_TIMEOUT | 30000 | If you are experiencing errors/timeouts because your system is slow, you can try to increase this value. Remember to increase the `maxTimeout` parameter too.
+BROWSER_TIMEOUT | 40000 | If you are experiencing errors/timeouts because your system is slow, you can try to increase this value. Remember to increase the `maxTimeout` parameter too.
+TEST_URL | https://www.google.com | FlareSolverr makes a request on start to make sure the web browser is working. You can change that URL if it is blocked in your country.
 PORT | 8191 | Listening port. You don't need to change this if you are running on Docker.
 HOST | 0.0.0.0 | Listening interface. You don't need to change this if you are running on Docker.
 

build-binaries.js

@@ -64,8 +64,8 @@ function getFirefoxNightlyVersion() {
   if (fs.existsSync('bin')) {
     fs.rmSync('bin', { recursive: true })
   }
-  execSync('./node_modules/.bin/pkg -t node14-win-x64,node14-linux-x64 --out-path bin .')
+  execSync('./node_modules/.bin/pkg -t node16-win-x64,node16-linux-x64 --out-path bin .')
-  // execSync('./node_modules/.bin/pkg -t node14-win-x64,node14-mac-x64,node14-linux-x64 --out-path bin .')
+  // execSync('./node_modules/.bin/pkg -t node16-win-x64,node16-mac-x64,node16-linux-x64 --out-path bin .')
 
   // get firefox revision
   const revision = await getFirefoxNightlyVersion();

package.json

@@ -1,12 +1,12 @@
 {
   "name": "flaresolverr",
-  "version": "2.0.2",
+  "version": "2.2.3",
   "description": "Proxy server to bypass Cloudflare protection.",
   "scripts": {
-    "start": "node ./dist/server.js",
+    "start": "tsc && node ./dist/server.js",
     "build": "tsc",
     "dev": "nodemon -e ts --exec ts-node src/server.ts",
-    "package": "node build-binaries.js",
+    "package": "tsc && node build-binaries.js",
     "test": "jest --runInBand"
   },
   "author": "Diego Heras (ngosang)",
@@ -20,27 +20,26 @@
   },
   "dependencies": {
     "await-timeout": "^1.1.1",
-    "body-parser": "^1.19.0",
+    "body-parser": "^1.20.0",
     "console-log-level": "^1.4.1",
-    "express": "^4.17.1",
+    "express": "^4.17.3",
-    "puppeteer": "^3.3.0",
+    "puppeteer": "^13.5.2",
     "uuid": "^8.3.2"
   },
   "devDependencies": {
     "@types/await-timeout": "^0.3.1",
     "@types/body-parser": "^1.19.1",
     "@types/express": "^4.17.13",
-    "@types/jest": "^27.0.2",
+    "@types/jest": "^27.4.1",
-    "@types/node": "^14.17.27",
+    "@types/node": "^16.11.27",
-    "@types/puppeteer": "^3.0.6",
+    "@types/supertest": "^2.0.12",
-    "@types/supertest": "^2.0.11",
     "@types/uuid": "^8.3.1",
-    "archiver": "^5.3.0",
+    "archiver": "^5.3.1",
     "nodemon": "^2.0.13",
-    "pkg": "^5.3.3",
+    "pkg": "^5.6.0",
     "supertest": "^6.1.6",
-    "ts-jest": "^27.0.7",
+    "ts-jest": "^27.1.4",
-    "ts-node": "^10.3.0",
+    "ts-node": "^10.7.0",
-    "typescript": "^4.4.4"
+    "typescript": "^4.6.3"
   }
 }

src/controllers/v1.ts

@@ -1,6 +1,5 @@
-// todo: avoid puppeter objects
-import {SetCookie, Headers, HttpMethod} from 'puppeteer'
 import {Request, Response} from 'express';
+import {Protocol} from "devtools-protocol";
 
 import log from '../services/log'
 import {browserRequest, ChallengeResolutionResultT, ChallengeResolutionT} from "../services/solver";
@@ -20,11 +19,11 @@ export interface Proxy {
 
 export interface V1RequestBase {
   cmd: string
-  cookies?: SetCookie[],
+  cookies?: Protocol.Network.CookieParam[],
   maxTimeout?: number
   proxy?: Proxy
   session: string
-  headers?: Headers // deprecated v2, not used
+  headers?: Record<string, string> // deprecated v2, not used
   userAgent?: string // deprecated v2, not used
 }
 
@@ -33,7 +32,7 @@ interface V1RequestSession extends V1RequestBase {
 
 export interface V1Request extends V1RequestBase {
   url: string
-  method?: HttpMethod
+  method?: string
   postData?: string
   returnOnlyCookies?: boolean
   download?: boolean // deprecated v2, not used

src/providers/cloudflare.ts

@@ -1,4 +1,4 @@
-import {Page, Response} from 'puppeteer'
+import {Page, HTTPResponse} from 'puppeteer'
 
 import log from "../services/log";
 
@@ -6,16 +6,34 @@ import log from "../services/log";
  *  This class contains the logic to solve protections provided by CloudFlare
  **/
 
-const BAN_SELECTORS = ['span[data-translate="error"]'];
+// the selector '.text-gray-600' is not working well because it can be hidden
-const CHALLENGE_SELECTORS = ['#trk_jschal_js', '.ray_id', '.attack-box', '#cf-please-wait'];
+// <span style="display: none;" class="text-gray-600" data-translate="error">error code: 1020</span>
-const CAPTCHA_SELECTORS = ['input[name="cf_captcha_kind"]'];
+const BAN_SELECTORS: string[] = [];
+const CHALLENGE_SELECTORS: string[] = [
+    '#trk_jschal_js', '.ray_id', '.attack-box', '#cf-please-wait', // CloudFlare
+    '#link-ddg', // DDoS-GUARD
+    'td.info #js_info' // Custom CloudFlare for EbookParadijs, Film-Paleis, MuziekFabriek and Puur-Hollands
+];
+const CAPTCHA_SELECTORS: string[] = [
+    'input[name="cf_captcha_kind"]'
+];
 
-export default async function resolveChallenge(url: string, page: Page, response: Response): Promise<Response> {
+export default async function resolveChallenge(url: string, page: Page, response: HTTPResponse): Promise<HTTPResponse> {
 
   // look for challenge and return fast if not detected
-  if (response.headers().server &&
+  let cfDetected = response.headers().server && response.headers().server.startsWith('cloudflare');
-      response.headers().server.startsWith('cloudflare') &&
+  if (cfDetected) {
-      (response.status() == 403 || response.status() == 503)) {
+    if (response.status() == 403 || response.status() == 503) {
+      cfDetected = true; // Defected CloudFlare and DDoS-GUARD
+    } else if (response.headers().vary && response.headers().vary.trim() == 'Accept-Encoding,User-Agent' &&
+        response.headers()['content-encoding'] && response.headers()['content-encoding'].trim() == 'br') {
+      cfDetected = true; // Detected Custom CloudFlare for EbookParadijs, Film-Paleis, MuziekFabriek and Puur-Hollands
+    } else {
+      cfDetected = false;
+    }
+  }
+
+  if (cfDetected) {
     log.info('Cloudflare detected');
   } else {
     log.info('Cloudflare not detected');
@@ -23,79 +41,71 @@ export default async function resolveChallenge(url: string, page: Page, response
   }
 
   if (await findAnySelector(page, BAN_SELECTORS)) {
-    throw new Error('Cloudflare has blocked this request. Probably your IP is banned for this site, check in your web browser.')
+    throw new Error('Cloudflare has blocked this request. Probably your IP is banned for this site, check in your web browser.');
   }
 
+  // find Cloudflare selectors
   let selectorFound = false;
-  if (response.status() > 400) {
+  let selector: string = await findAnySelector(page, CHALLENGE_SELECTORS)
+  if (selector) {
+    selectorFound = true;
+    log.debug(`Javascript challenge element '${selector}' detected.`)
+    log.debug('Waiting for Cloudflare challenge...')
 
-    // find Cloudflare selectors
+    while (true) {
-    let selector: string = await findAnySelector(page, CHALLENGE_SELECTORS)
+      try {
-    if (selector) {
-      selectorFound = true;
-      log.debug(`Javascript challenge element '${selector}' detected.`)
-      log.debug('Waiting for Cloudflare challenge...')
 
-      while (true) {
+        selector = await findAnySelector(page, CHALLENGE_SELECTORS)
-        try {
+        if (!selector) {
+          // solved!
+          log.debug('Challenge element not found')
+          break
+        } else {
+          log.debug(`Javascript challenge element '${selector}' detected.`)
 
-          selector = await findAnySelector(page, CHALLENGE_SELECTORS)
+          // new Cloudflare Challenge #cf-please-wait
-          if (!selector) {
+          const displayStyle = await page.evaluate((selector) => {
-            // solved!
+            return getComputedStyle(document.querySelector(selector)).getPropertyValue("display");
-            log.debug('Challenge element not found')
+          }, selector);
+          if (displayStyle == "none") {
+            // spinner is hidden, could be a captcha or not
+            log.debug('Challenge element is hidden')
+            // wait until redirecting disappears
+            while (true) {
+              try {
+                await page.waitForTimeout(1000)
+                const displayStyle2 = await page.evaluate(() => {
+                  return getComputedStyle(document.querySelector('#cf-spinner-redirecting')).getPropertyValue("display");
+                });
+                if (displayStyle2 == "none") {
+                  break // hCaptcha detected
+                }
+              } catch (error) {
+                break // redirection completed
+              }
+            }
             break
           } else {
-            log.debug(`Javascript challenge element '${selector}' detected.`)
+            log.debug('Challenge element is visible')
-
-            // new Cloudflare Challenge #cf-please-wait
-            const displayStyle = await page.evaluate((selector) => {
-              return getComputedStyle(document.querySelector(selector)).getPropertyValue("display");
-            }, selector);
-            if (displayStyle == "none") {
-              // spinner is hidden, could be a captcha or not
-              log.debug('Challenge element is hidden')
-              // wait until redirecting disappears
-              while (true) {
-                try {
-                  await page.waitFor(1000)
-                  const displayStyle2 = await page.evaluate(() => {
-                    return getComputedStyle(document.querySelector('#cf-spinner-redirecting')).getPropertyValue("display");
-                  });
-                  if (displayStyle2 == "none") {
-                    break // hCaptcha detected
-                  }
-                } catch (error) {
-                  break // redirection completed
-                }
-              }
-              break
-            } else {
-              log.debug('Challenge element is visible')
-            }
-          }
-          log.debug('Found challenge element again')
-
-        } catch (error)
-        {
-          log.debug("Unexpected error: " + error);
-          if (!error.toString().includes("Execution context was destroyed")) {
-            break
           }
         }
+        log.debug('Found challenge element again')
 
-        log.debug('Waiting for Cloudflare challenge...')
+      } catch (error)
-        await page.waitFor(1000)
+      {
+        log.debug("Unexpected error: " + error);
+        if (!error.toString().includes("Execution context was destroyed")) {
+          break
+        }
       }
 
-      log.debug('Validating HTML code...')
+      log.debug('Waiting for Cloudflare challenge...')
-    } else {
+      await page.waitForTimeout(1000)
-      log.debug(`No challenge element detected.`)
     }
 
+    log.debug('Validating HTML code...')
   } else {
-    // some sites use cloudflare but there is no challenge
+    log.debug(`No challenge element detected.`)
-    log.debug(`Javascript challenge not detected. Status code: ${response.status()}`);
-    selectorFound = true;
   }
 
   // check for CAPTCHA challenge

src/server.ts

@@ -39,6 +39,11 @@ process.on('SIGTERM', () => {
   process.exit(0)
 })
 
+process.on('uncaughtException', function(err) {
+  // Avoid crashing in NodeJS 17 due to UnhandledPromiseRejectionWarning: Unhandled promise rejection.
+  log.error(err)
+})
+
 validateEnvironmentVariables();
 
 testWebBrowserInstallation().then(() => {

src/services/sessions.ts

@@ -1,6 +1,7 @@
 import {v1 as UUIDv1} from 'uuid'
 import * as path from 'path'
-import {SetCookie, Browser} from 'puppeteer'
+import {Browser} from 'puppeteer'
+import {Protocol} from "devtools-protocol";
 
 import log from './log'
 import {Proxy} from "../controllers/v1";
@@ -20,7 +21,7 @@ interface SessionsCache {
 
 export interface SessionCreateOptions {
   oneTimeSession: boolean
-  cookies?: SetCookie[],
+  cookies?: Protocol.Network.CookieParam[],
   maxTimeout?: number
   proxy?: Proxy
 }
@@ -46,8 +47,9 @@ function buildExtraPrefsFirefox(proxy: Proxy): object {
     "startup.homepage_welcome_url": "about:blank",
     "startup.homepage_welcome_url.additional": "",
 
-    // Disable images to speed up load
+    // Detected !
-    "permissions.default.image": 2,
+    // // Disable images to speed up load
+    // "permissions.default.image": 2,
 
     // Limit content processes to 1
     "dom.ipc.processCount": 1
@@ -55,22 +57,43 @@ function buildExtraPrefsFirefox(proxy: Proxy): object {
 
   // proxy.url format => http://<host>:<port>
   if (proxy && proxy.url) {
+    log.debug(`Using proxy: ${proxy.url}`)
     const [host, portStr] = proxy.url.replace(/.+:\/\//g, '').split(':');
     const port = parseInt(portStr);
+    if (!host || !portStr || !port) {
+      throw new Error("Proxy configuration is invalid! Use the format: protocol://ip:port")
+    }
 
     const proxyPrefs = {
-      // Proxy configuration
+      "network.proxy.type": 1,
-      "network.proxy.ftp": host,
+      "network.proxy.share_proxy_settings": true
-      "network.proxy.ftp_port": port,
+    }
-      "network.proxy.http": host,
+    if (proxy.url.indexOf("socks") != -1) {
-      "network.proxy.http_port": port,
+      // SOCKSv4 & SOCKSv5
-      "network.proxy.share_proxy_settings": true,
+      Object.assign(proxyPrefs, {
-      "network.proxy.socks": host,
+        "network.proxy.socks": host,
-      "network.proxy.socks_port": port,
+        "network.proxy.socks_port": port,
-      "network.proxy.socks_remote_dns": true,
+        "network.proxy.socks_remote_dns": true
-      "network.proxy.ssl": host,
+      });
-      "network.proxy.ssl_port": port,
+      if (proxy.url.indexOf("socks4") != -1) {
-      "network.proxy.type": 1
+        Object.assign(proxyPrefs, {
+          "network.proxy.socks_version": 4
+        });
+      } else {
+        Object.assign(proxyPrefs, {
+          "network.proxy.socks_version": 5
+        });
+      }
+    } else {
+      // HTTP
+      Object.assign(proxyPrefs, {
+        "network.proxy.ftp": host,
+        "network.proxy.ftp_port": port,
+        "network.proxy.http": host,
+        "network.proxy.http_port": port,
+        "network.proxy.ssl": host,
+        "network.proxy.ssl_port": port
+      });
     }
 
     // merge objects
@@ -93,16 +116,19 @@ export async function testWebBrowserInstallation(): Promise<void> {
   log.debug("FlareSolverr user home directory is OK: " + homeDir)
 
   // test web browser
+  const testUrl = process.env.TEST_URL || "https://www.google.com";
+  log.debug("Test URL: " + testUrl)
   const session = await create(null, {
     oneTimeSession: true
   })
   const page = await session.browser.newPage()
-  await page.goto("https://www.google.com")
+  const pageTimeout = Number(process.env.BROWSER_TIMEOUT) || 40000
+  await page.goto(testUrl, {waitUntil: 'domcontentloaded', timeout: pageTimeout})
   webBrowserUserAgent = await page.evaluate(() => navigator.userAgent)
 
   // replace Linux ARM user-agent because it's detected
-  if (webBrowserUserAgent.toLocaleLowerCase().includes('linux arm')) {
+  if (["arm", "aarch64"].some(arch => webBrowserUserAgent.toLocaleLowerCase().includes('linux ' + arch))) {
-    webBrowserUserAgent = webBrowserUserAgent.replace(/linux arm[^;]+;/i, 'Linux x86_64;')
+    webBrowserUserAgent = webBrowserUserAgent.replace(/linux \w+;/i, 'Linux x86_64;')
   }
 
   log.info("FlareSolverr User-Agent: " + webBrowserUserAgent)
@@ -113,6 +139,8 @@ export async function testWebBrowserInstallation(): Promise<void> {
 }
 
 export async function create(session: string, options: SessionCreateOptions): Promise<SessionsCacheItem> {
+  log.debug('Creating new session...')
+
   const sessionId = session || UUIDv1()
 
   // NOTE: cookies can't be set in the session, you need to open the page first
@@ -120,7 +148,7 @@ export async function create(session: string, options: SessionCreateOptions): Pr
   const puppeteerOptions: any = {
     product: 'firefox',
     headless: process.env.HEADLESS !== 'false',
-    timeout: process.env.BROWSER_TIMEOUT || 30000
+    timeout: Number(process.env.BROWSER_TIMEOUT) || 40000
   }
 
   puppeteerOptions.extraPrefsFirefox = buildExtraPrefsFirefox(options.proxy)

src/services/solver.ts

@@ -1,4 +1,4 @@
-import {Response, Headers, Page} from 'puppeteer'
+import {Page, HTTPResponse} from 'puppeteer'
 const Timeout = require('await-timeout');
 
 import log from './log'
@@ -11,7 +11,7 @@ const sessions = require('./sessions')
 export interface ChallengeResolutionResultT {
     url: string
     status: number,
-    headers?: Headers,
+    headers?: Record<string, string>,
     response: string,
     cookies: object[]
     userAgent: string
@@ -64,7 +64,7 @@ async function resolveChallenge(params: V1Request, session: SessionsCacheItem):
 
         // go to the page
         log.debug(`Navigating to... ${params.url}`)
-        let response: Response = await gotoPage(params, page);
+        let response: HTTPResponse = await gotoPage(params, page);
 
         // set cookies
         if (params.cookies) {
@@ -89,7 +89,11 @@ async function resolveChallenge(params: V1Request, session: SessionsCacheItem):
             // is response is ok
             // reload the page to be sure we get the real page
             log.debug("Reloading the page")
-            response = await gotoPage(params, page);
+            try {
+                response = await gotoPage(params, page);
+            } catch (e) {
+                log.warn("Page not reloaded (do not report!): Cause: " + e.toString())
+            }
 
         } catch (e) {
             status = "error";
@@ -128,15 +132,18 @@ async function resolveChallenge(params: V1Request, session: SessionsCacheItem):
     }
 }
 
-async function gotoPage(params: V1Request, page: Page): Promise<Response> {
+async function gotoPage(params: V1Request, page: Page): Promise<HTTPResponse> {
-    let response: Response;
+    let pageTimeout = params.maxTimeout / 3;
-    if (params.method != 'POST') {
+    let response: HTTPResponse
-        response = await page.goto(params.url, {waitUntil: 'domcontentloaded'});
+    try {
+        response = await page.goto(params.url, {waitUntil: 'domcontentloaded', timeout: pageTimeout});
+    } catch (e) {
+        // retry
+        response = await page.goto(params.url, {waitUntil: 'domcontentloaded', timeout: pageTimeout});
+    }
 
-    } else {
+    if (params.method == 'POST') {
         // post hack
-        // first request a page without cloudflare
-        response = await page.goto(params.url, {waitUntil: 'domcontentloaded'});
         await page.setContent(
             `
 <!DOCTYPE html>
@@ -177,7 +184,7 @@ async function gotoPage(params: V1Request, page: Page): Promise<Response> {
 </html> 
             `
         );
-        await page.waitFor(2000)
+        await page.waitForTimeout(2000)
         try {
             await page.waitForNavigation({waitUntil: 'domcontentloaded', timeout: 2000})
         } catch (e) {}

src/tests/app.test.ts

@@ -9,11 +9,14 @@ const sessions = require('../services/sessions');
 const version: string = 'v' + require('../../package.json').version
 
 const proxyUrl = "http://127.0.0.1:8888"
+const proxySocksUrl = "socks5://127.0.0.1:1080"
 const googleUrl = "https://www.google.com";
 const postUrl = "https://ptsv2.com/t/qv4j3-1634496523";
 const cfUrl = "https://pirateiro.com/torrents/?search=harry";
 const cfCaptchaUrl = "https://idope.se"
 const cfBlockedUrl = "https://www.torrentmafya.org/table.php"
+const ddgUrl = "https://www.erai-raws.info/feed/?type=magnet";
+const ccfUrl = "https://www.muziekfabriek.org";
 
 beforeAll(async () => {
     // Init session
@@ -167,6 +170,64 @@ describe("Test '/v1' path", () => {
         expect(apiResponse.solution.url).toContain(cfBlockedUrl)
     });
 
+    test("Cmd 'request.get' should return OK with DDoS-GUARD JS", async () => {
+        const payload = {
+            "cmd": "request.get",
+            "url": ddgUrl
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSolution = response.body;
+        expect(apiResponse.status).toBe("ok");
+        expect(apiResponse.message).toBe("");
+        expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
+        expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
+        expect(apiResponse.version).toBe(version);
+
+        const solution = apiResponse.solution;
+        expect(solution.url).toContain(ddgUrl)
+        expect(solution.status).toBe(200);
+        expect(Object.keys(solution.headers).length).toBeGreaterThan(0)
+        expect(solution.response).toContain("<rss version")
+        expect(Object.keys(solution.cookies).length).toBeGreaterThan(0)
+        expect(solution.userAgent).toContain("Firefox/")
+
+        const cfCookie: string = (solution.cookies as any[]).filter(function(cookie) {
+            return cookie.name == "__ddg1";
+        })[0].value
+        expect(cfCookie.length).toBeGreaterThan(10)
+    });
+
+    test("Cmd 'request.get' should return OK with Custom CloudFlare JS", async () => {
+        const payload = {
+            "cmd": "request.get",
+            "url": ccfUrl
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSolution = response.body;
+        expect(apiResponse.status).toBe("ok");
+        expect(apiResponse.message).toBe("");
+        expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
+        expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
+        expect(apiResponse.version).toBe(version);
+
+        const solution = apiResponse.solution;
+        expect(solution.url).toContain(ccfUrl)
+        expect(solution.status).toBe(200);
+        expect(Object.keys(solution.headers).length).toBeGreaterThan(0)
+        expect(solution.response).toContain("<html><head>")
+        expect(Object.keys(solution.cookies).length).toBeGreaterThan(0)
+        expect(solution.userAgent).toContain("Firefox/")
+
+        const cfCookie: string = (solution.cookies as any[]).filter(function(cookie) {
+            return cookie.name == "ct_anti_ddos_key";
+        })[0].value
+        expect(cfCookie.length).toBeGreaterThan(10)
+    });
+
     test("Cmd 'request.get' should return OK with 'cookies' param", async () => {
         const payload = {
             "cmd": "request.get",
@@ -221,7 +282,7 @@ describe("Test '/v1' path", () => {
         expect(solution.userAgent).toBe(null)
     });
 
-    test("Cmd 'request.get' should return OK with 'proxy' param", async () => {
+    test("Cmd 'request.get' should return OK with HTTP 'proxy' param", async () => {
         /*
         To configure TinyProxy in local:
            * sudo vim /etc/tinyproxy/tinyproxy.conf
@@ -249,7 +310,7 @@ describe("Test '/v1' path", () => {
     });
 
     // todo: credentials are not working
-    test.skip("Cmd 'request.get' should return OK with 'proxy' param with credentials", async () => {
+    test.skip("Cmd 'request.get' should return OK with HTTP 'proxy' param with credentials", async () => {
         /*
         To configure TinyProxy in local:
            * sudo vim /etc/tinyproxy/tinyproxy.conf
@@ -279,6 +340,32 @@ describe("Test '/v1' path", () => {
         expect(solution.status).toContain(200)
     });
 
+    test("Cmd 'request.get' should return OK with SOCKSv5 'proxy' param", async () => {
+        /*
+        To configure Dante in local:
+           * https://linuxhint.com/set-up-a-socks5-proxy-on-ubuntu-with-dante/
+           * sudo vim /etc/sockd.conf
+           * sudo systemctl restart sockd.service
+           * curl --socks5 socks5://127.0.0.1:1080 https://www.google.com
+        */
+        const payload = {
+            "cmd": "request.get",
+            "url": googleUrl,
+            "proxy": {
+                "url": proxySocksUrl
+            }
+        }
+        const response: Response = await request(app).post("/v1").send(payload);
+        expect(response.statusCode).toBe(200);
+
+        const apiResponse: V1ResponseSolution = response.body;
+        expect(apiResponse.status).toBe("ok");
+
+        const solution = apiResponse.solution;
+        expect(solution.url).toContain(googleUrl)
+        expect(solution.status).toBe(200);
+    });
+
     test("Cmd 'request.get' should fail with wrong 'proxy' param", async () => {
         const payload = {
             "cmd": "request.get",