Compare commits

..

23 Commits

Author SHA1 Message Date
ngosang
07724e598f Bump version 2.2.0 2022-01-31 00:20:44 +01:00
ngosang
56fc688517 Increase default BROWSER_TIMEOUT=40000 (40 seconds) 2022-01-30 23:24:15 +01:00
ngosang
0a438358d1 Fix Puppeter deprecation warnings 2022-01-30 23:23:06 +01:00
ngosang
0cbca1fb79 Update base Docker image Alpine 3.15 / NodeJS 16 2022-01-30 23:17:14 +01:00
ngosang
05dcae979c Build precompiled binaries with NodeJS 16 2022-01-30 23:09:28 +01:00
ngosang
fe6cfd75b8 Update Puppeter and other dependencies 2022-01-30 22:49:15 +01:00
ngosang
bb7e82e6c4 Add support for Custom CloudFlare challenge
EbookParadijs, Film-Paleis, MuziekFabriek and Puur-Hollands
2022-01-30 21:32:16 +01:00
ngosang
fdd1d245f4 Add support for DDoS-GUARD challenge 2022-01-30 20:36:38 +01:00
ngosang
bc6ac68e52 Bump version 2.1.0 2021-12-12 16:47:33 +01:00
simonfr
a9ab2569bc Add aarch64 to user agents to be replaced (#248)
Co-authored-by: Simon <simon@perols.dev>
2021-12-12 16:46:20 +01:00
ngosang
b1a6ad7688 Fix SOCKSv4 and SOCKSv5 proxy. resolves #214 #220 2021-12-12 14:29:38 +01:00
David Refoua
642d67b927 Remove redundant JSON key (postData) (#242) 2021-12-12 12:38:10 +01:00
ngosang
c4ef6a472e Make test URL configurable with TEST_URL env var. resolves #240 2021-12-12 12:35:05 +01:00
ngosang
a24b665bd1 Bypass new Cloudflare protection 2021-12-12 12:35:05 +01:00
Diego Heras
6576e1908d Update donation links 2021-12-04 23:43:30 +01:00
ngosang
8e518d7267 Bump version 2.0.2 2021-10-31 22:46:12 +01:00
ngosang
3005ba3629 Fix SOCKS5 proxy. Resolves #214 2021-10-31 22:39:32 +01:00
ngosang
176c69d1e8 Replace Firefox ERS with a newer version 2021-10-31 22:22:28 +01:00
ngosang
7a1cf7dd80 Catch startup exceptions and give some advices 2021-10-31 22:12:55 +01:00
ngosang
456dfc222e Add env var BROWSER_TIMEOUT for slow systems 2021-10-31 21:56:25 +01:00
ngosang
23fde49f2b Fix NPM warning in Docker images 2021-10-31 21:38:57 +01:00
ngosang
78daf24bc3 Bump version 2.0.1 2021-10-24 16:38:15 +02:00
ngosang
47c83ded58 Check user home dir before testing web browser installation 2021-10-24 15:52:03 +02:00
11 changed files with 2996 additions and 4069 deletions

View File

@@ -1,13 +1,13 @@
FROM --platform=${TARGETPLATFORM:-linux/amd64} node:16-alpine3.14 FROM --platform=${TARGETPLATFORM:-linux/amd64} node:16-alpine3.15
# Print build information # Print build information
ARG TARGETPLATFORM ARG TARGETPLATFORM
ARG BUILDPLATFORM ARG BUILDPLATFORM
RUN printf "I am running on ${BUILDPLATFORM:-linux/amd64}, building for ${TARGETPLATFORM:-linux/amd64}\n$(uname -a)\n" RUN printf "I am running on ${BUILDPLATFORM:-linux/amd64}, building for ${TARGETPLATFORM:-linux/amd64}\n$(uname -a)\n"
# Install the web browser (package firefox is available too) # Install the web browser (package firefox-esr is available too)
RUN apk update && \ RUN apk update && \
apk add --no-cache firefox-esr dumb-init && \ apk add --no-cache firefox dumb-init && \
rm -Rf /var/cache rm -Rf /var/cache
# Copy FlareSolverr code # Copy FlareSolverr code
@@ -28,7 +28,7 @@ RUN npm install && \
EXPOSE 8191 EXPOSE 8191
ENTRYPOINT ["/usr/bin/dumb-init", "--"] ENTRYPOINT ["/usr/bin/dumb-init", "--"]
CMD ["npm", "start"] CMD ["node", "./dist/server.js"]
# docker build -t flaresolverr:custom . # docker build -t flaresolverr:custom .
# docker run -p 8191:8191 -e LOG_LEVEL=debug flaresolverr:custom # docker run -p 8191:8191 -e LOG_LEVEL=debug flaresolverr:custom

View File

@@ -5,8 +5,8 @@
[![GitHub issues](https://img.shields.io/github/issues/FlareSolverr/FlareSolverr)](https://github.com/FlareSolverr/FlareSolverr/issues) [![GitHub issues](https://img.shields.io/github/issues/FlareSolverr/FlareSolverr)](https://github.com/FlareSolverr/FlareSolverr/issues)
[![GitHub pull requests](https://img.shields.io/github/issues-pr/FlareSolverr/FlareSolverr)](https://github.com/FlareSolverr/FlareSolverr/pulls) [![GitHub pull requests](https://img.shields.io/github/issues-pr/FlareSolverr/FlareSolverr)](https://github.com/FlareSolverr/FlareSolverr/pulls)
[![Donate PayPal](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=X5NJLLX5GLTV6&source=url) [![Donate PayPal](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=X5NJLLX5GLTV6&source=url)
[![Donate Buy Me A Coffee](https://img.shields.io/badge/Donate-Buy%20me%20a%20coffee-yellow.svg)](https://www.buymeacoffee.com/ngosang) [![Donate Bitcoin](https://en.cryptobadges.io/badge/micro/13Hcv77AdnFWEUZ9qUpoPBttQsUT7q9TTh)](https://en.cryptobadges.io/donate/13Hcv77AdnFWEUZ9qUpoPBttQsUT7q9TTh)
[![Donate Bitcoin](https://img.shields.io/badge/Donate-Bitcoin-orange.svg)](https://en.cryptobadges.io/donate/13Hcv77AdnFWEUZ9qUpoPBttQsUT7q9TTh) [![Donate Ethereum](https://en.cryptobadges.io/badge/micro/0x0D1549BbB00926BF3D92c1A8A58695e982f1BE2E)](https://en.cryptobadges.io/donate/0x0D1549BbB00926BF3D92c1A8A58695e982f1BE2E)
FlareSolverr is a proxy server to bypass Cloudflare protection. FlareSolverr is a proxy server to bypass Cloudflare protection.
@@ -66,13 +66,13 @@ This is the recommended way for Windows users.
### From source code ### From source code
This is the recommended way for macOS users and for developers. This is the recommended way for macOS users and for developers.
* Install [NodeJS](https://nodejs.org/). * Install [NodeJS](https://nodejs.org/) 16.
* Clone this repository and open a shell in that path. * Clone this repository and open a shell in that path.
* Run `export PUPPETEER_PRODUCT=firefox` (Linux/macOS) or `set PUPPETEER_PRODUCT=firefox` (Windows). * Run `export PUPPETEER_PRODUCT=firefox` (Linux/macOS) or `set PUPPETEER_PRODUCT=firefox` (Windows).
* Run `npm install` command to install FlareSolverr dependencies. * Run `npm install` command to install FlareSolverr dependencies.
* Run `node node_modules/puppeteer/install.js` to install Firefox. * Run `npm start` command to compile TypeScript code and start FlareSolverr.
* Run `npm run build` command to compile TypeScript code.
* Run `npm start` command to start FlareSolverr. If you get errors related to firefox not installed try running `node node_modules/puppeteer/install.js` to install Firefox.
### Systemd service ### Systemd service
@@ -141,7 +141,7 @@ session | Optional. Will send the request from and existing browser instance. If
maxTimeout | Optional, default value 60000. Max timeout to solve the challenge in milliseconds. maxTimeout | Optional, default value 60000. Max timeout to solve the challenge in milliseconds.
cookies | Optional. Will be used by the headless browser. Follow [this](https://github.com/puppeteer/puppeteer/blob/v3.3.0/docs/api.md#pagesetcookiecookies) format. cookies | Optional. Will be used by the headless browser. Follow [this](https://github.com/puppeteer/puppeteer/blob/v3.3.0/docs/api.md#pagesetcookiecookies) format.
returnOnlyCookies | Optional, default false. Only returns the cookies. Response data, headers and other parts of the response are removed. returnOnlyCookies | Optional, default false. Only returns the cookies. Response data, headers and other parts of the response are removed.
proxy | Optional, default disabled. Eg: `"proxy": {"url": "http://127.0.0.1:8888"}`. Authorization (username/password) is not supported. proxy | Optional, default disabled. Eg: `"proxy": {"url": "http://127.0.0.1:8888"}`. You must include the proxy schema in the URL: `http://`, `socks4://` or `socks5://`. Authorization (username/password) is not supported.
:warning: If you want to use Cloudflare clearance cookie in your scripts, make sure you use the FlareSolverr User-Agent too. If they don't match you will see the challenge. :warning: If you want to use Cloudflare clearance cookie in your scripts, make sure you use the FlareSolverr User-Agent too. If they don't match you will see the challenge.
@@ -210,7 +210,7 @@ This is the same as `request.get` but it takes one more param:
Parameter | Notes Parameter | Notes
|--|--| |--|--|
postData | Must be a string with `application/x-www-form-urlencoded`. Eg: `postData": "a=b&c=d"` postData | Must be a string with `application/x-www-form-urlencoded`. Eg: `a=b&c=d`
## Environment variables ## Environment variables
@@ -221,6 +221,8 @@ LOG_HTML | false | Only for debugging. If `true` all HTML that passes through th
CAPTCHA_SOLVER | none | Captcha solving method. It is used when a captcha is encountered. See the Captcha Solvers section. CAPTCHA_SOLVER | none | Captcha solving method. It is used when a captcha is encountered. See the Captcha Solvers section.
TZ | UTC | Timezone used in the logs and the web browser. Example: `TZ=Europe/London`. TZ | UTC | Timezone used in the logs and the web browser. Example: `TZ=Europe/London`.
HEADLESS | true | Only for debugging. To run the web browser in headless mode or visible. HEADLESS | true | Only for debugging. To run the web browser in headless mode or visible.
BROWSER_TIMEOUT | 40000 | If you are experiencing errors/timeouts because your system is slow, you can try to increase this value. Remember to increase the `maxTimeout` parameter too.
TEST_URL | https://www.google.com | FlareSolverr makes a request on start to make sure the web browser is working. You can change that URL if it is blocked in your country.
PORT | 8191 | Listening port. You don't need to change this if you are running on Docker. PORT | 8191 | Listening port. You don't need to change this if you are running on Docker.
HOST | 0.0.0.0 | Listening interface. You don't need to change this if you are running on Docker. HOST | 0.0.0.0 | Listening interface. You don't need to change this if you are running on Docker.

View File

@@ -64,8 +64,8 @@ function getFirefoxNightlyVersion() {
if (fs.existsSync('bin')) { if (fs.existsSync('bin')) {
fs.rmSync('bin', { recursive: true }) fs.rmSync('bin', { recursive: true })
} }
execSync('./node_modules/.bin/pkg -t node14-win-x64,node14-linux-x64 --out-path bin .') execSync('./node_modules/.bin/pkg -t node16-win-x64,node16-linux-x64 --out-path bin .')
// execSync('./node_modules/.bin/pkg -t node14-win-x64,node14-mac-x64,node14-linux-x64 --out-path bin .') // execSync('./node_modules/.bin/pkg -t node16-win-x64,node16-mac-x64,node16-linux-x64 --out-path bin .')
// get firefox revision // get firefox revision
const revision = await getFirefoxNightlyVersion(); const revision = await getFirefoxNightlyVersion();

6673
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,12 +1,12 @@
{ {
"name": "flaresolverr", "name": "flaresolverr",
"version": "2.0.0", "version": "2.2.0",
"description": "Proxy server to bypass Cloudflare protection.", "description": "Proxy server to bypass Cloudflare protection.",
"scripts": { "scripts": {
"start": "node ./dist/server.js", "start": "tsc && node ./dist/server.js",
"build": "tsc", "build": "tsc",
"dev": "nodemon -e ts --exec ts-node src/server.ts", "dev": "nodemon -e ts --exec ts-node src/server.ts",
"package": "node build-binaries.js", "package": "tsc && node build-binaries.js",
"test": "jest --runInBand" "test": "jest --runInBand"
}, },
"author": "Diego Heras (ngosang)", "author": "Diego Heras (ngosang)",
@@ -23,7 +23,7 @@
"body-parser": "^1.19.0", "body-parser": "^1.19.0",
"console-log-level": "^1.4.1", "console-log-level": "^1.4.1",
"express": "^4.17.1", "express": "^4.17.1",
"puppeteer": "^3.3.0", "puppeteer": "^13.1.2",
"uuid": "^8.3.2" "uuid": "^8.3.2"
}, },
"devDependencies": { "devDependencies": {
@@ -31,13 +31,13 @@
"@types/body-parser": "^1.19.1", "@types/body-parser": "^1.19.1",
"@types/express": "^4.17.13", "@types/express": "^4.17.13",
"@types/jest": "^27.0.2", "@types/jest": "^27.0.2",
"@types/node": "^14.17.27", "@types/node": "^16.11.7",
"@types/puppeteer": "^3.0.6", "@types/puppeteer": "^5.4.4",
"@types/supertest": "^2.0.11", "@types/supertest": "^2.0.11",
"@types/uuid": "^8.3.1", "@types/uuid": "^8.3.1",
"archiver": "^5.3.0", "archiver": "^5.3.0",
"nodemon": "^2.0.13", "nodemon": "^2.0.13",
"pkg": "^5.3.3", "pkg": "^5.5.2",
"supertest": "^6.1.6", "supertest": "^6.1.6",
"ts-jest": "^27.0.7", "ts-jest": "^27.0.7",
"ts-node": "^10.3.0", "ts-node": "^10.3.0",

View File

@@ -1,6 +1,5 @@
// todo: avoid puppeter objects
import {SetCookie, Headers, HttpMethod} from 'puppeteer'
import {Request, Response} from 'express'; import {Request, Response} from 'express';
import {Protocol} from "devtools-protocol";
import log from '../services/log' import log from '../services/log'
import {browserRequest, ChallengeResolutionResultT, ChallengeResolutionT} from "../services/solver"; import {browserRequest, ChallengeResolutionResultT, ChallengeResolutionT} from "../services/solver";
@@ -20,11 +19,11 @@ export interface Proxy {
export interface V1RequestBase { export interface V1RequestBase {
cmd: string cmd: string
cookies?: SetCookie[], cookies?: Protocol.Network.CookieParam[],
maxTimeout?: number maxTimeout?: number
proxy?: Proxy proxy?: Proxy
session: string session: string
headers?: Headers // deprecated v2, not used headers?: Record<string, string> // deprecated v2, not used
userAgent?: string // deprecated v2, not used userAgent?: string // deprecated v2, not used
} }
@@ -33,7 +32,7 @@ interface V1RequestSession extends V1RequestBase {
export interface V1Request extends V1RequestBase { export interface V1Request extends V1RequestBase {
url: string url: string
method?: HttpMethod method?: string
postData?: string postData?: string
returnOnlyCookies?: boolean returnOnlyCookies?: boolean
download?: boolean // deprecated v2, not used download?: boolean // deprecated v2, not used

View File

@@ -1,4 +1,4 @@
import {Page, Response} from 'puppeteer' import {Page, HTTPResponse} from 'puppeteer'
import log from "../services/log"; import log from "../services/log";
@@ -6,16 +6,30 @@ import log from "../services/log";
* This class contains the logic to solve protections provided by CloudFlare * This class contains the logic to solve protections provided by CloudFlare
**/ **/
const BAN_SELECTORS = ['span[data-translate="error"]']; const BAN_SELECTORS = ['.text-gray-600'];
const CHALLENGE_SELECTORS = ['#trk_jschal_js', '.ray_id', '.attack-box', '#cf-please-wait']; const CHALLENGE_SELECTORS = [
'#trk_jschal_js', '.ray_id', '.attack-box', '#cf-please-wait', // CloudFlare
'#link-ddg', // DDoS-GUARD
'td.info #js_info' // Custom CloudFlare for EbookParadijs, Film-Paleis, MuziekFabriek and Puur-Hollands
];
const CAPTCHA_SELECTORS = ['input[name="cf_captcha_kind"]']; const CAPTCHA_SELECTORS = ['input[name="cf_captcha_kind"]'];
export default async function resolveChallenge(url: string, page: Page, response: Response): Promise<Response> { export default async function resolveChallenge(url: string, page: Page, response: HTTPResponse): Promise<HTTPResponse> {
// look for challenge and return fast if not detected // look for challenge and return fast if not detected
if (response.headers().server && let cfDetected = response.headers().server && response.headers().server.startsWith('cloudflare');
response.headers().server.startsWith('cloudflare') && if (cfDetected) {
(response.status() == 403 || response.status() == 503)) { if (response.status() == 403 || response.status() == 503) {
cfDetected = true; // Defected CloudFlare and DDoS-GUARD
} else if (response.headers().vary && response.headers().vary.trim() == 'Accept-Encoding,User-Agent' &&
response.headers()['content-encoding'] && response.headers()['content-encoding'].trim() == 'br') {
cfDetected = true; // Detected Custom CloudFlare for EbookParadijs, Film-Paleis, MuziekFabriek and Puur-Hollands
} else {
cfDetected = false;
}
}
if (cfDetected) {
log.info('Cloudflare detected'); log.info('Cloudflare detected');
} else { } else {
log.info('Cloudflare not detected'); log.info('Cloudflare not detected');
@@ -26,10 +40,8 @@ export default async function resolveChallenge(url: string, page: Page, response
throw new Error('Cloudflare has blocked this request. Probably your IP is banned for this site, check in your web browser.') throw new Error('Cloudflare has blocked this request. Probably your IP is banned for this site, check in your web browser.')
} }
let selectorFound = false;
if (response.status() > 400) {
// find Cloudflare selectors // find Cloudflare selectors
let selectorFound = false;
let selector: string = await findAnySelector(page, CHALLENGE_SELECTORS) let selector: string = await findAnySelector(page, CHALLENGE_SELECTORS)
if (selector) { if (selector) {
selectorFound = true; selectorFound = true;
@@ -57,7 +69,7 @@ export default async function resolveChallenge(url: string, page: Page, response
// wait until redirecting disappears // wait until redirecting disappears
while (true) { while (true) {
try { try {
await page.waitFor(1000) await page.waitForTimeout(1000)
const displayStyle2 = await page.evaluate(() => { const displayStyle2 = await page.evaluate(() => {
return getComputedStyle(document.querySelector('#cf-spinner-redirecting')).getPropertyValue("display"); return getComputedStyle(document.querySelector('#cf-spinner-redirecting')).getPropertyValue("display");
}); });
@@ -84,7 +96,7 @@ export default async function resolveChallenge(url: string, page: Page, response
} }
log.debug('Waiting for Cloudflare challenge...') log.debug('Waiting for Cloudflare challenge...')
await page.waitFor(1000) await page.waitForTimeout(1000)
} }
log.debug('Validating HTML code...') log.debug('Validating HTML code...')
@@ -92,12 +104,6 @@ export default async function resolveChallenge(url: string, page: Page, response
log.debug(`No challenge element detected.`) log.debug(`No challenge element detected.`)
} }
} else {
// some sites use cloudflare but there is no challenge
log.debug(`Javascript challenge not detected. Status code: ${response.status()}`);
selectorFound = true;
}
// check for CAPTCHA challenge // check for CAPTCHA challenge
if (await findAnySelector(page, CAPTCHA_SELECTORS)) { if (await findAnySelector(page, CAPTCHA_SELECTORS)) {
log.info('CAPTCHA challenge detected'); log.info('CAPTCHA challenge detected');

View File

@@ -46,4 +46,13 @@ testWebBrowserInstallation().then(() => {
app.listen(serverPort, serverHost, () => { app.listen(serverPort, serverHost, () => {
log.info(`Listening on http://${serverHost}:${serverPort}`); log.info(`Listening on http://${serverHost}:${serverPort}`);
}) })
}).catch(function(e) {
log.error(e);
const msg: string = "" + e;
if (msg.includes('while trying to connect to the browser!')) {
log.error(`It seems that the system is too slow to run FlareSolverr.
If you are running with Docker, try to remove CPU limits in the container.
If not, try setting the 'BROWSER_TIMEOUT' environment variable and the 'maxTimeout' parameter to higher values.`);
}
process.exit(1);
}) })

View File

@@ -1,10 +1,13 @@
import {v1 as UUIDv1} from 'uuid' import {v1 as UUIDv1} from 'uuid'
import * as path from 'path' import * as path from 'path'
import {SetCookie, Browser} from 'puppeteer' import {Browser} from 'puppeteer'
import {Protocol} from "devtools-protocol";
import log from './log' import log from './log'
import {Proxy} from "../controllers/v1"; import {Proxy} from "../controllers/v1";
const os = require('os');
const fs = require('fs');
const puppeteer = require('puppeteer'); const puppeteer = require('puppeteer');
export interface SessionsCacheItem { export interface SessionsCacheItem {
@@ -18,7 +21,7 @@ interface SessionsCache {
export interface SessionCreateOptions { export interface SessionCreateOptions {
oneTimeSession: boolean oneTimeSession: boolean
cookies?: SetCookie[], cookies?: Protocol.Network.CookieParam[],
maxTimeout?: number maxTimeout?: number
proxy?: Proxy proxy?: Proxy
} }
@@ -44,27 +47,49 @@ function buildExtraPrefsFirefox(proxy: Proxy): object {
"startup.homepage_welcome_url": "about:blank", "startup.homepage_welcome_url": "about:blank",
"startup.homepage_welcome_url.additional": "", "startup.homepage_welcome_url.additional": "",
// Disable images to speed up load // Detected !
"permissions.default.image": 2 // // Disable images to speed up load
// "permissions.default.image": 2,
// Limit content processes to 1
"dom.ipc.processCount": 1
} }
// proxy.url format => http://<host>:<port> // proxy.url format => http://<host>:<port>
if (proxy && proxy.url) { if (proxy && proxy.url) {
const [host, portStr] = proxy.url.replace(/https?:\/\//g, '').split(':'); const [host, portStr] = proxy.url.replace(/.+:\/\//g, '').split(':');
const port = parseInt(portStr); const port = parseInt(portStr);
const proxyPrefs = { const proxyPrefs = {
// Proxy configuration "network.proxy.type": 1,
"network.proxy.share_proxy_settings": true
}
if (proxy.url.indexOf("socks") != -1) {
// SOCKSv4 & SOCKSv5
Object.assign(proxyPrefs, {
"network.proxy.socks": host,
"network.proxy.socks_port": port,
"network.proxy.socks_remote_dns": true
});
if (proxy.url.indexOf("socks4") != -1) {
Object.assign(proxyPrefs, {
"network.proxy.socks_version": 4
});
} else {
Object.assign(proxyPrefs, {
"network.proxy.socks_version": 5
});
}
} else {
// HTTP
Object.assign(proxyPrefs, {
"network.proxy.ftp": host, "network.proxy.ftp": host,
"network.proxy.ftp_port": port, "network.proxy.ftp_port": port,
"network.proxy.http": host, "network.proxy.http": host,
"network.proxy.http_port": port, "network.proxy.http_port": port,
"network.proxy.share_proxy_settings": true,
"network.proxy.socks": host,
"network.proxy.socks_port": port,
"network.proxy.ssl": host, "network.proxy.ssl": host,
"network.proxy.ssl_port": port, "network.proxy.ssl_port": port
"network.proxy.type": 1 });
} }
// merge objects // merge objects
@@ -80,16 +105,25 @@ export function getUserAgent() {
export async function testWebBrowserInstallation(): Promise<void> { export async function testWebBrowserInstallation(): Promise<void> {
log.info("Testing web browser installation...") log.info("Testing web browser installation...")
// check user home dir. this dir will be used by Firefox
const homeDir = os.homedir();
fs.accessSync(homeDir, fs.constants.F_OK | fs.constants.R_OK | fs.constants.W_OK | fs.constants.X_OK);
log.debug("FlareSolverr user home directory is OK: " + homeDir)
// test web browser
const testUrl = process.env.TEST_URL || "https://www.google.com";
log.debug("Test URL: " + testUrl)
const session = await create(null, { const session = await create(null, {
oneTimeSession: true oneTimeSession: true
}) })
const page = await session.browser.newPage() const page = await session.browser.newPage()
await page.goto("https://www.google.com") await page.goto(testUrl)
webBrowserUserAgent = await page.evaluate(() => navigator.userAgent) webBrowserUserAgent = await page.evaluate(() => navigator.userAgent)
// replace Linux ARM user-agent because it's detected // replace Linux ARM user-agent because it's detected
if (webBrowserUserAgent.toLocaleLowerCase().includes('linux arm')) { if (["arm", "aarch64"].some(arch => webBrowserUserAgent.toLocaleLowerCase().includes('linux ' + arch))) {
webBrowserUserAgent = webBrowserUserAgent.replace(/linux arm[^;]+;/i, 'Linux x86_64;') webBrowserUserAgent = webBrowserUserAgent.replace(/linux \w+;/i, 'Linux x86_64;')
} }
log.info("FlareSolverr User-Agent: " + webBrowserUserAgent) log.info("FlareSolverr User-Agent: " + webBrowserUserAgent)
@@ -107,6 +141,7 @@ export async function create(session: string, options: SessionCreateOptions): Pr
const puppeteerOptions: any = { const puppeteerOptions: any = {
product: 'firefox', product: 'firefox',
headless: process.env.HEADLESS !== 'false', headless: process.env.HEADLESS !== 'false',
timeout: process.env.BROWSER_TIMEOUT || 40000
} }
puppeteerOptions.extraPrefsFirefox = buildExtraPrefsFirefox(options.proxy) puppeteerOptions.extraPrefsFirefox = buildExtraPrefsFirefox(options.proxy)

View File

@@ -1,4 +1,4 @@
import {Response, Headers, Page} from 'puppeteer' import {Page, HTTPResponse} from 'puppeteer'
const Timeout = require('await-timeout'); const Timeout = require('await-timeout');
import log from './log' import log from './log'
@@ -11,7 +11,7 @@ const sessions = require('./sessions')
export interface ChallengeResolutionResultT { export interface ChallengeResolutionResultT {
url: string url: string
status: number, status: number,
headers?: Headers, headers?: Record<string, string>,
response: string, response: string,
cookies: object[] cookies: object[]
userAgent: string userAgent: string
@@ -64,7 +64,7 @@ async function resolveChallenge(params: V1Request, session: SessionsCacheItem):
// go to the page // go to the page
log.debug(`Navigating to... ${params.url}`) log.debug(`Navigating to... ${params.url}`)
let response: Response = await gotoPage(params, page); let response: HTTPResponse = await gotoPage(params, page);
// set cookies // set cookies
if (params.cookies) { if (params.cookies) {
@@ -128,8 +128,8 @@ async function resolveChallenge(params: V1Request, session: SessionsCacheItem):
} }
} }
async function gotoPage(params: V1Request, page: Page): Promise<Response> { async function gotoPage(params: V1Request, page: Page): Promise<HTTPResponse> {
let response: Response; let response: HTTPResponse;
if (params.method != 'POST') { if (params.method != 'POST') {
response = await page.goto(params.url, {waitUntil: 'domcontentloaded'}); response = await page.goto(params.url, {waitUntil: 'domcontentloaded'});
@@ -177,7 +177,7 @@ async function gotoPage(params: V1Request, page: Page): Promise<Response> {
</html> </html>
` `
); );
await page.waitFor(2000) await page.waitForTimeout(2000)
try { try {
await page.waitForNavigation({waitUntil: 'domcontentloaded', timeout: 2000}) await page.waitForNavigation({waitUntil: 'domcontentloaded', timeout: 2000})
} catch (e) {} } catch (e) {}

View File

@@ -9,11 +9,14 @@ const sessions = require('../services/sessions');
const version: string = 'v' + require('../../package.json').version const version: string = 'v' + require('../../package.json').version
const proxyUrl = "http://127.0.0.1:8888" const proxyUrl = "http://127.0.0.1:8888"
const proxySocksUrl = "socks5://127.0.0.1:1080"
const googleUrl = "https://www.google.com"; const googleUrl = "https://www.google.com";
const postUrl = "https://ptsv2.com/t/qv4j3-1634496523"; const postUrl = "https://ptsv2.com/t/qv4j3-1634496523";
const cfUrl = "https://pirateiro.com/torrents/?search=harry"; const cfUrl = "https://pirateiro.com/torrents/?search=harry";
const cfCaptchaUrl = "https://idope.se" const cfCaptchaUrl = "https://idope.se"
const cfBlockedUrl = "https://www.torrentmafya.org/table.php" const cfBlockedUrl = "https://www.torrentmafya.org/table.php"
const ddgUrl = "https://www.erai-raws.info/feed/?type=magnet";
const ccfUrl = "https://www.muziekfabriek.org";
beforeAll(async () => { beforeAll(async () => {
// Init session // Init session
@@ -167,6 +170,64 @@ describe("Test '/v1' path", () => {
expect(apiResponse.solution.url).toContain(cfBlockedUrl) expect(apiResponse.solution.url).toContain(cfBlockedUrl)
}); });
test("Cmd 'request.get' should return OK with DDoS-GUARD JS", async () => {
const payload = {
"cmd": "request.get",
"url": ddgUrl
}
const response: Response = await request(app).post("/v1").send(payload);
expect(response.statusCode).toBe(200);
const apiResponse: V1ResponseSolution = response.body;
expect(apiResponse.status).toBe("ok");
expect(apiResponse.message).toBe("");
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
expect(apiResponse.version).toBe(version);
const solution = apiResponse.solution;
expect(solution.url).toContain(ddgUrl)
expect(solution.status).toBe(200);
expect(Object.keys(solution.headers).length).toBeGreaterThan(0)
expect(solution.response).toContain("<rss version")
expect(Object.keys(solution.cookies).length).toBeGreaterThan(0)
expect(solution.userAgent).toContain("Firefox/")
const cfCookie: string = (solution.cookies as any[]).filter(function(cookie) {
return cookie.name == "__ddg1";
})[0].value
expect(cfCookie.length).toBeGreaterThan(10)
});
test("Cmd 'request.get' should return OK with Custom CloudFlare JS", async () => {
const payload = {
"cmd": "request.get",
"url": ccfUrl
}
const response: Response = await request(app).post("/v1").send(payload);
expect(response.statusCode).toBe(200);
const apiResponse: V1ResponseSolution = response.body;
expect(apiResponse.status).toBe("ok");
expect(apiResponse.message).toBe("");
expect(apiResponse.startTimestamp).toBeGreaterThan(1000);
expect(apiResponse.endTimestamp).toBeGreaterThan(apiResponse.startTimestamp);
expect(apiResponse.version).toBe(version);
const solution = apiResponse.solution;
expect(solution.url).toContain(ccfUrl)
expect(solution.status).toBe(200);
expect(Object.keys(solution.headers).length).toBeGreaterThan(0)
expect(solution.response).toContain("<html><head>")
expect(Object.keys(solution.cookies).length).toBeGreaterThan(0)
expect(solution.userAgent).toContain("Firefox/")
const cfCookie: string = (solution.cookies as any[]).filter(function(cookie) {
return cookie.name == "ct_anti_ddos_key";
})[0].value
expect(cfCookie.length).toBeGreaterThan(10)
});
test("Cmd 'request.get' should return OK with 'cookies' param", async () => { test("Cmd 'request.get' should return OK with 'cookies' param", async () => {
const payload = { const payload = {
"cmd": "request.get", "cmd": "request.get",
@@ -221,7 +282,7 @@ describe("Test '/v1' path", () => {
expect(solution.userAgent).toBe(null) expect(solution.userAgent).toBe(null)
}); });
test("Cmd 'request.get' should return OK with 'proxy' param", async () => { test("Cmd 'request.get' should return OK with HTTP 'proxy' param", async () => {
/* /*
To configure TinyProxy in local: To configure TinyProxy in local:
* sudo vim /etc/tinyproxy/tinyproxy.conf * sudo vim /etc/tinyproxy/tinyproxy.conf
@@ -249,7 +310,7 @@ describe("Test '/v1' path", () => {
}); });
// todo: credentials are not working // todo: credentials are not working
test.skip("Cmd 'request.get' should return OK with 'proxy' param with credentials", async () => { test.skip("Cmd 'request.get' should return OK with HTTP 'proxy' param with credentials", async () => {
/* /*
To configure TinyProxy in local: To configure TinyProxy in local:
* sudo vim /etc/tinyproxy/tinyproxy.conf * sudo vim /etc/tinyproxy/tinyproxy.conf
@@ -279,6 +340,32 @@ describe("Test '/v1' path", () => {
expect(solution.status).toContain(200) expect(solution.status).toContain(200)
}); });
test("Cmd 'request.get' should return OK with SOCKSv5 'proxy' param", async () => {
/*
To configure Dante in local:
* https://linuxhint.com/set-up-a-socks5-proxy-on-ubuntu-with-dante/
* sudo vim /etc/sockd.conf
* sudo systemctl restart sockd.service
* curl --socks5 socks5://127.0.0.1:1080 https://www.google.com
*/
const payload = {
"cmd": "request.get",
"url": googleUrl,
"proxy": {
"url": proxySocksUrl
}
}
const response: Response = await request(app).post("/v1").send(payload);
expect(response.statusCode).toBe(200);
const apiResponse: V1ResponseSolution = response.body;
expect(apiResponse.status).toBe("ok");
const solution = apiResponse.solution;
expect(solution.url).toContain(googleUrl)
expect(solution.status).toBe(200);
});
test("Cmd 'request.get' should fail with wrong 'proxy' param", async () => { test("Cmd 'request.get' should fail with wrong 'proxy' param", async () => {
const payload = { const payload = {
"cmd": "request.get", "cmd": "request.get",