From 15fb7d24f286be59260d8808bfde057cbee241ff Mon Sep 17 00:00:00 2001 From: Pim de Bree Date: Fri, 1 May 2020 21:55:28 +0200 Subject: [PATCH 1/2] getting cookies in json format from response with proxy & header support --- examples/example_cookies.js | 99 +++++++++++++++++++++ examples/example_cookies.ts | 58 +++++++++++++ package-lock.json | 169 +++++++++++++++++++++++++++++++++++- package.json | 6 +- 4 files changed, 327 insertions(+), 5 deletions(-) create mode 100644 examples/example_cookies.js create mode 100644 examples/example_cookies.ts diff --git a/examples/example_cookies.js b/examples/example_cookies.js new file mode 100644 index 0000000..63a384a --- /dev/null +++ b/examples/example_cookies.js @@ -0,0 +1,99 @@ +"use strict"; +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (_) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +exports.__esModule = true; +var got_1 = require("got"); +var tough_cookie_1 = require("tough-cookie"); +var tunnel = require('tunnel'); +var parse = require('parse-headers'); +var cookie = require('cookie'); +var index_1 = require("../src/index"); +// example helper function +function main() { + return __awaiter(this, void 0, void 0, function () { + var cookieJar, options, res, error_1; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: + cookieJar = new tough_cookie_1.CookieJar(); + options = { + // use "url: to describe path + url: 'http://soap2day.is', + cookieJar: cookieJar, + // either disable retry or remove status code 503 from retries + retry: 0, + agent: { + https: tunnel.httpOverHttp({ + proxy: { + host: 'ip', + port: 1256, + // Basic authorization for proxy server if necessary + proxyAuth: '', + // Header fields for proxy server if necessary + headers: { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36' + } + } + }) + } + }; + _a.label = 1; + case 1: + _a.trys.push([1, 3, , 5]); + return [4 /*yield*/, got_1["default"].get(options)]; + case 2: + // success without cloudflare? + // const res = await cloudscraper({ + // method: 'GET', + // url: 'https://rlsbb.ru/support-us', + // }); + res = _a.sent(); + return [2 /*return*/, res]; + case 3: + error_1 = _a.sent(); + return [4 /*yield*/, index_1.catchCloudflare(error_1, options)]; + case 4: + // success with cloudflare? + res = _a.sent(); + return [3 /*break*/, 5]; + case 5: return [2 /*return*/, cookie.parse(parse(res.req._header).cookie)]; + } + }); + }); +} +main() + .then(function (res) { return console.log(res); })["catch"](function (err) { return console.error('fail', err); }); diff --git a/examples/example_cookies.ts b/examples/example_cookies.ts new file mode 100644 index 0000000..8f92d4b --- /dev/null +++ b/examples/example_cookies.ts @@ -0,0 +1,58 @@ +import got from 'got'; +import { CookieJar } from 'tough-cookie'; +const tunnel = require('tunnel'); +const parse = require('parse-headers') +const cookie = require('cookie') + +import { catchCloudflare } from '../src/index'; + +// example helper function +async function main() { + // cookie jar is required! + const cookieJar = new CookieJar(); + const options = { + // use "url: to describe path + url: 'http://soap2day.is', + cookieJar, + // either disable retry or remove status code 503 from retries + retry: 0, + + agent: { + https: tunnel.httpOverHttp({ + proxy: { + host: 'ip', + port: 1256, + + // Basic authorization for proxy server if necessary + proxyAuth: '', + + // Header fields for proxy server if necessary + headers: { + 'User-Agent': + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36', + }, + }, + }), + }, + }; + + let res: any; + try { + // success without cloudflare? + // const res = await cloudscraper({ + // method: 'GET', + // url: 'https://rlsbb.ru/support-us', + // }); + res = await got.get(options); + return res; + } catch (error) { + // success with cloudflare? + res = await catchCloudflare(error, options); + } + + return cookie.parse(parse(res.req._header).cookie); +} + +main() + .then((res) => console.log(res)) + .catch((err) => console.error('fail', err)); diff --git a/package-lock.json b/package-lock.json index 6cd75f8..4bb53bd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2836,10 +2836,9 @@ } }, "cookie": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.0.tgz", - "integrity": "sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg==", - "dev": true + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.1.tgz", + "integrity": "sha512-ZwrFkGJxUR3EIoXtO+yVE69Eb7KlixbaeAWfBQB9vVsNn/o+Yw69gBWSSDK825hQNdN+wF8zELf3dFNl/kxkUA==" }, "cookie-signature": { "version": "1.0.6", @@ -3205,6 +3204,12 @@ "is-obj": "^1.0.0" } }, + "duplexer": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.1.tgz", + "integrity": "sha1-rOb/gIwc5mtX0ev5eXessCM0z8E=", + "dev": true + }, "duplexer2": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz", @@ -3728,6 +3733,32 @@ "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=", "dev": true }, + "event-stream": { + "version": "3.3.4", + "resolved": "https://registry.npmjs.org/event-stream/-/event-stream-3.3.4.tgz", + "integrity": "sha1-SrTJoPWlTbkzi0w02Gv86PSzVXE=", + "dev": true, + "requires": { + "duplexer": "~0.1.1", + "from": "~0", + "map-stream": "~0.1.0", + "pause-stream": "0.0.11", + "split": "0.3", + "stream-combiner": "~0.0.4", + "through": "~2.3.1" + }, + "dependencies": { + "split": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/split/-/split-0.3.3.tgz", + "integrity": "sha1-zQ7qXmOiEd//frDwkcQTPi0N0o8=", + "dev": true, + "requires": { + "through": "2" + } + } + } + }, "exec-sh": { "version": "0.3.4", "resolved": "https://registry.npmjs.org/exec-sh/-/exec-sh-0.3.4.tgz", @@ -3882,6 +3913,12 @@ "vary": "~1.1.2" }, "dependencies": { + "cookie": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.0.tgz", + "integrity": "sha512-+Hp8fLp57wnUSt0tY0tHEXh4voZRDnoIrZPqlo3DPiI4y9lwg/jqx+1Om94/W6ZaPDOUbnjOt/99w66zk+l1Xg==", + "dev": true + }, "debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -4208,6 +4245,12 @@ "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=", "dev": true }, + "from": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/from/-/from-0.1.7.tgz", + "integrity": "sha1-g8YK/Fi5xWmXAH7Rp2izqzA6RP4=", + "dev": true + }, "from2": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/from2/-/from2-2.3.0.tgz", @@ -6673,6 +6716,12 @@ "integrity": "sha1-plzSkIepJZi4eRJXpSPgISIqwfk=", "dev": true }, + "map-stream": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/map-stream/-/map-stream-0.1.0.tgz", + "integrity": "sha1-5WqpTEyAVaFkBKBnS3jyFffI4ZQ=", + "dev": true + }, "map-visit": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/map-visit/-/map-visit-1.0.0.tgz", @@ -7067,6 +7116,12 @@ "propagate": "^2.0.0" } }, + "node-cleanup": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/node-cleanup/-/node-cleanup-2.1.2.tgz", + "integrity": "sha1-esGavSl+Caf3KnFUXZUbUX5N3iw=", + "dev": true + }, "node-emoji": { "version": "1.10.0", "resolved": "https://registry.npmjs.org/node-emoji/-/node-emoji-1.10.0.tgz", @@ -10941,6 +10996,11 @@ "callsites": "^3.0.0" } }, + "parse-headers": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/parse-headers/-/parse-headers-2.0.3.tgz", + "integrity": "sha512-QhhZ+DCCit2Coi2vmAKbq5RGTRcQUOE2+REgv8vdyu7MnYx2eZztegqtTx99TZ86GTIwqiy3+4nQTWZ2tgmdCA==" + }, "parse-json": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz", @@ -11010,6 +11070,15 @@ "pify": "^2.0.0" } }, + "pause-stream": { + "version": "0.0.11", + "resolved": "https://registry.npmjs.org/pause-stream/-/pause-stream-0.0.11.tgz", + "integrity": "sha1-/lo0sMvOErWqaitAPuLnO2AvFEU=", + "dev": true, + "requires": { + "through": "~2.3" + } + }, "performance-now": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", @@ -11154,6 +11223,21 @@ "ipaddr.js": "1.9.1" } }, + "ps-tree": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/ps-tree/-/ps-tree-1.2.0.tgz", + "integrity": "sha512-0VnamPPYHl4uaU/nSFeZZpR21QAWRz+sRv4iW9+v/GS/J5U5iZB5BNN6J0RMoOvdx2gWM2+ZFMIm58q24e4UYA==", + "dev": true, + "requires": { + "event-stream": "=3.3.4" + } + }, + "pseudomap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", + "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=", + "dev": true + }, "psl": { "version": "1.8.0", "resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz", @@ -12582,6 +12666,15 @@ "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=", "dev": true }, + "stream-combiner": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/stream-combiner/-/stream-combiner-0.0.4.tgz", + "integrity": "sha1-TV5DPBhSYd3mI8o/RMWGvPXErRQ=", + "dev": true, + "requires": { + "duplexer": "~0.1.1" + } + }, "stream-combiner2": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/stream-combiner2/-/stream-combiner2-1.1.1.tgz", @@ -12592,6 +12685,12 @@ "readable-stream": "^2.0.2" } }, + "string-argv": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/string-argv/-/string-argv-0.1.2.tgz", + "integrity": "sha512-mBqPGEOMNJKXRo7z0keX0wlAhbBAjilUdPW13nN0PecVryZxdHIeM7TqbsSUA7VYuS00HGC6mojP7DlQzfa9ZA==", + "dev": true + }, "string-length": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/string-length/-/string-length-3.1.0.tgz", @@ -13003,6 +13102,63 @@ } } }, + "tsc-watch": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/tsc-watch/-/tsc-watch-4.2.3.tgz", + "integrity": "sha512-M1Lo37+ggVfavGX3ObUVMz9QBH7moqq2RlmBdxnz6a6etwecetznZ/ZgYOi2c9HQ4Ki2qStj7V9J/gSf0rThig==", + "dev": true, + "requires": { + "cross-spawn": "^5.1.0", + "node-cleanup": "^2.1.2", + "ps-tree": "^1.2.0", + "string-argv": "^0.1.1", + "strip-ansi": "^4.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=", + "dev": true + }, + "cross-spawn": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-5.1.0.tgz", + "integrity": "sha1-6L0O/uWPz/b4+UUQoKVUu/ojVEk=", + "dev": true, + "requires": { + "lru-cache": "^4.0.1", + "shebang-command": "^1.2.0", + "which": "^1.2.9" + } + }, + "lru-cache": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz", + "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", + "dev": true, + "requires": { + "pseudomap": "^1.0.2", + "yallist": "^2.1.2" + } + }, + "strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "dev": true, + "requires": { + "ansi-regex": "^3.0.0" + } + }, + "yallist": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", + "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=", + "dev": true + } + } + }, "tslib": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.11.1.tgz", @@ -13018,6 +13174,11 @@ "tslib": "^1.8.1" } }, + "tunnel": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", + "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==" + }, "tunnel-agent": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", diff --git a/package.json b/package.json index 2f11bbe..add2661 100644 --- a/package.json +++ b/package.json @@ -34,8 +34,11 @@ "semantic-release": "semantic-release" }, "dependencies": { + "cookie": "^0.4.1", "delay": "^4.3.0", - "got": "^11.0.3" + "got": "^11.0.3", + "parse-headers": "^2.0.3", + "tunnel": "0.0.6" }, "devDependencies": { "@pollyjs/adapter-fetch": "4.2.1", @@ -62,6 +65,7 @@ "semantic-release": "17.0.7", "tough-cookie": "4.0.0", "ts-jest": "25.4.0", + "tsc-watch": "^4.2.3", "typescript": "3.8.3" }, "jest": { From d2f7386090dc37049aadd5e7e17e568c70ddb630 Mon Sep 17 00:00:00 2001 From: Pim de Bree Date: Fri, 1 May 2020 22:01:59 +0200 Subject: [PATCH 2/2] removed .js files & added as - gave error in typescript compiler --- examples/example_cookies.js | 99 ------------------------------------- src/catchCloudflare.ts | 6 +-- 2 files changed, 3 insertions(+), 102 deletions(-) delete mode 100644 examples/example_cookies.js diff --git a/examples/example_cookies.js b/examples/example_cookies.js deleted file mode 100644 index 63a384a..0000000 --- a/examples/example_cookies.js +++ /dev/null @@ -1,99 +0,0 @@ -"use strict"; -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; -var __generator = (this && this.__generator) || function (thisArg, body) { - var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; - return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; - function verb(n) { return function (v) { return step([n, v]); }; } - function step(op) { - if (f) throw new TypeError("Generator is already executing."); - while (_) try { - if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; - if (y = 0, t) op = [op[0] & 2, t.value]; - switch (op[0]) { - case 0: case 1: t = op; break; - case 4: _.label++; return { value: op[1], done: false }; - case 5: _.label++; y = op[1]; op = [0]; continue; - case 7: op = _.ops.pop(); _.trys.pop(); continue; - default: - if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } - if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } - if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } - if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } - if (t[2]) _.ops.pop(); - _.trys.pop(); continue; - } - op = body.call(thisArg, _); - } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } - if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; - } -}; -exports.__esModule = true; -var got_1 = require("got"); -var tough_cookie_1 = require("tough-cookie"); -var tunnel = require('tunnel'); -var parse = require('parse-headers'); -var cookie = require('cookie'); -var index_1 = require("../src/index"); -// example helper function -function main() { - return __awaiter(this, void 0, void 0, function () { - var cookieJar, options, res, error_1; - return __generator(this, function (_a) { - switch (_a.label) { - case 0: - cookieJar = new tough_cookie_1.CookieJar(); - options = { - // use "url: to describe path - url: 'http://soap2day.is', - cookieJar: cookieJar, - // either disable retry or remove status code 503 from retries - retry: 0, - agent: { - https: tunnel.httpOverHttp({ - proxy: { - host: 'ip', - port: 1256, - // Basic authorization for proxy server if necessary - proxyAuth: '', - // Header fields for proxy server if necessary - headers: { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 Safari/537.36' - } - } - }) - } - }; - _a.label = 1; - case 1: - _a.trys.push([1, 3, , 5]); - return [4 /*yield*/, got_1["default"].get(options)]; - case 2: - // success without cloudflare? - // const res = await cloudscraper({ - // method: 'GET', - // url: 'https://rlsbb.ru/support-us', - // }); - res = _a.sent(); - return [2 /*return*/, res]; - case 3: - error_1 = _a.sent(); - return [4 /*yield*/, index_1.catchCloudflare(error_1, options)]; - case 4: - // success with cloudflare? - res = _a.sent(); - return [3 /*break*/, 5]; - case 5: return [2 /*return*/, cookie.parse(parse(res.req._header).cookie)]; - } - }); - }); -} -main() - .then(function (res) { return console.log(res); })["catch"](function (err) { return console.error('fail', err); }); diff --git a/src/catchCloudflare.ts b/src/catchCloudflare.ts index 2bd561a..06a14c3 100644 --- a/src/catchCloudflare.ts +++ b/src/catchCloudflare.ts @@ -1,8 +1,8 @@ -import crypto from 'crypto'; +import * as crypto from 'crypto'; import delay from 'delay'; import got, { Response, RequiredRetryOptions } from 'got'; -import https from 'https'; -import vm from 'vm'; +import * as https from 'https'; +import * as vm from 'vm'; import { URLSearchParams } from 'url'; const BUG_REPORT = `