tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Change Flaresolverr mechanic to use cookies after initial flaresolverr request

Browse Source
This commit is contained in:
Cedric
2024-01-31 14:19:46 -06:00
parent 07d4d020e6
commit ee34b6db17
1 changed files with 67 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
scraper/scrapers/1337x/1337x_api.js
View File

@@ -10,9 +10,12 @@ const { parseSize } = require("../scraperHelper");
const defaultProxies = [ const defaultProxies = [
'https://1337x.to' 'https://1337x.to'
]; ];
const defaultTimeout = 10000; const defaultTimeout = 50000;
const maxSearchPage = 50; const maxSearchPage = 50;
let flaresolverrUserAgent = '';
let flaresolverrCookies = '';
const Categories = { const Categories = {
MOVIE: 'Movies', MOVIE: 'Movies',
TV: 'TV', TV: 'TV',
@@ -76,13 +79,18 @@ function browse(config = {}, retries = 2) {
return Promises.first(proxyList return Promises.first(proxyList
.map((proxyUrl) => singleRequest(requestUrl(proxyUrl), config))) .map((proxyUrl) => singleRequest(requestUrl(proxyUrl), config)))
.then((body) => parseTableBody(body)) .then((body) => parseTableBody(body))
.catch((err) => browse(config, retries - 1)); .catch((err) => {
console.error(err);
browse(config, retries - 1);
});
} }
function singleRequest(requestUrl, config = {}) { function singleRequest(requestUrl, config = {}) {
const timeout = config.timeout || defaultTimeout; const timeout = config.timeout || defaultTimeout;
const options = { headers: { 'User-Agent': getRandomUserAgent() }, timeout: timeout }; let options = { headers: { 'User-Agent': getRandomUserAgent() }, timeout: timeout };
if (flaresolverrUserAgent === '' || flaresolverrCookies === '') {
console.log("using flaresolverr");
return axios.post('http://flaresolverr:8191/v1', { return axios.post('http://flaresolverr:8191/v1', {
cmd: 'request.get', cmd: 'request.get',
url: requestUrl, url: requestUrl,
@@ -100,9 +108,32 @@ function singleRequest(requestUrl, config = {}) {
!(body.includes('1337x</title>'))) { !(body.includes('1337x</title>'))) {
throw new Error(`Invalid body contents: ${requestUrl}`); throw new Error(`Invalid body contents: ${requestUrl}`);
} }
flaresolverrUserAgent = response.data.solution.userAgent;
response.data.solution.cookies.forEach(cookie => {
flaresolverrCookies = flaresolverrCookies + `${cookie.name}=${cookie.value}; `;
});
return body; return body;
}); });
} }
else {
console.log("using direct request");
options.headers['User-Agent'] = flaresolverrUserAgent;
options.headers['Cookie'] = flaresolverrCookies;
return axios.get(requestUrl, options)
.then((response) => {
const body = response.data;
if (!body) {
throw new Error(`No body: ${requestUrl}`);
} else if (body.includes('502: Bad gateway') ||
body.includes('403 Forbidden') ||
!(body.includes('1337x</title>'))) {
throw new Error(`Invalid body contents: ${requestUrl}`);
}
return body;
})
}
}
function parseTableBody(body) { function parseTableBody(body) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {