tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #19 from KillTrot/master

Browse Source 
Change 1337x FlareSolverr mechanic to only use FlareSolverr on the first request
This commit is contained in:
Gabisonfire
2024-01-31 16:12:44 -05:00
committed by GitHub
parent 3556821253 a0017f3491
commit d63f1d9716
2 changed files with 73 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
scraper/scrapers/1337x/1337x_api.js
View File

@@ -10,9 +10,12 @@ const { parseSize } = require("../scraperHelper");
const defaultProxies = [ const defaultProxies = [
'https://1337x.to' 'https://1337x.to'
]; ];
const defaultTimeout = 10000; const defaultTimeout = 50000;
const maxSearchPage = 50; const maxSearchPage = 50;
let FlaresolverrUserAgent = '';
let FlaresolverrCookies = '';
const Categories = { const Categories = {
MOVIE: 'Movies', MOVIE: 'Movies',
TV: 'TV', TV: 'TV',
@@ -76,19 +79,24 @@ function browse(config = {}, retries = 2) {
return Promises.first(proxyList return Promises.first(proxyList
.map((proxyUrl) => singleRequest(requestUrl(proxyUrl), config))) .map((proxyUrl) => singleRequest(requestUrl(proxyUrl), config)))
.then((body) => parseTableBody(body)) .then((body) => parseTableBody(body))
.catch((err) => browse(config, retries - 1)); .catch((err) => {
console.error(err);
browse(config, retries - 1);
});
} }
function singleRequest(requestUrl, config = {}) { function singleRequest(requestUrl, config = {}) {
const timeout = config.timeout || defaultTimeout; const timeout = config.timeout || defaultTimeout;
const options = { headers: { 'User-Agent': getRandomUserAgent() }, timeout: timeout }; let options = { headers: { 'User-Agent': getRandomUserAgent() }, timeout: timeout };
if (FlaresolverrUserAgent === '' || FlaresolverrCookies === '') {
console.log("using flaresolverr");
return axios.post('http://flaresolverr:8191/v1', { return axios.post('http://flaresolverr:8191/v1', {
cmd: 'request.get', cmd: 'request.get',
url: requestUrl, url: requestUrl,
}, options) }, options)
.then((response) => { .then((response) => {
if (response.data.status !== 'ok'){ if (response.data.status !== 'ok') {
throw new Error(`FlareSolverr did not return status 'ok': ${response.data.message}`) throw new Error(`FlareSolverr did not return status 'ok': ${response.data.message}`)
} }
@@ -100,8 +108,31 @@ function singleRequest(requestUrl, config = {}) {
!(body.includes('1337x</title>'))) { !(body.includes('1337x</title>'))) {
throw new Error(`Invalid body contents: ${requestUrl}`); throw new Error(`Invalid body contents: ${requestUrl}`);
} }
FlaresolverrUserAgent = response.data.solution.userAgent;
response.data.solution.cookies.forEach(cookie => {
FlaresolverrCookies = FlaresolverrCookies + `${cookie.name}=${cookie.value}; `;
});
return body; return body;
}); });
}
else {
console.log("using direct request");
options.headers['User-Agent'] = FlaresolverrUserAgent;
options.headers['Cookie'] = FlaresolverrCookies;
return axios.get(requestUrl, options)
.then((response) => {
const body = response.data;
if (!body) {
throw new Error(`No body: ${requestUrl}`);
} else if (body.includes('502: Bad gateway') ||
body.includes('403 Forbidden') ||
!(body.includes('1337x</title>'))) {
throw new Error(`Invalid body contents: ${requestUrl}`);
}
return body;
})
}
} }
function parseTableBody(body) { function parseTableBody(body) {
@@ -172,4 +203,4 @@ function parseDate(dateString) {
return Sugar.Date.create(dateString); return Sugar.Date.create(dateString);
} }
module.exports = { torrent, search, browse, Categories }; module.exports = { torrent, search, browse, Categories, FlaresolverrCookies, FlaresolverrUserAgent };

6
scraper/scrapers/1337x/1337x_scraper.js
View File

@@ -22,7 +22,11 @@ async function scrape() {
lastScrape.lastScraped = scrapeStart; lastScrape.lastScraped = scrapeStart;
return lastScrape.save(); return lastScrape.save();
}) })
.then(() => console.log(`[${moment()}] finished ${NAME} scrape`)); .then(() => console.log(`[${moment()}] finished ${NAME} scrape`))
.then(() => {
leetx.FlaresolverrCookies = '';
leetx.FlaresolverrUserAgent = '';
});
} }
async function updateSeeders(torrent) { async function updateSeeders(torrent) {