tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Remove FlareSolverr Variables after scraping

Browse Source
This commit is contained in:
Cedric
2024-01-31 14:59:41 -06:00
parent aa101758f0
commit 3f6c95b12f
2 changed files with 13 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
scraper/scrapers/1337x/1337x_api.js
View File

@@ -13,8 +13,8 @@ const defaultProxies = [
const defaultTimeout = 50000; const defaultTimeout = 50000;
const maxSearchPage = 50; const maxSearchPage = 50;
let flaresolverrUserAgent = ''; let FlaresolverrUserAgent = '';
let flaresolverrCookies = ''; let FlaresolverrCookies = '';
const Categories = { const Categories = {
MOVIE: 'Movies', MOVIE: 'Movies',
@@ -89,7 +89,7 @@ function singleRequest(requestUrl, config = {}) {
const timeout = config.timeout || defaultTimeout; const timeout = config.timeout || defaultTimeout;
let options = { headers: { 'User-Agent': getRandomUserAgent() }, timeout: timeout }; let options = { headers: { 'User-Agent': getRandomUserAgent() }, timeout: timeout };
if (flaresolverrUserAgent === '' || flaresolverrCookies === '') { if (FlaresolverrUserAgent === '' || FlaresolverrCookies === '') {
console.log("using flaresolverr"); console.log("using flaresolverr");
return axios.post('http://flaresolverr:8191/v1', { return axios.post('http://flaresolverr:8191/v1', {
cmd: 'request.get', cmd: 'request.get',
@@ -108,9 +108,9 @@ function singleRequest(requestUrl, config = {}) {
!(body.includes('1337x</title>'))) { !(body.includes('1337x</title>'))) {
throw new Error(`Invalid body contents: ${requestUrl}`); throw new Error(`Invalid body contents: ${requestUrl}`);
} }
flaresolverrUserAgent = response.data.solution.userAgent; FlaresolverrUserAgent = response.data.solution.userAgent;
response.data.solution.cookies.forEach(cookie => { response.data.solution.cookies.forEach(cookie => {
flaresolverrCookies = flaresolverrCookies + `${cookie.name}=${cookie.value}; `; FlaresolverrCookies = FlaresolverrCookies + `${cookie.name}=${cookie.value}; `;
}); });
return body; return body;
@@ -118,8 +118,8 @@ function singleRequest(requestUrl, config = {}) {
} }
else { else {
console.log("using direct request"); console.log("using direct request");
options.headers['User-Agent'] = flaresolverrUserAgent; options.headers['User-Agent'] = FlaresolverrUserAgent;
options.headers['Cookie'] = flaresolverrCookies; options.headers['Cookie'] = FlaresolverrCookies;
return axios.get(requestUrl, options) return axios.get(requestUrl, options)
.then((response) => { .then((response) => {
const body = response.data; const body = response.data;
@@ -203,4 +203,4 @@ function parseDate(dateString) {
return Sugar.Date.create(dateString); return Sugar.Date.create(dateString);
} }
module.exports = { torrent, search, browse, Categories }; module.exports = { torrent, search, browse, Categories, FlaresolverrCookies, FlaresolverrUserAgent };

6
scraper/scrapers/1337x/1337x_scraper.js
View File

@@ -22,7 +22,11 @@ async function scrape() {
lastScrape.lastScraped = scrapeStart; lastScrape.lastScraped = scrapeStart;
return lastScrape.save(); return lastScrape.save();
}) })
.then(() => console.log(`[${moment()}] finished ${NAME} scrape`)); .then(() => console.log(`[${moment()}] finished ${NAME} scrape`))
.then(() => {
leetx.FlaresolverrCookies = '';
leetx.FlaresolverrUserAgent = '';
});
} }
async function updateSeeders(torrent) { async function updateSeeders(torrent) {