From 3f6c95b12f2e7105be5b5087a2c59b1973103588 Mon Sep 17 00:00:00 2001 From: Cedric Date: Wed, 31 Jan 2024 14:59:41 -0600 Subject: [PATCH] Remove FlareSolverr Variables after scraping --- scraper/scrapers/1337x/1337x_api.js | 16 ++++++++-------- scraper/scrapers/1337x/1337x_scraper.js | 6 +++++- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/scraper/scrapers/1337x/1337x_api.js b/scraper/scrapers/1337x/1337x_api.js index 78b7175..0a979cc 100644 --- a/scraper/scrapers/1337x/1337x_api.js +++ b/scraper/scrapers/1337x/1337x_api.js @@ -13,8 +13,8 @@ const defaultProxies = [ const defaultTimeout = 50000; const maxSearchPage = 50; -let flaresolverrUserAgent = ''; -let flaresolverrCookies = ''; +let FlaresolverrUserAgent = ''; +let FlaresolverrCookies = ''; const Categories = { MOVIE: 'Movies', @@ -89,7 +89,7 @@ function singleRequest(requestUrl, config = {}) { const timeout = config.timeout || defaultTimeout; let options = { headers: { 'User-Agent': getRandomUserAgent() }, timeout: timeout }; - if (flaresolverrUserAgent === '' || flaresolverrCookies === '') { + if (FlaresolverrUserAgent === '' || FlaresolverrCookies === '') { console.log("using flaresolverr"); return axios.post('http://flaresolverr:8191/v1', { cmd: 'request.get', @@ -108,9 +108,9 @@ function singleRequest(requestUrl, config = {}) { !(body.includes('1337x'))) { throw new Error(`Invalid body contents: ${requestUrl}`); } - flaresolverrUserAgent = response.data.solution.userAgent; + FlaresolverrUserAgent = response.data.solution.userAgent; response.data.solution.cookies.forEach(cookie => { - flaresolverrCookies = flaresolverrCookies + `${cookie.name}=${cookie.value}; `; + FlaresolverrCookies = FlaresolverrCookies + `${cookie.name}=${cookie.value}; `; }); return body; @@ -118,8 +118,8 @@ function singleRequest(requestUrl, config = {}) { } else { console.log("using direct request"); - options.headers['User-Agent'] = flaresolverrUserAgent; - options.headers['Cookie'] = flaresolverrCookies; + options.headers['User-Agent'] = FlaresolverrUserAgent; + options.headers['Cookie'] = FlaresolverrCookies; return axios.get(requestUrl, options) .then((response) => { const body = response.data; @@ -203,4 +203,4 @@ function parseDate(dateString) { return Sugar.Date.create(dateString); } -module.exports = { torrent, search, browse, Categories }; +module.exports = { torrent, search, browse, Categories, FlaresolverrCookies, FlaresolverrUserAgent }; diff --git a/scraper/scrapers/1337x/1337x_scraper.js b/scraper/scrapers/1337x/1337x_scraper.js index 188c8ec..7e1a1de 100644 --- a/scraper/scrapers/1337x/1337x_scraper.js +++ b/scraper/scrapers/1337x/1337x_scraper.js @@ -22,7 +22,11 @@ async function scrape() { lastScrape.lastScraped = scrapeStart; return lastScrape.save(); }) - .then(() => console.log(`[${moment()}] finished ${NAME} scrape`)); + .then(() => console.log(`[${moment()}] finished ${NAME} scrape`)) + .then(() => { + leetx.FlaresolverrCookies = ''; + leetx.FlaresolverrUserAgent = ''; + }); } async function updateSeeders(torrent) {