Merge pull request #19 from KillTrot/master

Change 1337x FlareSolverr mechanic to only use FlareSolverr on the first request

scraper/scrapers/1337x/1337x_api.js

@@ -10,9 +10,12 @@ const { parseSize } = require("../scraperHelper");
 const defaultProxies = [
   'https://1337x.to'
 ];
-const defaultTimeout = 10000;
+const defaultTimeout = 50000;
 const maxSearchPage = 50;
 
+let FlaresolverrUserAgent = '';
+let FlaresolverrCookies = '';
+
 const Categories = {
   MOVIE: 'Movies',
   TV: 'TV',
@@ -76,13 +79,18 @@ function browse(config = {}, retries = 2) {
   return Promises.first(proxyList
     .map((proxyUrl) => singleRequest(requestUrl(proxyUrl), config)))
     .then((body) => parseTableBody(body))
-      .catch((err) => browse(config, retries - 1));
+    .catch((err) => {
+      console.error(err);
+      browse(config, retries - 1);
+    });
 }
 
 function singleRequest(requestUrl, config = {}) {
   const timeout = config.timeout || defaultTimeout;
-  const options = { headers: { 'User-Agent': getRandomUserAgent() }, timeout: timeout };
+  let options = { headers: { 'User-Agent': getRandomUserAgent() }, timeout: timeout };
 
+  if (FlaresolverrUserAgent === '' || FlaresolverrCookies === '') {
+    console.log("using flaresolverr");
     return axios.post('http://flaresolverr:8191/v1', {
       cmd: 'request.get',
       url: requestUrl,
@@ -100,9 +108,32 @@ function singleRequest(requestUrl, config = {}) {
           !(body.includes('1337x</title>'))) {
           throw new Error(`Invalid body contents: ${requestUrl}`);
         }
+        FlaresolverrUserAgent = response.data.solution.userAgent;
+        response.data.solution.cookies.forEach(cookie => {
+          FlaresolverrCookies = FlaresolverrCookies + `${cookie.name}=${cookie.value}; `;
+        });
+
         return body;
       });
   }
+  else {
+    console.log("using direct request");
+    options.headers['User-Agent'] = FlaresolverrUserAgent;
+    options.headers['Cookie'] = FlaresolverrCookies;
+    return axios.get(requestUrl, options)
+      .then((response) => {
+        const body = response.data;
+        if (!body) {
+          throw new Error(`No body: ${requestUrl}`);
+        } else if (body.includes('502: Bad gateway') ||
+          body.includes('403 Forbidden') ||
+          !(body.includes('1337x</title>'))) {
+          throw new Error(`Invalid body contents: ${requestUrl}`);
+        }
+        return body;
+      })
+  }
+}
 
 function parseTableBody(body) {
   return new Promise((resolve, reject) => {
@@ -172,4 +203,4 @@ function parseDate(dateString) {
   return Sugar.Date.create(dateString);
 }
 
-module.exports = { torrent, search, browse, Categories };
+module.exports = { torrent, search, browse, Categories, FlaresolverrCookies, FlaresolverrUserAgent };

scraper/scrapers/1337x/1337x_scraper.js

@@ -22,7 +22,11 @@ async function scrape() {
         lastScrape.lastScraped = scrapeStart;
         return lastScrape.save();
       })
-      .then(() => console.log(`[${moment()}] finished ${NAME} scrape`));
+      .then(() => console.log(`[${moment()}] finished ${NAME} scrape`))
+      .then(() => {
+        leetx.FlaresolverrCookies = '';
+        leetx.FlaresolverrUserAgent = '';
+      });
 }
 
 async function updateSeeders(torrent) {