Merge pull request #19 from KillTrot/master

Change 1337x FlareSolverr mechanic to only use FlareSolverr on the first request

scraper/scrapers/1337x/1337x_api.js

@@ -10,9 +10,12 @@ const { parseSize } = require("../scraperHelper");
 const defaultProxies = [
   'https://1337x.to'
 ];
-const defaultTimeout = 10000;
+const defaultTimeout = 50000;
 const maxSearchPage = 50;
 
+let FlaresolverrUserAgent = '';
+let FlaresolverrCookies = '';
+
 const Categories = {
   MOVIE: 'Movies',
   TV: 'TV',
@@ -33,10 +36,10 @@ function torrent(torrentId, config = {}, retries = 2) {
   const slug = torrentId.startsWith('/torrent/') ? torrentId.replace('/torrent/', '') : torrentId;
 
   return Promises.first(proxyList
-          .map((proxyUrl) => singleRequest(`${proxyUrl}/torrent/${slug}`, config)))
-      .then((body) => parseTorrentPage(body))
-      .then((torrent) => ({ torrentId: slug, ...torrent }))
-      .catch((err) => torrent(slug, config, retries - 1));
+    .map((proxyUrl) => singleRequest(`${proxyUrl}/torrent/${slug}`, config)))
+    .then((body) => parseTorrentPage(body))
+    .then((torrent) => ({ torrentId: slug, ...torrent }))
+    .catch((err) => torrent(slug, config, retries - 1));
 }
 
 function search(keyword, config = {}, retries = 2) {
@@ -48,17 +51,17 @@ function search(keyword, config = {}, retries = 2) {
   const category = config.category;
   const extendToPage = Math.min(maxSearchPage, (config.extendToPage || 1))
   const requestUrl = proxyUrl => category
-      ? `${proxyUrl}/category-search/${keyword}/${category}/${page}/`
-      : `${proxyUrl}/search/${keyword}/${page}/`;
+    ? `${proxyUrl}/category-search/${keyword}/${category}/${page}/`
+    : `${proxyUrl}/search/${keyword}/${page}/`;
 
   return Promises.first(proxyList
-          .map(proxyUrl => singleRequest(requestUrl(proxyUrl), config)))
-      .then(body => parseTableBody(body))
-      .then(torrents => torrents.length === 40 && page < extendToPage
-          ? search(keyword, { ...config, page: page + 1 }).catch(() => [])
-              .then(nextTorrents => torrents.concat(nextTorrents))
-          : torrents)
-      .catch((err) => search(keyword, config, retries - 1));
+    .map(proxyUrl => singleRequest(requestUrl(proxyUrl), config)))
+    .then(body => parseTableBody(body))
+    .then(torrents => torrents.length === 40 && page < extendToPage
+      ? search(keyword, { ...config, page: page + 1 }).catch(() => [])
+        .then(nextTorrents => torrents.concat(nextTorrents))
+      : torrents)
+    .catch((err) => search(keyword, config, retries - 1));
 }
 
 function browse(config = {}, retries = 2) {
@@ -70,38 +73,66 @@ function browse(config = {}, retries = 2) {
   const category = config.category;
   const sort = config.sort;
   const requestUrl = proxyUrl => sort
-      ? `${proxyUrl}/sort-cat/${category}/${sort}/desc/${page}/`
-      : `${proxyUrl}/cat/${category}/${page}/`;
+    ? `${proxyUrl}/sort-cat/${category}/${sort}/desc/${page}/`
+    : `${proxyUrl}/cat/${category}/${page}/`;
 
   return Promises.first(proxyList
-          .map((proxyUrl) => singleRequest(requestUrl(proxyUrl), config)))
-      .then((body) => parseTableBody(body))
-      .catch((err) => browse(config, retries - 1));
+    .map((proxyUrl) => singleRequest(requestUrl(proxyUrl), config)))
+    .then((body) => parseTableBody(body))
+    .catch((err) => {
+      console.error(err);
+      browse(config, retries - 1);
+    });
 }
 
 function singleRequest(requestUrl, config = {}) {
   const timeout = config.timeout || defaultTimeout;
-  const options = { headers: { 'User-Agent': getRandomUserAgent() }, timeout: timeout };
+  let options = { headers: { 'User-Agent': getRandomUserAgent() }, timeout: timeout };
 
-  return axios.post('http://flaresolverr:8191/v1', {
-    cmd: 'request.get',
-    url: requestUrl,
-  }, options)
+  if (FlaresolverrUserAgent === '' || FlaresolverrCookies === '') {
+    console.log("using flaresolverr");
+    return axios.post('http://flaresolverr:8191/v1', {
+      cmd: 'request.get',
+      url: requestUrl,
+    }, options)
       .then((response) => {
-        if (response.data.status !== 'ok'){
+        if (response.data.status !== 'ok') {
           throw new Error(`FlareSolverr did not return status 'ok': ${response.data.message}`)
         }
-        
+
         const body = response.data.solution.response;
         if (!body) {
           throw new Error(`No body: ${requestUrl}`);
         } else if (body.includes('502: Bad gateway') ||
-            body.includes('403 Forbidden') ||
-            !(body.includes('1337x</title>'))) {
+          body.includes('403 Forbidden') ||
+          !(body.includes('1337x</title>'))) {
+          throw new Error(`Invalid body contents: ${requestUrl}`);
+        }
+        FlaresolverrUserAgent = response.data.solution.userAgent;
+        response.data.solution.cookies.forEach(cookie => {
+          FlaresolverrCookies = FlaresolverrCookies + `${cookie.name}=${cookie.value}; `;
+        });
+
+        return body;
+      });
+  }
+  else {
+    console.log("using direct request");
+    options.headers['User-Agent'] = FlaresolverrUserAgent;
+    options.headers['Cookie'] = FlaresolverrCookies;
+    return axios.get(requestUrl, options)
+      .then((response) => {
+        const body = response.data;
+        if (!body) {
+          throw new Error(`No body: ${requestUrl}`);
+        } else if (body.includes('502: Bad gateway') ||
+          body.includes('403 Forbidden') ||
+          !(body.includes('1337x</title>'))) {
           throw new Error(`Invalid body contents: ${requestUrl}`);
         }
         return body;
-      });
+      })
+  }
 }
 
 function parseTableBody(body) {
@@ -153,13 +184,13 @@ function parseTorrentPage(body) {
       uploadDate: parseDate(details.find('strong:contains(\'Date uploaded\')').next().text()),
       imdbId: imdbIdMatch && imdbIdMatch[1],
       files: details.find('div[id=\'files\']').first().find('li')
-          .map((i, elem) => $(elem).text())
-          .map((i, text) => ({
-            fileIndex: i,
-            name: text.match(/^(.+)\s\(.+\)$/)[1].replace(/^.+\//g, ''),
-            path: text.match(/^(.+)\s\(.+\)$/)[1],
-            size: parseSize(text.match(/^.+\s\((.+)\)$/)[1])
-          })).get()
+        .map((i, elem) => $(elem).text())
+        .map((i, text) => ({
+          fileIndex: i,
+          name: text.match(/^(.+)\s\(.+\)$/)[1].replace(/^.+\//g, ''),
+          path: text.match(/^(.+)\s\(.+\)$/)[1],
+          size: parseSize(text.match(/^.+\s\((.+)\)$/)[1])
+        })).get()
     };
     resolve(torrent);
   });
@@ -172,4 +203,4 @@ function parseDate(dateString) {
   return Sugar.Date.create(dateString);
 }
 
-module.exports = { torrent, search, browse, Categories };
+module.exports = { torrent, search, browse, Categories, FlaresolverrCookies, FlaresolverrUserAgent };

scraper/scrapers/1337x/1337x_scraper.js

@@ -22,7 +22,11 @@ async function scrape() {
         lastScrape.lastScraped = scrapeStart;
         return lastScrape.save();
       })
-      .then(() => console.log(`[${moment()}] finished ${NAME} scrape`));
+      .then(() => console.log(`[${moment()}] finished ${NAME} scrape`))
+      .then(() => {
+        leetx.FlaresolverrCookies = '';
+        leetx.FlaresolverrUserAgent = '';
+      });
 }
 
 async function updateSeeders(torrent) {