[scraper] removes moch resolver from scraper

2024-12-20 03:29:51 +00:00 · 2020-04-09 13:35:11 +02:00
parent fed5098b88
commit cf7b4411d2
5 changed files with 123 additions and 120 deletions
--- a/package-lock.json
+++ b/package-lock.json
@@ -998,6 +998,11 @@
        "readable-stream": "^3.1.1"
      }
    },
+    "http": {
+      "version": "0.0.0",
+      "resolved": "https://registry.npmjs.org/http/-/http-0.0.0.tgz",
+      "integrity": "sha1-huYybSnF0Dnen6xYSkVon5KfT3I="
+    },
    "http-errors": {
      "version": "1.6.3",
      "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz",
@@ -1934,6 +1939,11 @@
      "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz",
      "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA=="
    },
+    "querystring": {
+      "version": "0.2.0",
+      "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz",
+      "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA="
+    },
    "random-access-file": {
      "version": "2.1.4",
      "resolved": "https://registry.npmjs.org/random-access-file/-/random-access-file-2.1.4.tgz",
@@ -2117,6 +2127,113 @@
      "resolved": "https://registry.npmjs.org/rusha/-/rusha-0.8.13.tgz",
      "integrity": "sha1-mghOe4YLF7/zAVuSxnpqM2GRUTo="
    },
+    "rutracker-api-2": {
+      "version": "1.10.0",
+      "resolved": "https://registry.npmjs.org/rutracker-api-2/-/rutracker-api-2-1.10.0.tgz",
+      "integrity": "sha512-wBsMQNpjNFV92lIUHwbQXd0Gn+5bnIH6JHfn7dN/nhcCHiprUH2bVntzZBHULWGYf5MokOaqQwgxBueg4QtADw==",
+      "requires": {
+        "bluebird": "^3.5.1",
+        "cheerio": "^0.19.0",
+        "http": "0.0.0",
+        "querystring": "^0.2.0",
+        "windows-1251": "^0.1.2"
+      },
+      "dependencies": {
+        "cheerio": {
+          "version": "0.19.0",
+          "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-0.19.0.tgz",
+          "integrity": "sha1-dy5wFfLuKZZQltcepBdbdas1SSU=",
+          "requires": {
+            "css-select": "~1.0.0",
+            "dom-serializer": "~0.1.0",
+            "entities": "~1.1.1",
+            "htmlparser2": "~3.8.1",
+            "lodash": "^3.2.0"
+          }
+        },
+        "css-select": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.0.0.tgz",
+          "integrity": "sha1-sRIcpRhI3SZOIkTQWM7iVN7rRLA=",
+          "requires": {
+            "boolbase": "~1.0.0",
+            "css-what": "1.0",
+            "domutils": "1.4",
+            "nth-check": "~1.0.0"
+          }
+        },
+        "css-what": {
+          "version": "1.0.0",
+          "resolved": "https://registry.npmjs.org/css-what/-/css-what-1.0.0.tgz",
+          "integrity": "sha1-18wt9FGAZm+Z0rFEYmOUaeAPc2w="
+        },
+        "domhandler": {
+          "version": "2.3.0",
+          "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.3.0.tgz",
+          "integrity": "sha1-LeWaCCLVAn+r/28DLCsloqir5zg=",
+          "requires": {
+            "domelementtype": "1"
+          }
+        },
+        "domutils": {
+          "version": "1.4.3",
+          "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.4.3.tgz",
+          "integrity": "sha1-CGVRN5bGswYDGFDhdVFrr4C3Km8=",
+          "requires": {
+            "domelementtype": "1"
+          }
+        },
+        "htmlparser2": {
+          "version": "3.8.3",
+          "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.8.3.tgz",
+          "integrity": "sha1-mWwosZFRaovoZQGn15dX5ccMEGg=",
+          "requires": {
+            "domelementtype": "1",
+            "domhandler": "2.3",
+            "domutils": "1.5",
+            "entities": "1.0",
+            "readable-stream": "1.1"
+          },
+          "dependencies": {
+            "domutils": {
+              "version": "1.5.1",
+              "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz",
+              "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=",
+              "requires": {
+                "dom-serializer": "0",
+                "domelementtype": "1"
+              }
+            },
+            "entities": {
+              "version": "1.0.0",
+              "resolved": "https://registry.npmjs.org/entities/-/entities-1.0.0.tgz",
+              "integrity": "sha1-sph6o4ITR/zeZCsk/fyeT7cSvyY="
+            }
+          }
+        },
+        "lodash": {
+          "version": "3.10.1",
+          "resolved": "https://registry.npmjs.org/lodash/-/lodash-3.10.1.tgz",
+          "integrity": "sha1-W/Rejkm6QYnhfUgnid/RW9FAt7Y="
+        },
+        "readable-stream": {
+          "version": "1.1.14",
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz",
+          "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=",
+          "requires": {
+            "core-util-is": "~1.0.0",
+            "inherits": "~2.0.1",
+            "isarray": "0.0.1",
+            "string_decoder": "~0.10.x"
+          }
+        },
+        "string_decoder": {
+          "version": "0.10.31",
+          "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz",
+          "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ="
+        }
+      }
+    },
    "safe-buffer": {
      "version": "5.1.2",
      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
@@ -2741,6 +2858,11 @@
        "defaults": "^1.0.3"
      }
    },
+    "windows-1251": {
+      "version": "0.1.2",
+      "resolved": "https://registry.npmjs.org/windows-1251/-/windows-1251-0.1.2.tgz",
+      "integrity": "sha1-SIHY+QU9Dre1+l008PlNYN5DwtM="
+    },
    "wkx": {
      "version": "0.4.8",
      "resolved": "https://registry.npmjs.org/wkx/-/wkx-0.4.8.tgz",
--- a/package.json
+++ b/package.json
@@ -26,7 +26,6 @@
    "magnet-uri": "^5.1.7",
    "moment": "^2.24.0",
    "name-to-imdb": "^2.3.0",
-    "named-queue": "^2.2.1",
    "needle": "^2.2.4",
    "node-gzip": "^1.1.2",
    "node-schedule": "^1.3.2",
@@ -38,6 +37,7 @@
    "pg-hstore": "^2.3.2",
    "rarbg-api": "^1.1.3",
    "real-debrid-api": "git://github.com/TheBeastLT/node-real-debrid.git#935a5c23ae809edbcd2a111526a7f74d6767c50d",
+    "rutracker-api-2": "^1.10.0",
    "sequelize": "^5.21.5",
    "sugar-date": "^2.0.6",
    "torrent-stream": "^1.2.0",
--- a/scraper/index.js
+++ b/scraper/index.js
@@ -3,7 +3,6 @@ const express = require("express");
 const server = express();
 const schedule = require('node-schedule');
 const { connect, getUpdateSeedersTorrents } = require('./lib/repository');
-const realDebrid = require('./moch/realdebrid');
 const thepiratebayScraper = require('./scrapers/thepiratebay/thepiratebay_scraper');
 const horribleSubsScraper = require('./scrapers/horriblesubs/horriblesubs_scraper');
 const leetxScraper = require('./scrapers/1337x/1337x_scraper');
@@ -62,20 +61,6 @@ server.get('/', function (req, res) {
  res.sendStatus(200);
 });

-server.get('/realdebrid/:apiKey/:infoHash/:cachedFileIds/:fileIndex?', (req, res) => {
-  const { apiKey, infoHash, cachedFileIds, fileIndex } = req.params;
-  realDebrid.resolve(req.ip, apiKey, infoHash, cachedFileIds, isNaN(fileIndex) ? undefined : parseInt(fileIndex))
-      .then(url => {
-        res.writeHead(301, { Location: url });
-        res.end();
-      })
-      .catch(error => {
-        console.log(error);
-        res.statusCode = 404;
-        res.end();
-      });
-});
-
 server.listen(process.env.PORT || 7000, async () => {
  await connect();
  console.log('Scraper started');
--- a/scraper/lib/cache.js
+++ b/scraper/lib/cache.js
@@ -5,16 +5,10 @@ const GLOBAL_KEY_PREFIX = 'stremio-torrentio';
 const IMDB_ID_PREFIX = `${GLOBAL_KEY_PREFIX}|imdb_id`;
 const KITSU_ID_PREFIX = `${GLOBAL_KEY_PREFIX}|kitsu_id`;
 const METADATA_PREFIX = `${GLOBAL_KEY_PREFIX}|metadata`;
-const RESOLVED_URL_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|moch`;
-const PROXY_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|proxy`;
-const USER_AGENT_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|agent`;
 const TRACKERS_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|trackers`;

 const GLOBAL_TTL = process.env.METADATA_TTL || 7 * 24 * 60 * 60; // 7 days
 const MEMORY_TTL = process.env.METADATA_TTL || 2 * 60 * 60; // 2 hours
-const RESOLVED_URL_TTL = 2 * 60; // 2 minutes
-const PROXY_TTL = 60 * 60; // 60 minutes
-const USER_AGENT_TTL = 2 * 24 * 60 * 60; // 2 days
 const TRACKERS_TTL = 2 * 24 * 60 * 60; // 2 days

 const MONGO_URI = process.env.MONGODB_URI;
@@ -65,18 +59,6 @@ function cacheWrapMetadata(id, method) {
  return cacheWrap(memoryCache, `${METADATA_PREFIX}:${id}`, method, { ttl: MEMORY_TTL });
 }

-function cacheWrapResolvedUrl(id, method) {
-  return cacheWrap(memoryCache, `${RESOLVED_URL_KEY_PREFIX}:${id}`, method, { ttl: { RESOLVED_URL_TTL } });
-}
-
-function cacheWrapProxy(id, method) {
-  return cacheWrap(memoryCache, `${PROXY_KEY_PREFIX}:${id}`, method, { ttl: { PROXY_TTL } });
-}
-
-function cacheUserAgent(id, method) {
-  return cacheWrap(memoryCache, `${USER_AGENT_KEY_PREFIX}:${id}`, method, { ttl: { USER_AGENT_TTL } });
-}
-
 function cacheTrackers(method) {
  return cacheWrap(memoryCache, `${TRACKERS_KEY_PREFIX}`, method, { ttl: { TRACKERS_TTL } });
 }
@@ -85,9 +67,6 @@ module.exports = {
  cacheWrapImdbId,
  cacheWrapKitsuId,
  cacheWrapMetadata,
-  cacheWrapResolvedUrl,
-  cacheWrapProxy,
-  cacheUserAgent,
  cacheTrackers
 };

--- a/scraper/moch/realdebrid.js
+++ b/scraper/moch/realdebrid.js
@@ -1,83 +0,0 @@
-const { encode } = require('magnet-uri');
-const RealDebridClient = require('real-debrid-api');
-const namedQueue = require('named-queue');
-const { cacheWrapResolvedUrl, cacheWrapProxy, cacheUserAgent } = require('../lib/cache');
-const { getRandomProxy, getRandomUserAgent } = require('../lib/request_helper');
-
-const unrestrictQueue = new namedQueue((task, callback) => task.method()
-    .then(result => callback(false, result))
-    .catch((error => callback(error))));
-
-async function resolve(ip, apiKey, infoHash, cachedFileIds, fileIndex) {
-  if (!apiKey || !infoHash || !cachedFileIds || !cachedFileIds.length) {
-    return Promise.reject("No valid parameters passed");
-  }
-  const id = `${apiKey}_${infoHash}_${fileIndex}`;
-  const method = () => cacheWrapResolvedUrl(id, () => _unrestrict(ip, apiKey, infoHash, cachedFileIds, fileIndex));
-
-  return new Promise(((resolve, reject) => {
-    unrestrictQueue.push({ id, method }, (error, result) => result ? resolve(result) : reject(error));
-  }));
-}
-
-async function _unrestrict(ip, apiKey, infoHash, cachedFileIds, fileIndex) {
-  console.log(`Unrestricting ${infoHash} [${fileIndex}]`);
-  const options = await getDefaultOptions(ip);
-  const RD = new RealDebridClient(apiKey, options);
-  const torrentId = await _createOrFindTorrentId(RD, infoHash, cachedFileIds);
-  if (torrentId) {
-    const info = await RD.torrents.info(torrentId);
-    const targetFile = info.files.find(file => file.id === fileIndex + 1)
-        || info.files.filter(file => file.selected).sort((a, b) => b.bytes - a.bytes)[0];
-    const selectedFiles = info.files.filter(file => file.selected);
-    const fileLink = info.links.length === 1
-        ? info.links[0]
-        : info.links[selectedFiles.indexOf(targetFile)];
-    const unrestrictedLink = await _unrestrictLink(RD, fileLink);
-    console.log(`Unrestricted ${infoHash} [${fileIndex}] to ${unrestrictedLink}`);
-    return unrestrictedLink;
-  }
-  return Promise.reject("Failed adding torrent");
-}
-
-async function _createOrFindTorrentId(RD, infoHash, cachedFileIds) {
-  return RD.torrents.get(0, 1)
-      .then(torrents => torrents.find(torrent => torrent.hash.toLowerCase() === infoHash))
-      .then(torrent => torrent && torrent.id || Promise.reject('No recent torrent found'))
-      .catch((error) => RD.torrents.addMagnet(encode({ infoHash }))
-          .then(response => RD.torrents.selectFiles(response.id, cachedFileIds)
-              .then((() => response.id))))
-      .catch(error => {
-        console.warn('Failed RealDebrid torrent retrieval', error);
-        return undefined;
-      });
-}
-
-async function _unrestrictLink(RD, link) {
-  if (!link || !link.length) {
-    return Promise.reject("No available links found");
-  }
-  return RD.unrestrict.link(link)
-      .then(unrestrictedLink => unrestrictedLink.download);
-  // .then(unrestrictedLink => RD.streaming.transcode(unrestrictedLink.id))
-  // .then(transcodedLink => {
-  //   const url = transcodedLink.apple && transcodedLink.apple.full
-  //       || transcodedLink[Object.keys(transcodedLink)[0]].full;
-  //   console.log(`Unrestricted ${link} to ${url}`);
-  //   return url;
-  // });
-}
-
-async function getDefaultOptions(ip) {
-  const userAgent = await cacheUserAgent(ip, () => getRandomUserAgent()).catch(() => getRandomUserAgent());
-  const proxy = await cacheWrapProxy('realdebrid', () => getRandomProxy()).catch(() => getRandomProxy());
-
-  return {
-    proxy: proxy,
-    headers: {
-      'User-Agent': userAgent
-    }
-  };
-}
-
-module.exports = { resolve };