[addon] moves rd resolver to addon

2020-04-08 21:27:39 +02:00
parent 9ea008572d
commit fed5098b88
7 changed files with 211 additions and 10 deletions
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -40,5 +40,5 @@ jobs:
          docker load -i /tmp/docker/torrentio_addon_latest.tar
          docker stop torrentio-addon
          docker rm torrentio-addon
-          docker run -p 80:7000 -d --name torrentio-addon --restart always -e MONGODB_URI=${{ secrets.MONGODB_URI }} -e DATABASE_URI=${{ secrets.DATABASE_URI }} -e RESOLVER_HOST=${{ secrets.RESOLVER_HOST }} torrentio-addon:latest
+          docker run -p 80:7000 -d --name torrentio-addon --restart always -e MONGODB_URI=${{ secrets.MONGODB_URI }} -e DATABASE_URI=${{ secrets.DATABASE_URI }} -e RESOLVER_HOST=${{ secrets.RESOLVER_HOST }} -e PROXY_HOSTS=${{ secrets.PROXY_HOSTS }} -e PROXY_USERNAME=${{ secrets.PROXY_USERNAME }} -e PROXY_PASSWORD=${{ secrets.PROXY_PASSWORD }} torrentio-addon:latest
          docker image prune -f
--- a/addon/lib/cache.js
+++ b/addon/lib/cache.js
@@ -3,17 +3,24 @@ const mangodbStore = require('cache-manager-mongodb');

 const GLOBAL_KEY_PREFIX = 'torrentio-addon';
 const STREAM_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|stream`;
+const RESOLVED_URL_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|resolved`;
+const PROXY_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|proxy`;
+const USER_AGENT_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|agent`;

 const STREAM_TTL = process.env.STREAM_TTL || 4 * 60 * 60; // 4 hours
 const STREAM_EMPTY_TTL = process.env.STREAM_EMPTY_TTL || 30 * 60; // 30 minutes
+const RESOLVED_URL_TTL = 2 * 60; // 2 minutes
+const PROXY_TTL = 60 * 60; // 60 minutes
+const USER_AGENT_TTL = 2 * 24 * 60 * 60; // 2 days
 // When the streams are empty we want to cache it for less time in case of timeouts or failures

 const MONGO_URI = process.env.MONGODB_URI;
 const NO_CACHE = process.env.NO_CACHE || false;

-const remoteCache = initiateCache();
+const memoryCache = initiateMemoryCache();
+const remoteCache = initiateRemoteCache();

-function initiateCache() {
+function initiateRemoteCache() {
  if (NO_CACHE) {
    return null;
  } else if (MONGO_URI) {
@@ -37,6 +44,13 @@ function initiateCache() {
  }
 }

+function initiateMemoryCache() {
+  return cacheManager.caching({
+    store: 'memory',
+    ttl: STREAM_TTL
+  });
+}
+
 function cacheWrap(cache, key, method, options) {
  if (NO_CACHE || !cache) {
    return method();
@@ -50,5 +64,17 @@ function cacheWrapStream(id, method) {
  });
 }

-module.exports = { cacheWrapStream };
+function cacheWrapResolvedUrl(id, method) {
+  return cacheWrap(memoryCache, `${RESOLVED_URL_KEY_PREFIX}:${id}`, method, { ttl: { RESOLVED_URL_TTL } });
+}
+
+function cacheWrapProxy(id, method) {
+  return cacheWrap(memoryCache, `${PROXY_KEY_PREFIX}:${id}`, method, { ttl: { PROXY_TTL } });
+}
+
+function cacheUserAgent(id, method) {
+  return cacheWrap(memoryCache, `${USER_AGENT_KEY_PREFIX}:${id}`, method, { ttl: { USER_AGENT_TTL } });
+}
+
+module.exports = { cacheWrapStream, cacheWrapResolvedUrl, cacheWrapProxy, cacheUserAgent };

--- a/addon/lib/request_helper.js
+++ b/addon/lib/request_helper.js
@@ -0,0 +1,20 @@
+const UserAgent = require('user-agents');
+
+const PROXY_HOSTS = process.env.PROXY_HOSTS && process.env.PROXY_HOSTS.split(',');
+const PROXY_USERNAME = process.env.PROXY_USERNAME;
+const PROXY_PASSWORD = process.env.PROXY_PASSWORD;
+const userAgent = new UserAgent();
+
+function getRandomUserAgent() {
+  return userAgent.random().toString();
+}
+
+function getRandomProxy() {
+  if (PROXY_HOSTS && PROXY_HOSTS.length && PROXY_USERNAME && PROXY_PASSWORD) {
+    return `http://${PROXY_USERNAME}:${PROXY_PASSWORD}@${PROXY_HOSTS[Math.floor(Math.random() * PROXY_HOSTS.length)]}`;
+  }
+  console.warn('No proxy configured!');
+  return undefined;
+}
+
+module.exports = { getRandomUserAgent, getRandomProxy };
--- a/addon/moch/realdebrid.js
+++ b/addon/moch/realdebrid.js
@@ -1,10 +1,19 @@
 const RealDebridClient = require('real-debrid-api');
+const namedQueue = require('named-queue');
+const { encode } = require('magnet-uri');
 const isVideo = require('../lib/video');
+const { getRandomProxy, getRandomUserAgent } = require('../lib/request_helper');
+const { cacheWrapResolvedUrl, cacheWrapProxy, cacheUserAgent } = require('../lib/cache');

-const RESOLVER_HOST = process.env.RESOLVER_HOST || 'http://localhost:7000';
+const RESOLVER_HOST = process.env.RESOLVER_HOST || 'http://localhost:7050';
+
+const unrestrictQueue = new namedQueue((task, callback) => task.method()
+    .then(result => callback(false, result))
+    .catch((error => callback(error))));

 async function applyMoch(streams, apiKey) {
-  const RD = new RealDebridClient(apiKey);
+  const options = await getDefaultOptions(apiKey);
+  const RD = new RealDebridClient(apiKey, options);
  const hashes = streams.map(stream => stream.infoHash);
  const available = await RD.torrents.instantAvailability(hashes)
      .catch(error => {
@@ -14,7 +23,7 @@ async function applyMoch(streams, apiKey) {
  if (available) {
    streams.forEach(stream => {
      const cachedEntry = available[stream.infoHash];
-      const cachedIds = getCachedFileIds(stream.fileIdx, cachedEntry).join(',');
+      const cachedIds = _getCachedFileIds(stream.fileIdx, cachedEntry).join(',');
      if (cachedIds.length) {
        stream.name = `[RD+] ${stream.name}`;
        stream.url = `${RESOLVER_HOST}/realdebrid/${apiKey}/${stream.infoHash}/${cachedIds}/${stream.fileIdx}`;
@@ -27,7 +36,19 @@ async function applyMoch(streams, apiKey) {
  return streams;
 }

-function getCachedFileIds(fileIndex, hosterResults) {
+async function resolve(apiKey, infoHash, cachedFileIds, fileIndex) {
+  if (!apiKey || !infoHash || !cachedFileIds || !cachedFileIds.length) {
+    return Promise.reject("No valid parameters passed");
+  }
+  const id = `${apiKey}_${infoHash}_${fileIndex}`;
+  const method = () => cacheWrapResolvedUrl(id, () => _unrestrict(apiKey, infoHash, cachedFileIds, fileIndex));
+
+  return new Promise(((resolve, reject) => {
+    unrestrictQueue.push({ id, method }, (error, result) => result ? resolve(result) : reject(error));
+  }));
+}
+
+function _getCachedFileIds(fileIndex, hosterResults) {
  if (!hosterResults || Array.isArray(hosterResults)) {
    return [];
  }
@@ -41,4 +62,59 @@ function getCachedFileIds(fileIndex, hosterResults) {
  return cachedTorrents.length && cachedTorrents[0] || [];
 }

-module.exports = { applyMoch };
+async function _unrestrict(apiKey, infoHash, cachedFileIds, fileIndex) {
+  console.log(`Unrestricting ${infoHash} [${fileIndex}]`);
+  const options = await getDefaultOptions(apiKey);
+  const RD = new RealDebridClient(apiKey, options);
+  const torrentId = await _createOrFindTorrentId(RD, infoHash, cachedFileIds);
+  if (torrentId) {
+    const info = await RD.torrents.info(torrentId);
+    const targetFile = info.files.find(file => file.id === fileIndex + 1)
+        || info.files.filter(file => file.selected).sort((a, b) => b.bytes - a.bytes)[0];
+    const selectedFiles = info.files.filter(file => file.selected);
+    const fileLink = info.links.length === 1
+        ? info.links[0]
+        : info.links[selectedFiles.indexOf(targetFile)];
+    const unrestrictedLink = await _unrestrictLink(RD, fileLink);
+    console.log(`Unrestricted ${infoHash} [${fileIndex}] to ${unrestrictedLink}`);
+    return unrestrictedLink;
+  }
+  return Promise.reject("Failed adding torrent");
+}
+
+async function _createOrFindTorrentId(RD, infoHash, cachedFileIds) {
+  return RD.torrents.get(0, 1)
+      .then(torrents => torrents.find(torrent => torrent.hash.toLowerCase() === infoHash))
+      .then(torrent => torrent && torrent.id || Promise.reject('No recent torrent found'))
+      .catch((error) => RD.torrents.addMagnet(encode({ infoHash }))
+          .then(response => RD.torrents.selectFiles(response.id, cachedFileIds)
+              .then((() => response.id))))
+      .catch(error => {
+        console.warn('Failed RealDebrid torrent retrieval', error);
+        return undefined;
+      });
+}
+
+async function _unrestrictLink(RD, link) {
+  if (!link || !link.length) {
+    return Promise.reject("No available links found");
+  }
+  return RD.unrestrict.link(link)
+      .then(unrestrictedLink => unrestrictedLink.download);
+  // .then(unrestrictedLink => RD.streaming.transcode(unrestrictedLink.id))
+  // .then(transcodedLink => {
+  //   const url = transcodedLink.apple && transcodedLink.apple.full
+  //       || transcodedLink[Object.keys(transcodedLink)[0]].full;
+  //   console.log(`Unrestricted ${link} to ${url}`);
+  //   return url;
+  // });
+}
+
+async function getDefaultOptions(id) {
+  const userAgent = await cacheUserAgent(id, () => getRandomUserAgent()).catch(() => getRandomUserAgent());
+  const proxy = await cacheWrapProxy('realdebrid', () => getRandomProxy()).catch(() => getRandomProxy());
+
+  return { proxy: proxy, headers: { 'User-Agent': userAgent } };
+}
+
+module.exports = { applyMoch, resolve };
--- a/addon/package-lock.json
+++ b/addon/package-lock.json
@@ -420,6 +420,16 @@
      "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz",
      "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA="
    },
+    "detect-indent": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/detect-indent/-/detect-indent-6.0.0.tgz",
+      "integrity": "sha512-oSyFlqaTHCItVRGK5RmrmjB+CmaMOW7IaNA/kdxqhoa6d17j/5ce9O9eWXmV/KEdRwqpQA+Vqe8a8Bsybu4YnA=="
+    },
+    "docopt": {
+      "version": "0.6.2",
+      "resolved": "https://registry.npmjs.org/docopt/-/docopt-0.6.2.tgz",
+      "integrity": "sha1-so6eIiDaXsSffqW7JKR3h0Be6xE="
+    },
    "doctrine": {
      "version": "3.0.0",
      "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz",
@@ -429,6 +439,16 @@
        "esutils": "^2.0.2"
      }
    },
+    "dot-json": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/dot-json/-/dot-json-1.2.0.tgz",
+      "integrity": "sha512-4bEM7KHFl/U9gAI5nIvU0/fwVzNnE713K339vcxAMtxd2D9mZP6o65UwlcXigJL4rfk90UM0J+D7IPIFYZMQ8Q==",
+      "requires": {
+        "detect-indent": "~6.0.0",
+        "docopt": "~0.6.2",
+        "underscore-keypath": "~0.0.22"
+      }
+    },
    "dottie": {
      "version": "2.0.2",
      "resolved": "https://registry.npmjs.org/dottie/-/dottie-2.0.2.tgz",
@@ -1339,6 +1359,15 @@
        "yallist": "^2.0.0"
      }
    },
+    "magnet-uri": {
+      "version": "5.2.4",
+      "resolved": "https://registry.npmjs.org/magnet-uri/-/magnet-uri-5.2.4.tgz",
+      "integrity": "sha512-VYaJMxhr8B9BrCiNINUsuhaEe40YnG+AQBwcqUKO66lSVaI9I3A1iH/6EmEwRI8OYUg5Gt+4lLE7achg676lrg==",
+      "requires": {
+        "thirty-two": "^1.0.1",
+        "uniq": "^1.0.1"
+      }
+    },
    "media-typer": {
      "version": "0.3.0",
      "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
@@ -1441,6 +1470,11 @@
      "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.7.tgz",
      "integrity": "sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s="
    },
+    "named-queue": {
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/named-queue/-/named-queue-2.2.1.tgz",
+      "integrity": "sha1-GBRURVNZnVqeQD0N+pN6TODR5qc="
+    },
    "natural-compare": {
      "version": "1.4.0",
      "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz",
@@ -2251,6 +2285,11 @@
      "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=",
      "dev": true
    },
+    "thirty-two": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/thirty-two/-/thirty-two-1.0.2.tgz",
+      "integrity": "sha1-TKL//AKlEpDSdEueP1V2k8prYno="
+    },
    "through": {
      "version": "2.3.8",
      "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
@@ -2330,6 +2369,19 @@
      "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.9.2.tgz",
      "integrity": "sha512-D39qtimx0c1fI3ya1Lnhk3E9nONswSKhnffBI0gME9C99fYOkNi04xs8K6pePLhvl1frbDemkaBQ5ikWllR2HQ=="
    },
+    "underscore-keypath": {
+      "version": "0.0.22",
+      "resolved": "https://registry.npmjs.org/underscore-keypath/-/underscore-keypath-0.0.22.tgz",
+      "integrity": "sha1-SKUoOSu278QkvhyqVtpLX6zPJk0=",
+      "requires": {
+        "underscore": "*"
+      }
+    },
+    "uniq": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/uniq/-/uniq-1.0.1.tgz",
+      "integrity": "sha1-sxxa6CVIRKOoKBVBzisEuGWnNP8="
+    },
    "unpipe": {
      "version": "1.0.0",
      "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
@@ -2343,6 +2395,15 @@
        "punycode": "^2.1.0"
      }
    },
+    "user-agents": {
+      "version": "1.0.559",
+      "resolved": "https://registry.npmjs.org/user-agents/-/user-agents-1.0.559.tgz",
+      "integrity": "sha512-HdAlNS3vDxOGMRwmv8or05xL96MV3CEwQhUSFTCRoOvTOEnWhTEBPAHRry/xZpVTTOtx77UHMal8YKcx6fs7Lg==",
+      "requires": {
+        "dot-json": "^1.2.0",
+        "lodash.clonedeep": "^4.5.0"
+      }
+    },
    "util-deprecate": {
      "version": "1.0.2",
      "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
--- a/addon/package.json
+++ b/addon/package.json
@@ -11,13 +11,16 @@
    "cache-manager": "^2.11.1",
    "cache-manager-mongodb": "^0.2.2",
    "express-rate-limit": "^5.1.1",
+    "named-queue": "^2.2.1",
    "needle": "^2.2.4",
+    "magnet-uri": "^5.1.7",
    "parse-torrent-title": "git://github.com/TheBeastLT/parse-torrent-title.git#afd4a374276420c13c52df8e3d07ae7699c46b60",
    "pg": "^7.8.2",
    "pg-hstore": "^2.3.2",
    "real-debrid-api": "git://github.com/TheBeastLT/node-real-debrid.git#935a5c23ae809edbcd2a111526a7f74d6767c50d",
    "sequelize": "^4.43.0",
-    "stremio-addon-sdk": "^1.6.1"
+    "stremio-addon-sdk": "^1.6.1",
+    "user-agents": "^1.0.559"
  },
  "devDependencies": {
    "eslint": "^6.4.0",
--- a/addon/serverless.js
+++ b/addon/serverless.js
@@ -4,6 +4,7 @@ const addonInterface = require('./addon');
 const { manifest } = require('./lib/manifest');
 const parseConfiguration = require('./lib/configuration');
 const landingTemplate = require('./lib/landingTemplate');
+const realDebrid = require('./moch/realdebrid');

 const router = getRouter(addonInterface);
 const limiter = rateLimit({
@@ -68,6 +69,20 @@ router.get('/:configuration/:resource/:type/:id.json', (req, res, next) => {
      });
 });

+router.get('/realdebrid/:apiKey/:infoHash/:cachedFileIds/:fileIndex?', (req, res) => {
+  const { apiKey, infoHash, cachedFileIds, fileIndex } = req.params;
+  realDebrid.resolve(apiKey, infoHash, cachedFileIds, isNaN(fileIndex) ? undefined : parseInt(fileIndex))
+      .then(url => {
+        res.writeHead(301, { Location: url });
+        res.end();
+      })
+      .catch(error => {
+        console.log(error);
+        res.statusCode = 404;
+        res.end();
+      });
+});
+
 module.exports = function (req, res) {
  router(req, res, function () {
    res.statusCode = 404;