[scraper] moves moch url resolving to scraper

package-lock.json

@@ -1726,6 +1726,14 @@
         "util-deprecate": "^1.0.1"
       }
     },
+    "real-debrid-api": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/real-debrid-api/-/real-debrid-api-1.0.1.tgz",
+      "integrity": "sha512-Rva3aZ62R/KDV7ydUALrVu8YbAuPLLmThWMlmVMshUEBQk74ybRyUv5WTvfrd9i94v+0V9YeZle0pldv5cQ2+g==",
+      "requires": {
+        "request": "^2.83.0"
+      }
+    },
     "request": {
       "version": "2.88.0",
       "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz",

package.json

@@ -24,6 +24,7 @@
     "magnet-uri": "^5.1.7",
     "moment": "^2.24.0",
     "name-to-imdb": "^2.3.0",
+    "named-queue": "^2.2.1",
     "needle": "^2.2.4",
     "node-gzip": "^1.1.2",
     "node-schedule": "^1.3.2",
@@ -34,6 +35,7 @@
     "pg": "^7.8.2",
     "pg-hstore": "^2.3.2",
     "rarbg-api": "^1.1.3",
+    "real-debrid-api": "^1.0.1",
     "sequelize": "^5.21.5",
     "sugar-date": "^2.0.6",
     "torrent-stream": "^1.1.0"

scraper/index.js

@@ -3,6 +3,7 @@ const express = require("express");
 const server = express();
 const schedule = require('node-schedule');
 const { connect } = require('./lib/repository');
+const realDebrid = require('./moch/realdebrid');
 const thepiratebayScraper = require('./scrapers/thepiratebay/thepiratebay_scraper');
 const horribleSubsScraper = require('./scrapers/horriblesubs/horriblesubs_scraper');
 const leetxScraper = require('./scrapers/1337x/1337x_scraper');
@@ -47,6 +48,22 @@ server.get('/', function (req, res) {
   res.sendStatus(200);
 });
 
+server.get('/realdebrid/:apiKey/:infoHash/:cachedFileIds/:fileIndex?', (req, res) => {
+  const { apiKey, infoHash, cachedFileIds, fileIndex } = req.params;
+  console.log(`Unrestricting ${infoHash} [${fileIndex}]`);
+  realDebrid.resolve(apiKey, infoHash, cachedFileIds, isNaN(fileIndex) ? undefined : parseInt(fileIndex))
+      .then(url => {
+        console.log(`Unrestricted ${infoHash} [${fileIndex}] to ${url}`);
+        res.writeHead(301, { Location: url });
+        res.end();
+      })
+      .catch(error => {
+        console.log(error);
+        res.statusCode = 404;
+        res.end();
+      });
+});
+
 server.listen(process.env.PORT || 7000, async () => {
   await connect();
   console.log('Scraper started');

scraper/lib/cache.js

@@ -5,6 +5,7 @@ const GLOBAL_KEY_PREFIX = 'stremio-torrentio';
 const IMDB_ID_PREFIX = `${GLOBAL_KEY_PREFIX}|imdb_id`;
 const KITSU_ID_PREFIX = `${GLOBAL_KEY_PREFIX}|kitsu_id`;
 const METADATA_PREFIX = `${GLOBAL_KEY_PREFIX}|metadata`;
+const RESOLVED_URL_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|moch`;
 const TORRENT_FILES_KEY_PREFIX = `stremio-tpb|files`;
 
 const GLOBAL_TTL = process.env.METADATA_TTL || 7 * 24 * 60 * 60; // 7 days
@@ -84,5 +85,9 @@ function cacheWrapMetadata(id, method) {
   return cacheWrap(memoryCache, `${METADATA_PREFIX}:${id}`, method, { ttl: MEMORY_TTL });
 }
 
-module.exports = { cacheWrapImdbId, cacheWrapKitsuId, cacheWrapMetadata, retrieveTorrentFiles };
+function cacheWrapResolvedUrl(id, method) {
+  return cacheWrap(memoryCache, `${RESOLVED_URL_KEY_PREFIX}:${id}`, method, { ttl: { MEMORY_TTL } });
+}
+
+module.exports = { cacheWrapImdbId, cacheWrapKitsuId, cacheWrapMetadata, retrieveTorrentFiles, cacheWrapResolvedUrl };
 

scraper/moch/realdebrid.js

@@ -0,0 +1,77 @@
+const { encode } = require('magnet-uri');
+const RealDebridClient = require('real-debrid-api');
+const namedQueue = require('named-queue');
+const { cacheWrapResolvedUrl } = require('../lib/cache');
+
+const PROXY_HOST = process.env.PROXY_HOST;
+const PROXY_USERNAME = process.env.PROXY_USERNAME;
+const PROXY_PASSWORD = process.env.PROXY_PASSWORD;
+
+const unrestrictQueue = new namedQueue((task, callback) => task.method()
+    .then(result => callback(false, result))
+    .catch((error => callback(error))));
+
+async function resolve(apiKey, infoHash, cachedFileIds, fileIndex) {
+  if (!apiKey || !infoHash || !cachedFileIds || !cachedFileIds.length) {
+    return Promise.reject("No valid parameters passed");
+  }
+  const id = `${apiKey}_${infoHash}_${fileIndex}`;
+  const method = () => cacheWrapResolvedUrl(id, () => _unrestrict(apiKey, infoHash, cachedFileIds, fileIndex));
+
+  return new Promise(((resolve, reject) => {
+    unrestrictQueue.push({ id, method }, (error, result) => result ? resolve(result) : reject(error));
+  }));
+}
+
+async function _unrestrict(apiKey, infoHash, cachedFileIds, fileIndex) {
+  const RD = new RealDebridClient(apiKey);
+  const torrentId = await _createOrFindTorrentId(RD, infoHash, cachedFileIds);
+  if (torrentId) {
+    const info = await RD.torrents.info(torrentId);
+    const targetFile = info.files.find(file => file.id === fileIndex + 1)
+        || info.files.filter(file => file.selected).sort((a, b) => b.bytes - a.bytes)[0];
+    const selectedFiles = info.files.filter(file => file.selected);
+    const fileLink = info.links.length === 1
+        ? info.links[0]
+        : info.links[selectedFiles.indexOf(targetFile)];
+    return _unrestrictLink(RD, fileLink);
+  }
+  return Promise.reject("Failed adding torrent");
+}
+
+async function _createOrFindTorrentId(RD, infoHash, cachedFileIds) {
+  return RD.torrents.get(0, 1)
+      .then(torrents => torrents.find(torrent => torrent.hash.toLowerCase() === infoHash))
+      .then(torrent => torrent && torrent.id || Promise.reject('No recent torrent found'))
+      .catch((error) => RD.torrents.addMagnet(encode({ infoHash }))
+          .then(response => RD.torrents.selectFiles(response.id, cachedFileIds)
+              .then((() => response.id))))
+      .catch(error => {
+        console.warn('Failed RealDebrid torrent retrieval', error);
+        return undefined;
+      });
+}
+
+async function _unrestrictLink(RD, link) {
+  if (!link || !link.length) {
+    return Promise.reject("No available links found");
+  }
+  return RD._post('unrestrict/link', { form: { link }, proxy: getProxy() })
+      .then(unrestrictedLink => unrestrictedLink.download);
+  // .then(unrestrictedLink => RD.streaming.transcode(unrestrictedLink.id))
+  // .then(transcodedLink => {
+  //   const url = transcodedLink.apple && transcodedLink.apple.full
+  //       || transcodedLink[Object.keys(transcodedLink)[0]].full;
+  //   console.log(`Unrestricted ${link} to ${url}`);
+  //   return url;
+  // });
+}
+
+function getProxy() {
+  if (PROXY_HOST && PROXY_USERNAME && PROXY_PASSWORD) {
+    return `http://${PROXY_USERNAME}:${PROXY_PASSWORD}@${PROXY_HOST}`;
+  }
+  return undefined;
+}
+
+module.exports = { resolve };