From 084744b35b9ddd09db4eeee01c8f8b7b81be17f4 Mon Sep 17 00:00:00 2001 From: TheBeastLT Date: Thu, 19 Mar 2020 12:21:10 +0100 Subject: [PATCH] [scraper] moves moch url resolving to scraper --- package-lock.json | 8 ++++ package.json | 2 + scraper/index.js | 17 +++++++++ scraper/lib/cache.js | 7 +++- scraper/moch/realdebrid.js | 77 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 110 insertions(+), 1 deletion(-) create mode 100644 scraper/moch/realdebrid.js diff --git a/package-lock.json b/package-lock.json index a194ed7..ac96b72 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1726,6 +1726,14 @@ "util-deprecate": "^1.0.1" } }, + "real-debrid-api": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/real-debrid-api/-/real-debrid-api-1.0.1.tgz", + "integrity": "sha512-Rva3aZ62R/KDV7ydUALrVu8YbAuPLLmThWMlmVMshUEBQk74ybRyUv5WTvfrd9i94v+0V9YeZle0pldv5cQ2+g==", + "requires": { + "request": "^2.83.0" + } + }, "request": { "version": "2.88.0", "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", diff --git a/package.json b/package.json index 777747e..e5ad7b8 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "magnet-uri": "^5.1.7", "moment": "^2.24.0", "name-to-imdb": "^2.3.0", + "named-queue": "^2.2.1", "needle": "^2.2.4", "node-gzip": "^1.1.2", "node-schedule": "^1.3.2", @@ -34,6 +35,7 @@ "pg": "^7.8.2", "pg-hstore": "^2.3.2", "rarbg-api": "^1.1.3", + "real-debrid-api": "^1.0.1", "sequelize": "^5.21.5", "sugar-date": "^2.0.6", "torrent-stream": "^1.1.0" diff --git a/scraper/index.js b/scraper/index.js index efbc27a..d0f71e9 100644 --- a/scraper/index.js +++ b/scraper/index.js @@ -3,6 +3,7 @@ const express = require("express"); const server = express(); const schedule = require('node-schedule'); const { connect } = require('./lib/repository'); +const realDebrid = require('./moch/realdebrid'); const thepiratebayScraper = require('./scrapers/thepiratebay/thepiratebay_scraper'); const horribleSubsScraper = require('./scrapers/horriblesubs/horriblesubs_scraper'); const leetxScraper = require('./scrapers/1337x/1337x_scraper'); @@ -47,6 +48,22 @@ server.get('/', function (req, res) { res.sendStatus(200); }); +server.get('/realdebrid/:apiKey/:infoHash/:cachedFileIds/:fileIndex?', (req, res) => { + const { apiKey, infoHash, cachedFileIds, fileIndex } = req.params; + console.log(`Unrestricting ${infoHash} [${fileIndex}]`); + realDebrid.resolve(apiKey, infoHash, cachedFileIds, isNaN(fileIndex) ? undefined : parseInt(fileIndex)) + .then(url => { + console.log(`Unrestricted ${infoHash} [${fileIndex}] to ${url}`); + res.writeHead(301, { Location: url }); + res.end(); + }) + .catch(error => { + console.log(error); + res.statusCode = 404; + res.end(); + }); +}); + server.listen(process.env.PORT || 7000, async () => { await connect(); console.log('Scraper started'); diff --git a/scraper/lib/cache.js b/scraper/lib/cache.js index 34cffc9..399bd80 100644 --- a/scraper/lib/cache.js +++ b/scraper/lib/cache.js @@ -5,6 +5,7 @@ const GLOBAL_KEY_PREFIX = 'stremio-torrentio'; const IMDB_ID_PREFIX = `${GLOBAL_KEY_PREFIX}|imdb_id`; const KITSU_ID_PREFIX = `${GLOBAL_KEY_PREFIX}|kitsu_id`; const METADATA_PREFIX = `${GLOBAL_KEY_PREFIX}|metadata`; +const RESOLVED_URL_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|moch`; const TORRENT_FILES_KEY_PREFIX = `stremio-tpb|files`; const GLOBAL_TTL = process.env.METADATA_TTL || 7 * 24 * 60 * 60; // 7 days @@ -84,5 +85,9 @@ function cacheWrapMetadata(id, method) { return cacheWrap(memoryCache, `${METADATA_PREFIX}:${id}`, method, { ttl: MEMORY_TTL }); } -module.exports = { cacheWrapImdbId, cacheWrapKitsuId, cacheWrapMetadata, retrieveTorrentFiles }; +function cacheWrapResolvedUrl(id, method) { + return cacheWrap(memoryCache, `${RESOLVED_URL_KEY_PREFIX}:${id}`, method, { ttl: { MEMORY_TTL } }); +} + +module.exports = { cacheWrapImdbId, cacheWrapKitsuId, cacheWrapMetadata, retrieveTorrentFiles, cacheWrapResolvedUrl }; diff --git a/scraper/moch/realdebrid.js b/scraper/moch/realdebrid.js new file mode 100644 index 0000000..f0db71a --- /dev/null +++ b/scraper/moch/realdebrid.js @@ -0,0 +1,77 @@ +const { encode } = require('magnet-uri'); +const RealDebridClient = require('real-debrid-api'); +const namedQueue = require('named-queue'); +const { cacheWrapResolvedUrl } = require('../lib/cache'); + +const PROXY_HOST = process.env.PROXY_HOST; +const PROXY_USERNAME = process.env.PROXY_USERNAME; +const PROXY_PASSWORD = process.env.PROXY_PASSWORD; + +const unrestrictQueue = new namedQueue((task, callback) => task.method() + .then(result => callback(false, result)) + .catch((error => callback(error)))); + +async function resolve(apiKey, infoHash, cachedFileIds, fileIndex) { + if (!apiKey || !infoHash || !cachedFileIds || !cachedFileIds.length) { + return Promise.reject("No valid parameters passed"); + } + const id = `${apiKey}_${infoHash}_${fileIndex}`; + const method = () => cacheWrapResolvedUrl(id, () => _unrestrict(apiKey, infoHash, cachedFileIds, fileIndex)); + + return new Promise(((resolve, reject) => { + unrestrictQueue.push({ id, method }, (error, result) => result ? resolve(result) : reject(error)); + })); +} + +async function _unrestrict(apiKey, infoHash, cachedFileIds, fileIndex) { + const RD = new RealDebridClient(apiKey); + const torrentId = await _createOrFindTorrentId(RD, infoHash, cachedFileIds); + if (torrentId) { + const info = await RD.torrents.info(torrentId); + const targetFile = info.files.find(file => file.id === fileIndex + 1) + || info.files.filter(file => file.selected).sort((a, b) => b.bytes - a.bytes)[0]; + const selectedFiles = info.files.filter(file => file.selected); + const fileLink = info.links.length === 1 + ? info.links[0] + : info.links[selectedFiles.indexOf(targetFile)]; + return _unrestrictLink(RD, fileLink); + } + return Promise.reject("Failed adding torrent"); +} + +async function _createOrFindTorrentId(RD, infoHash, cachedFileIds) { + return RD.torrents.get(0, 1) + .then(torrents => torrents.find(torrent => torrent.hash.toLowerCase() === infoHash)) + .then(torrent => torrent && torrent.id || Promise.reject('No recent torrent found')) + .catch((error) => RD.torrents.addMagnet(encode({ infoHash })) + .then(response => RD.torrents.selectFiles(response.id, cachedFileIds) + .then((() => response.id)))) + .catch(error => { + console.warn('Failed RealDebrid torrent retrieval', error); + return undefined; + }); +} + +async function _unrestrictLink(RD, link) { + if (!link || !link.length) { + return Promise.reject("No available links found"); + } + return RD._post('unrestrict/link', { form: { link }, proxy: getProxy() }) + .then(unrestrictedLink => unrestrictedLink.download); + // .then(unrestrictedLink => RD.streaming.transcode(unrestrictedLink.id)) + // .then(transcodedLink => { + // const url = transcodedLink.apple && transcodedLink.apple.full + // || transcodedLink[Object.keys(transcodedLink)[0]].full; + // console.log(`Unrestricted ${link} to ${url}`); + // return url; + // }); +} + +function getProxy() { + if (PROXY_HOST && PROXY_USERNAME && PROXY_PASSWORD) { + return `http://${PROXY_USERNAME}:${PROXY_PASSWORD}@${PROXY_HOST}`; + } + return undefined; +} + +module.exports = { resolve }; \ No newline at end of file