diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 018484b..eb5c01b 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -40,5 +40,5 @@ jobs: docker load -i /tmp/docker/torrentio_addon_latest.tar docker stop torrentio-addon docker rm torrentio-addon - docker run -p 80:7000 -d --name torrentio-addon --restart always -e MONGODB_URI=${{ secrets.MONGODB_URI }} -e DATABASE_URI=${{ secrets.DATABASE_URI }} -e RESOLVER_HOST=${{ secrets.RESOLVER_HOST }} torrentio-addon:latest + docker run -p 80:7000 -d --name torrentio-addon --restart always -e MONGODB_URI=${{ secrets.MONGODB_URI }} -e DATABASE_URI=${{ secrets.DATABASE_URI }} -e RESOLVER_HOST=${{ secrets.RESOLVER_HOST }} -e PROXY_HOSTS=${{ secrets.PROXY_HOSTS }} -e PROXY_USERNAME=${{ secrets.PROXY_USERNAME }} -e PROXY_PASSWORD=${{ secrets.PROXY_PASSWORD }} torrentio-addon:latest docker image prune -f diff --git a/addon/lib/cache.js b/addon/lib/cache.js index 341be6d..a2976a1 100644 --- a/addon/lib/cache.js +++ b/addon/lib/cache.js @@ -3,17 +3,24 @@ const mangodbStore = require('cache-manager-mongodb'); const GLOBAL_KEY_PREFIX = 'torrentio-addon'; const STREAM_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|stream`; +const RESOLVED_URL_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|resolved`; +const PROXY_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|proxy`; +const USER_AGENT_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|agent`; const STREAM_TTL = process.env.STREAM_TTL || 4 * 60 * 60; // 4 hours const STREAM_EMPTY_TTL = process.env.STREAM_EMPTY_TTL || 30 * 60; // 30 minutes +const RESOLVED_URL_TTL = 2 * 60; // 2 minutes +const PROXY_TTL = 60 * 60; // 60 minutes +const USER_AGENT_TTL = 2 * 24 * 60 * 60; // 2 days // When the streams are empty we want to cache it for less time in case of timeouts or failures const MONGO_URI = process.env.MONGODB_URI; const NO_CACHE = process.env.NO_CACHE || false; -const remoteCache = initiateCache(); +const memoryCache = initiateMemoryCache(); +const remoteCache = initiateRemoteCache(); -function initiateCache() { +function initiateRemoteCache() { if (NO_CACHE) { return null; } else if (MONGO_URI) { @@ -37,6 +44,13 @@ function initiateCache() { } } +function initiateMemoryCache() { + return cacheManager.caching({ + store: 'memory', + ttl: STREAM_TTL + }); +} + function cacheWrap(cache, key, method, options) { if (NO_CACHE || !cache) { return method(); @@ -50,5 +64,17 @@ function cacheWrapStream(id, method) { }); } -module.exports = { cacheWrapStream }; +function cacheWrapResolvedUrl(id, method) { + return cacheWrap(memoryCache, `${RESOLVED_URL_KEY_PREFIX}:${id}`, method, { ttl: { RESOLVED_URL_TTL } }); +} + +function cacheWrapProxy(id, method) { + return cacheWrap(memoryCache, `${PROXY_KEY_PREFIX}:${id}`, method, { ttl: { PROXY_TTL } }); +} + +function cacheUserAgent(id, method) { + return cacheWrap(memoryCache, `${USER_AGENT_KEY_PREFIX}:${id}`, method, { ttl: { USER_AGENT_TTL } }); +} + +module.exports = { cacheWrapStream, cacheWrapResolvedUrl, cacheWrapProxy, cacheUserAgent }; diff --git a/addon/lib/request_helper.js b/addon/lib/request_helper.js new file mode 100644 index 0000000..bbd799c --- /dev/null +++ b/addon/lib/request_helper.js @@ -0,0 +1,20 @@ +const UserAgent = require('user-agents'); + +const PROXY_HOSTS = process.env.PROXY_HOSTS && process.env.PROXY_HOSTS.split(','); +const PROXY_USERNAME = process.env.PROXY_USERNAME; +const PROXY_PASSWORD = process.env.PROXY_PASSWORD; +const userAgent = new UserAgent(); + +function getRandomUserAgent() { + return userAgent.random().toString(); +} + +function getRandomProxy() { + if (PROXY_HOSTS && PROXY_HOSTS.length && PROXY_USERNAME && PROXY_PASSWORD) { + return `http://${PROXY_USERNAME}:${PROXY_PASSWORD}@${PROXY_HOSTS[Math.floor(Math.random() * PROXY_HOSTS.length)]}`; + } + console.warn('No proxy configured!'); + return undefined; +} + +module.exports = { getRandomUserAgent, getRandomProxy }; \ No newline at end of file diff --git a/addon/moch/realdebrid.js b/addon/moch/realdebrid.js index 500bb2c..7d51ebc 100644 --- a/addon/moch/realdebrid.js +++ b/addon/moch/realdebrid.js @@ -1,10 +1,19 @@ const RealDebridClient = require('real-debrid-api'); +const namedQueue = require('named-queue'); +const { encode } = require('magnet-uri'); const isVideo = require('../lib/video'); +const { getRandomProxy, getRandomUserAgent } = require('../lib/request_helper'); +const { cacheWrapResolvedUrl, cacheWrapProxy, cacheUserAgent } = require('../lib/cache'); -const RESOLVER_HOST = process.env.RESOLVER_HOST || 'http://localhost:7000'; +const RESOLVER_HOST = process.env.RESOLVER_HOST || 'http://localhost:7050'; + +const unrestrictQueue = new namedQueue((task, callback) => task.method() + .then(result => callback(false, result)) + .catch((error => callback(error)))); async function applyMoch(streams, apiKey) { - const RD = new RealDebridClient(apiKey); + const options = await getDefaultOptions(apiKey); + const RD = new RealDebridClient(apiKey, options); const hashes = streams.map(stream => stream.infoHash); const available = await RD.torrents.instantAvailability(hashes) .catch(error => { @@ -14,7 +23,7 @@ async function applyMoch(streams, apiKey) { if (available) { streams.forEach(stream => { const cachedEntry = available[stream.infoHash]; - const cachedIds = getCachedFileIds(stream.fileIdx, cachedEntry).join(','); + const cachedIds = _getCachedFileIds(stream.fileIdx, cachedEntry).join(','); if (cachedIds.length) { stream.name = `[RD+] ${stream.name}`; stream.url = `${RESOLVER_HOST}/realdebrid/${apiKey}/${stream.infoHash}/${cachedIds}/${stream.fileIdx}`; @@ -27,7 +36,19 @@ async function applyMoch(streams, apiKey) { return streams; } -function getCachedFileIds(fileIndex, hosterResults) { +async function resolve(apiKey, infoHash, cachedFileIds, fileIndex) { + if (!apiKey || !infoHash || !cachedFileIds || !cachedFileIds.length) { + return Promise.reject("No valid parameters passed"); + } + const id = `${apiKey}_${infoHash}_${fileIndex}`; + const method = () => cacheWrapResolvedUrl(id, () => _unrestrict(apiKey, infoHash, cachedFileIds, fileIndex)); + + return new Promise(((resolve, reject) => { + unrestrictQueue.push({ id, method }, (error, result) => result ? resolve(result) : reject(error)); + })); +} + +function _getCachedFileIds(fileIndex, hosterResults) { if (!hosterResults || Array.isArray(hosterResults)) { return []; } @@ -41,4 +62,59 @@ function getCachedFileIds(fileIndex, hosterResults) { return cachedTorrents.length && cachedTorrents[0] || []; } -module.exports = { applyMoch }; \ No newline at end of file +async function _unrestrict(apiKey, infoHash, cachedFileIds, fileIndex) { + console.log(`Unrestricting ${infoHash} [${fileIndex}]`); + const options = await getDefaultOptions(apiKey); + const RD = new RealDebridClient(apiKey, options); + const torrentId = await _createOrFindTorrentId(RD, infoHash, cachedFileIds); + if (torrentId) { + const info = await RD.torrents.info(torrentId); + const targetFile = info.files.find(file => file.id === fileIndex + 1) + || info.files.filter(file => file.selected).sort((a, b) => b.bytes - a.bytes)[0]; + const selectedFiles = info.files.filter(file => file.selected); + const fileLink = info.links.length === 1 + ? info.links[0] + : info.links[selectedFiles.indexOf(targetFile)]; + const unrestrictedLink = await _unrestrictLink(RD, fileLink); + console.log(`Unrestricted ${infoHash} [${fileIndex}] to ${unrestrictedLink}`); + return unrestrictedLink; + } + return Promise.reject("Failed adding torrent"); +} + +async function _createOrFindTorrentId(RD, infoHash, cachedFileIds) { + return RD.torrents.get(0, 1) + .then(torrents => torrents.find(torrent => torrent.hash.toLowerCase() === infoHash)) + .then(torrent => torrent && torrent.id || Promise.reject('No recent torrent found')) + .catch((error) => RD.torrents.addMagnet(encode({ infoHash })) + .then(response => RD.torrents.selectFiles(response.id, cachedFileIds) + .then((() => response.id)))) + .catch(error => { + console.warn('Failed RealDebrid torrent retrieval', error); + return undefined; + }); +} + +async function _unrestrictLink(RD, link) { + if (!link || !link.length) { + return Promise.reject("No available links found"); + } + return RD.unrestrict.link(link) + .then(unrestrictedLink => unrestrictedLink.download); + // .then(unrestrictedLink => RD.streaming.transcode(unrestrictedLink.id)) + // .then(transcodedLink => { + // const url = transcodedLink.apple && transcodedLink.apple.full + // || transcodedLink[Object.keys(transcodedLink)[0]].full; + // console.log(`Unrestricted ${link} to ${url}`); + // return url; + // }); +} + +async function getDefaultOptions(id) { + const userAgent = await cacheUserAgent(id, () => getRandomUserAgent()).catch(() => getRandomUserAgent()); + const proxy = await cacheWrapProxy('realdebrid', () => getRandomProxy()).catch(() => getRandomProxy()); + + return { proxy: proxy, headers: { 'User-Agent': userAgent } }; +} + +module.exports = { applyMoch, resolve }; \ No newline at end of file diff --git a/addon/package-lock.json b/addon/package-lock.json index c1abe27..e726b90 100644 --- a/addon/package-lock.json +++ b/addon/package-lock.json @@ -420,6 +420,16 @@ "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" }, + "detect-indent": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/detect-indent/-/detect-indent-6.0.0.tgz", + "integrity": "sha512-oSyFlqaTHCItVRGK5RmrmjB+CmaMOW7IaNA/kdxqhoa6d17j/5ce9O9eWXmV/KEdRwqpQA+Vqe8a8Bsybu4YnA==" + }, + "docopt": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/docopt/-/docopt-0.6.2.tgz", + "integrity": "sha1-so6eIiDaXsSffqW7JKR3h0Be6xE=" + }, "doctrine": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", @@ -429,6 +439,16 @@ "esutils": "^2.0.2" } }, + "dot-json": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/dot-json/-/dot-json-1.2.0.tgz", + "integrity": "sha512-4bEM7KHFl/U9gAI5nIvU0/fwVzNnE713K339vcxAMtxd2D9mZP6o65UwlcXigJL4rfk90UM0J+D7IPIFYZMQ8Q==", + "requires": { + "detect-indent": "~6.0.0", + "docopt": "~0.6.2", + "underscore-keypath": "~0.0.22" + } + }, "dottie": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/dottie/-/dottie-2.0.2.tgz", @@ -1339,6 +1359,15 @@ "yallist": "^2.0.0" } }, + "magnet-uri": { + "version": "5.2.4", + "resolved": "https://registry.npmjs.org/magnet-uri/-/magnet-uri-5.2.4.tgz", + "integrity": "sha512-VYaJMxhr8B9BrCiNINUsuhaEe40YnG+AQBwcqUKO66lSVaI9I3A1iH/6EmEwRI8OYUg5Gt+4lLE7achg676lrg==", + "requires": { + "thirty-two": "^1.0.1", + "uniq": "^1.0.1" + } + }, "media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -1441,6 +1470,11 @@ "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.7.tgz", "integrity": "sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s=" }, + "named-queue": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/named-queue/-/named-queue-2.2.1.tgz", + "integrity": "sha1-GBRURVNZnVqeQD0N+pN6TODR5qc=" + }, "natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -2251,6 +2285,11 @@ "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=", "dev": true }, + "thirty-two": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/thirty-two/-/thirty-two-1.0.2.tgz", + "integrity": "sha1-TKL//AKlEpDSdEueP1V2k8prYno=" + }, "through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", @@ -2330,6 +2369,19 @@ "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.9.2.tgz", "integrity": "sha512-D39qtimx0c1fI3ya1Lnhk3E9nONswSKhnffBI0gME9C99fYOkNi04xs8K6pePLhvl1frbDemkaBQ5ikWllR2HQ==" }, + "underscore-keypath": { + "version": "0.0.22", + "resolved": "https://registry.npmjs.org/underscore-keypath/-/underscore-keypath-0.0.22.tgz", + "integrity": "sha1-SKUoOSu278QkvhyqVtpLX6zPJk0=", + "requires": { + "underscore": "*" + } + }, + "uniq": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/uniq/-/uniq-1.0.1.tgz", + "integrity": "sha1-sxxa6CVIRKOoKBVBzisEuGWnNP8=" + }, "unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", @@ -2343,6 +2395,15 @@ "punycode": "^2.1.0" } }, + "user-agents": { + "version": "1.0.559", + "resolved": "https://registry.npmjs.org/user-agents/-/user-agents-1.0.559.tgz", + "integrity": "sha512-HdAlNS3vDxOGMRwmv8or05xL96MV3CEwQhUSFTCRoOvTOEnWhTEBPAHRry/xZpVTTOtx77UHMal8YKcx6fs7Lg==", + "requires": { + "dot-json": "^1.2.0", + "lodash.clonedeep": "^4.5.0" + } + }, "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", diff --git a/addon/package.json b/addon/package.json index 895b729..f4ff4ea 100644 --- a/addon/package.json +++ b/addon/package.json @@ -11,13 +11,16 @@ "cache-manager": "^2.11.1", "cache-manager-mongodb": "^0.2.2", "express-rate-limit": "^5.1.1", + "named-queue": "^2.2.1", "needle": "^2.2.4", + "magnet-uri": "^5.1.7", "parse-torrent-title": "git://github.com/TheBeastLT/parse-torrent-title.git#afd4a374276420c13c52df8e3d07ae7699c46b60", "pg": "^7.8.2", "pg-hstore": "^2.3.2", "real-debrid-api": "git://github.com/TheBeastLT/node-real-debrid.git#935a5c23ae809edbcd2a111526a7f74d6767c50d", "sequelize": "^4.43.0", - "stremio-addon-sdk": "^1.6.1" + "stremio-addon-sdk": "^1.6.1", + "user-agents": "^1.0.559" }, "devDependencies": { "eslint": "^6.4.0", diff --git a/addon/serverless.js b/addon/serverless.js index e1b4c3f..a299351 100644 --- a/addon/serverless.js +++ b/addon/serverless.js @@ -4,6 +4,7 @@ const addonInterface = require('./addon'); const { manifest } = require('./lib/manifest'); const parseConfiguration = require('./lib/configuration'); const landingTemplate = require('./lib/landingTemplate'); +const realDebrid = require('./moch/realdebrid'); const router = getRouter(addonInterface); const limiter = rateLimit({ @@ -68,6 +69,20 @@ router.get('/:configuration/:resource/:type/:id.json', (req, res, next) => { }); }); +router.get('/realdebrid/:apiKey/:infoHash/:cachedFileIds/:fileIndex?', (req, res) => { + const { apiKey, infoHash, cachedFileIds, fileIndex } = req.params; + realDebrid.resolve(apiKey, infoHash, cachedFileIds, isNaN(fileIndex) ? undefined : parseInt(fileIndex)) + .then(url => { + res.writeHead(301, { Location: url }); + res.end(); + }) + .catch(error => { + console.log(error); + res.statusCode = 404; + res.end(); + }); +}); + module.exports = function (req, res) { router(req, res, function () { res.statusCode = 404;