From cf7b4411d2c03b3ad21f15bcb6928bf2efa39d08 Mon Sep 17 00:00:00 2001 From: TheBeastLT Date: Thu, 9 Apr 2020 13:35:11 +0200 Subject: [PATCH] [scraper] removes moch resolver from scraper --- package-lock.json | 122 +++++++++++++++++++++++++++++++++++++ package.json | 2 +- scraper/index.js | 15 ----- scraper/lib/cache.js | 21 ------- scraper/moch/realdebrid.js | 83 ------------------------- 5 files changed, 123 insertions(+), 120 deletions(-) delete mode 100644 scraper/moch/realdebrid.js diff --git a/package-lock.json b/package-lock.json index 3af81f9..b66438e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -998,6 +998,11 @@ "readable-stream": "^3.1.1" } }, + "http": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/http/-/http-0.0.0.tgz", + "integrity": "sha1-huYybSnF0Dnen6xYSkVon5KfT3I=" + }, "http-errors": { "version": "1.6.3", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", @@ -1934,6 +1939,11 @@ "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" }, + "querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" + }, "random-access-file": { "version": "2.1.4", "resolved": "https://registry.npmjs.org/random-access-file/-/random-access-file-2.1.4.tgz", @@ -2117,6 +2127,113 @@ "resolved": "https://registry.npmjs.org/rusha/-/rusha-0.8.13.tgz", "integrity": "sha1-mghOe4YLF7/zAVuSxnpqM2GRUTo=" }, + "rutracker-api-2": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/rutracker-api-2/-/rutracker-api-2-1.10.0.tgz", + "integrity": "sha512-wBsMQNpjNFV92lIUHwbQXd0Gn+5bnIH6JHfn7dN/nhcCHiprUH2bVntzZBHULWGYf5MokOaqQwgxBueg4QtADw==", + "requires": { + "bluebird": "^3.5.1", + "cheerio": "^0.19.0", + "http": "0.0.0", + "querystring": "^0.2.0", + "windows-1251": "^0.1.2" + }, + "dependencies": { + "cheerio": { + "version": "0.19.0", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-0.19.0.tgz", + "integrity": "sha1-dy5wFfLuKZZQltcepBdbdas1SSU=", + "requires": { + "css-select": "~1.0.0", + "dom-serializer": "~0.1.0", + "entities": "~1.1.1", + "htmlparser2": "~3.8.1", + "lodash": "^3.2.0" + } + }, + "css-select": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.0.0.tgz", + "integrity": "sha1-sRIcpRhI3SZOIkTQWM7iVN7rRLA=", + "requires": { + "boolbase": "~1.0.0", + "css-what": "1.0", + "domutils": "1.4", + "nth-check": "~1.0.0" + } + }, + "css-what": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-1.0.0.tgz", + "integrity": "sha1-18wt9FGAZm+Z0rFEYmOUaeAPc2w=" + }, + "domhandler": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.3.0.tgz", + "integrity": "sha1-LeWaCCLVAn+r/28DLCsloqir5zg=", + "requires": { + "domelementtype": "1" + } + }, + "domutils": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.4.3.tgz", + "integrity": "sha1-CGVRN5bGswYDGFDhdVFrr4C3Km8=", + "requires": { + "domelementtype": "1" + } + }, + "htmlparser2": { + "version": "3.8.3", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.8.3.tgz", + "integrity": "sha1-mWwosZFRaovoZQGn15dX5ccMEGg=", + "requires": { + "domelementtype": "1", + "domhandler": "2.3", + "domutils": "1.5", + "entities": "1.0", + "readable-stream": "1.1" + }, + "dependencies": { + "domutils": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz", + "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=", + "requires": { + "dom-serializer": "0", + "domelementtype": "1" + } + }, + "entities": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-1.0.0.tgz", + "integrity": "sha1-sph6o4ITR/zeZCsk/fyeT7cSvyY=" + } + } + }, + "lodash": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-3.10.1.tgz", + "integrity": "sha1-W/Rejkm6QYnhfUgnid/RW9FAt7Y=" + }, + "readable-stream": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + } + }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + } + } + }, "safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", @@ -2741,6 +2858,11 @@ "defaults": "^1.0.3" } }, + "windows-1251": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/windows-1251/-/windows-1251-0.1.2.tgz", + "integrity": "sha1-SIHY+QU9Dre1+l008PlNYN5DwtM=" + }, "wkx": { "version": "0.4.8", "resolved": "https://registry.npmjs.org/wkx/-/wkx-0.4.8.tgz", diff --git a/package.json b/package.json index 6725b31..8aacbc2 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,6 @@ "magnet-uri": "^5.1.7", "moment": "^2.24.0", "name-to-imdb": "^2.3.0", - "named-queue": "^2.2.1", "needle": "^2.2.4", "node-gzip": "^1.1.2", "node-schedule": "^1.3.2", @@ -38,6 +37,7 @@ "pg-hstore": "^2.3.2", "rarbg-api": "^1.1.3", "real-debrid-api": "git://github.com/TheBeastLT/node-real-debrid.git#935a5c23ae809edbcd2a111526a7f74d6767c50d", + "rutracker-api-2": "^1.10.0", "sequelize": "^5.21.5", "sugar-date": "^2.0.6", "torrent-stream": "^1.2.0", diff --git a/scraper/index.js b/scraper/index.js index 0ba0816..46299be 100644 --- a/scraper/index.js +++ b/scraper/index.js @@ -3,7 +3,6 @@ const express = require("express"); const server = express(); const schedule = require('node-schedule'); const { connect, getUpdateSeedersTorrents } = require('./lib/repository'); -const realDebrid = require('./moch/realdebrid'); const thepiratebayScraper = require('./scrapers/thepiratebay/thepiratebay_scraper'); const horribleSubsScraper = require('./scrapers/horriblesubs/horriblesubs_scraper'); const leetxScraper = require('./scrapers/1337x/1337x_scraper'); @@ -62,20 +61,6 @@ server.get('/', function (req, res) { res.sendStatus(200); }); -server.get('/realdebrid/:apiKey/:infoHash/:cachedFileIds/:fileIndex?', (req, res) => { - const { apiKey, infoHash, cachedFileIds, fileIndex } = req.params; - realDebrid.resolve(req.ip, apiKey, infoHash, cachedFileIds, isNaN(fileIndex) ? undefined : parseInt(fileIndex)) - .then(url => { - res.writeHead(301, { Location: url }); - res.end(); - }) - .catch(error => { - console.log(error); - res.statusCode = 404; - res.end(); - }); -}); - server.listen(process.env.PORT || 7000, async () => { await connect(); console.log('Scraper started'); diff --git a/scraper/lib/cache.js b/scraper/lib/cache.js index 381d308..60df89e 100644 --- a/scraper/lib/cache.js +++ b/scraper/lib/cache.js @@ -5,16 +5,10 @@ const GLOBAL_KEY_PREFIX = 'stremio-torrentio'; const IMDB_ID_PREFIX = `${GLOBAL_KEY_PREFIX}|imdb_id`; const KITSU_ID_PREFIX = `${GLOBAL_KEY_PREFIX}|kitsu_id`; const METADATA_PREFIX = `${GLOBAL_KEY_PREFIX}|metadata`; -const RESOLVED_URL_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|moch`; -const PROXY_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|proxy`; -const USER_AGENT_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|agent`; const TRACKERS_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|trackers`; const GLOBAL_TTL = process.env.METADATA_TTL || 7 * 24 * 60 * 60; // 7 days const MEMORY_TTL = process.env.METADATA_TTL || 2 * 60 * 60; // 2 hours -const RESOLVED_URL_TTL = 2 * 60; // 2 minutes -const PROXY_TTL = 60 * 60; // 60 minutes -const USER_AGENT_TTL = 2 * 24 * 60 * 60; // 2 days const TRACKERS_TTL = 2 * 24 * 60 * 60; // 2 days const MONGO_URI = process.env.MONGODB_URI; @@ -65,18 +59,6 @@ function cacheWrapMetadata(id, method) { return cacheWrap(memoryCache, `${METADATA_PREFIX}:${id}`, method, { ttl: MEMORY_TTL }); } -function cacheWrapResolvedUrl(id, method) { - return cacheWrap(memoryCache, `${RESOLVED_URL_KEY_PREFIX}:${id}`, method, { ttl: { RESOLVED_URL_TTL } }); -} - -function cacheWrapProxy(id, method) { - return cacheWrap(memoryCache, `${PROXY_KEY_PREFIX}:${id}`, method, { ttl: { PROXY_TTL } }); -} - -function cacheUserAgent(id, method) { - return cacheWrap(memoryCache, `${USER_AGENT_KEY_PREFIX}:${id}`, method, { ttl: { USER_AGENT_TTL } }); -} - function cacheTrackers(method) { return cacheWrap(memoryCache, `${TRACKERS_KEY_PREFIX}`, method, { ttl: { TRACKERS_TTL } }); } @@ -85,9 +67,6 @@ module.exports = { cacheWrapImdbId, cacheWrapKitsuId, cacheWrapMetadata, - cacheWrapResolvedUrl, - cacheWrapProxy, - cacheUserAgent, cacheTrackers }; diff --git a/scraper/moch/realdebrid.js b/scraper/moch/realdebrid.js deleted file mode 100644 index 037e6ab..0000000 --- a/scraper/moch/realdebrid.js +++ /dev/null @@ -1,83 +0,0 @@ -const { encode } = require('magnet-uri'); -const RealDebridClient = require('real-debrid-api'); -const namedQueue = require('named-queue'); -const { cacheWrapResolvedUrl, cacheWrapProxy, cacheUserAgent } = require('../lib/cache'); -const { getRandomProxy, getRandomUserAgent } = require('../lib/request_helper'); - -const unrestrictQueue = new namedQueue((task, callback) => task.method() - .then(result => callback(false, result)) - .catch((error => callback(error)))); - -async function resolve(ip, apiKey, infoHash, cachedFileIds, fileIndex) { - if (!apiKey || !infoHash || !cachedFileIds || !cachedFileIds.length) { - return Promise.reject("No valid parameters passed"); - } - const id = `${apiKey}_${infoHash}_${fileIndex}`; - const method = () => cacheWrapResolvedUrl(id, () => _unrestrict(ip, apiKey, infoHash, cachedFileIds, fileIndex)); - - return new Promise(((resolve, reject) => { - unrestrictQueue.push({ id, method }, (error, result) => result ? resolve(result) : reject(error)); - })); -} - -async function _unrestrict(ip, apiKey, infoHash, cachedFileIds, fileIndex) { - console.log(`Unrestricting ${infoHash} [${fileIndex}]`); - const options = await getDefaultOptions(ip); - const RD = new RealDebridClient(apiKey, options); - const torrentId = await _createOrFindTorrentId(RD, infoHash, cachedFileIds); - if (torrentId) { - const info = await RD.torrents.info(torrentId); - const targetFile = info.files.find(file => file.id === fileIndex + 1) - || info.files.filter(file => file.selected).sort((a, b) => b.bytes - a.bytes)[0]; - const selectedFiles = info.files.filter(file => file.selected); - const fileLink = info.links.length === 1 - ? info.links[0] - : info.links[selectedFiles.indexOf(targetFile)]; - const unrestrictedLink = await _unrestrictLink(RD, fileLink); - console.log(`Unrestricted ${infoHash} [${fileIndex}] to ${unrestrictedLink}`); - return unrestrictedLink; - } - return Promise.reject("Failed adding torrent"); -} - -async function _createOrFindTorrentId(RD, infoHash, cachedFileIds) { - return RD.torrents.get(0, 1) - .then(torrents => torrents.find(torrent => torrent.hash.toLowerCase() === infoHash)) - .then(torrent => torrent && torrent.id || Promise.reject('No recent torrent found')) - .catch((error) => RD.torrents.addMagnet(encode({ infoHash })) - .then(response => RD.torrents.selectFiles(response.id, cachedFileIds) - .then((() => response.id)))) - .catch(error => { - console.warn('Failed RealDebrid torrent retrieval', error); - return undefined; - }); -} - -async function _unrestrictLink(RD, link) { - if (!link || !link.length) { - return Promise.reject("No available links found"); - } - return RD.unrestrict.link(link) - .then(unrestrictedLink => unrestrictedLink.download); - // .then(unrestrictedLink => RD.streaming.transcode(unrestrictedLink.id)) - // .then(transcodedLink => { - // const url = transcodedLink.apple && transcodedLink.apple.full - // || transcodedLink[Object.keys(transcodedLink)[0]].full; - // console.log(`Unrestricted ${link} to ${url}`); - // return url; - // }); -} - -async function getDefaultOptions(ip) { - const userAgent = await cacheUserAgent(ip, () => getRandomUserAgent()).catch(() => getRandomUserAgent()); - const proxy = await cacheWrapProxy('realdebrid', () => getRandomProxy()).catch(() => getRandomProxy()); - - return { - proxy: proxy, - headers: { - 'User-Agent': userAgent - } - }; -} - -module.exports = { resolve }; \ No newline at end of file