diff --git a/package-lock.json b/package-lock.json index 6435ef5..3af81f9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -192,11 +192,6 @@ "ms": "2.0.0" } }, - "minimist": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", - "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" - }, "ms": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", @@ -401,6 +396,31 @@ "lodash.some": "^4.4.0" } }, + "chrome-dgram": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/chrome-dgram/-/chrome-dgram-3.0.4.tgz", + "integrity": "sha512-G8rOANSvSRC4hGny/K/ec1gXtNuZGzryFeoev49u0J4g/qws7H25vMKQlbD9izuedFVHwXFTdKQG62Tf/7Cmwg==", + "requires": { + "inherits": "^2.0.1", + "run-series": "^1.1.2" + } + }, + "chrome-dns": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/chrome-dns/-/chrome-dns-1.0.1.tgz", + "integrity": "sha512-HqsYJgIc8ljJJOqOzLphjAs79EUuWSX3nzZi2LNkzlw3GIzAeZbaSektC8iT/tKvLqZq8yl1GJu5o6doA4TRbg==", + "requires": { + "chrome-net": "^3.3.2" + } + }, + "chrome-net": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/chrome-net/-/chrome-net-3.3.3.tgz", + "integrity": "sha512-11jL8+Ogna8M5TEdyalE8IG6cpaFEU3YcaxAj3YjZKjRM/PeT70pZbrUY+xoGwqiEJZwJE4Td2CvGxUvS9ytKQ==", + "requires": { + "inherits": "^2.0.1" + } + }, "cli-cursor": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz", @@ -864,11 +884,11 @@ }, "dependencies": { "mkdirp": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", - "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.4.tgz", + "integrity": "sha512-iG9AK/dJLtJ0XNgTuDbSyNS3zECqDlAhnQW4CsNxBG3LQJBbHmRX1egw39DmtOdCAqY+dKXV+sgPgilNWUKMVw==", "requires": { - "minimist": "0.0.8" + "minimist": "^1.2.5" } } } @@ -897,9 +917,9 @@ } }, "glob": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", - "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==", + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", + "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -1037,17 +1057,17 @@ "integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo=" }, "ip-set": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/ip-set/-/ip-set-1.0.1.tgz", - "integrity": "sha1-Yztm0L1sjQ3paNBTJjyRINO2cn4=", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/ip-set/-/ip-set-1.0.2.tgz", + "integrity": "sha512-Mb6kv78bTi4RNAIIWL8Bbre7hXOR2pNUi3j8FaQkLaitf/ZWxkq3/iIwXNYk2ACO3IMfdVdQrOkUtwZblO7uBA==", "requires": { "ip": "^1.1.3" } }, "ipaddr.js": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.0.tgz", - "integrity": "sha512-M4Sjn6N/+O6/IXSJseKqHoFc+5FdGJ22sXqnjTpdZweHK64MzEPAyQZyEU3R/KRv2GLoa7nNtg/C2Ev6m7z+eA==" + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==" }, "is-bluebird": { "version": "1.0.2", @@ -1149,13 +1169,14 @@ } }, "k-rpc-socket": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/k-rpc-socket/-/k-rpc-socket-1.8.0.tgz", - "integrity": "sha512-f/9TynsO8YYjZ6JjNNtSSH7CJcIHcio1buy3zqByGxb/GX8AWLdL6FZEWTrN8V3/J7W4/E0ZTQQ+Jt2rVq7ELg==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/k-rpc-socket/-/k-rpc-socket-1.11.1.tgz", + "integrity": "sha512-8xtA8oqbZ6v1Niryp2/g4GxW16EQh5MvrUylQoOG+zcrDff5CKttON2XUXvMwlIHq4/2zfPVFiinAccJ+WhxoA==", "requires": { "bencode": "^2.0.0", - "buffer-equals": "^1.0.4", - "safe-buffer": "^5.1.1" + "chrome-dgram": "^3.0.2", + "chrome-dns": "^1.0.0", + "chrome-net": "^3.3.2" }, "dependencies": { "bencode": { @@ -1391,15 +1412,20 @@ } }, "minimist": { - "version": "0.0.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", - "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" }, "mkdirp": { "version": "0.3.5", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.3.5.tgz", "integrity": "sha1-3j5fiWHIjHh+4TaN+EmsRBPsqNc=" }, + "mkdirp-classic": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.2.tgz", + "integrity": "sha512-ejdnDQcR75gwknmMw/tx02AuRs8jCtqFoFqDZMjiNxsu85sRIJVXDKHuLYvUUPRBUtV2FpSZa9bL1BUa3BdR2g==" + }, "moment": { "version": "2.24.0", "resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz", @@ -1868,9 +1894,9 @@ } }, "process-nextick-args": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", - "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==" + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, "proxy-addr": { "version": "2.0.4", @@ -1909,28 +1935,18 @@ "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" }, "random-access-file": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/random-access-file/-/random-access-file-2.1.0.tgz", - "integrity": "sha512-W2hY3DboLETMclybTVzyqCNVKx1MjqUwZPzkpkkMD2t9mbGEtkV2SKWPqAJ/FTrAtnWB7aGwl0NDUS82da0KdQ==", + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/random-access-file/-/random-access-file-2.1.4.tgz", + "integrity": "sha512-WAcBP5iLhg1pbjZA40WyMenjK7c5gJUY6Pi5HJ3fLJCeVFNSZv3juf20yFMKxBdvcX5GKbX/HZSfFzlLBdGTdQ==", "requires": { - "mkdirp": "^0.5.1", + "mkdirp-classic": "^0.5.2", "random-access-storage": "^1.1.1" - }, - "dependencies": { - "mkdirp": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", - "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", - "requires": { - "minimist": "0.0.8" - } - } } }, "random-access-storage": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/random-access-storage/-/random-access-storage-1.3.0.tgz", - "integrity": "sha512-pdS9Mcb9TB7oICypPRALlheaSuszuAKmLVEPKJMuYor7R/zDuHh5ALuQoS+ox31XRwQUL+tDwWH2GPdyspwelA==", + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/random-access-storage/-/random-access-storage-1.4.1.tgz", + "integrity": "sha512-DbCc2TIzOxPaHF6KCbr8zLtiYOJQQQCBHUVNHV/SckUQobCBB2YkDtbLdxGnPwPNpJfEyMWxDAm36A2xkbxxtw==", "requires": { "inherits": "^2.0.3" } @@ -1980,9 +1996,9 @@ } }, "re-emitter": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/re-emitter/-/re-emitter-1.1.3.tgz", - "integrity": "sha1-+p4xn/3u6zWycpbvDz03TawvUqc=" + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/re-emitter/-/re-emitter-1.1.4.tgz", + "integrity": "sha512-C0SIXdXDSus2yqqvV7qifnb4NoWP7mEBXJq3axci301mXHCZb8Djwm4hrEZo4UeXRaEnfjH98uQ8EBppk2oNWA==" }, "readable-stream": { "version": "3.2.0", @@ -2079,9 +2095,9 @@ } }, "rimraf": { - "version": "2.6.3", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", - "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", "requires": { "glob": "^7.1.3" } @@ -2276,9 +2292,9 @@ "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, "readable-stream": { - "version": "2.3.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", - "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", "requires": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", @@ -2339,9 +2355,9 @@ "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, "readable-stream": { - "version": "2.3.6", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", - "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", "requires": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", @@ -2488,9 +2504,9 @@ "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" }, "thunky": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/thunky/-/thunky-1.0.3.tgz", - "integrity": "sha512-YwT8pjmNcAXBZqrubu22P4FYsh2D4dxRmnWBOL8Jk8bUcRUtc5326kx32tuTmFDAZtLOGEVNl8POAR8j896Iow==" + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/thunky/-/thunky-1.1.0.tgz", + "integrity": "sha512-eHY7nBftgThBqOyHGVN+l8gF0BucP09fMo0oO/Lb0w1OF80dJv+lDVpXG60WMQvkcxAkNybKsrEIE3ZtKGmPrA==" }, "toposort-class": { "version": "1.0.1", @@ -2532,9 +2548,9 @@ "integrity": "sha512-ElXPyXKKG73o+uziHJ8qlYE9EuyDVxnK2zWL+pW/2bma7RsLpSwFFIJAb8Qui7/tel2hsHQW1z3zBnfQNREpWA==" }, "torrent-stream": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/torrent-stream/-/torrent-stream-1.1.0.tgz", - "integrity": "sha512-yjKU8l985+/D2CdnAR2+pEpyMX13rlQ1kNYik34EHxcul7BjifW5sMizT+u47suOeBTji3lHBA7eZGhBjpnM6g==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/torrent-stream/-/torrent-stream-1.2.0.tgz", + "integrity": "sha512-piQP9/wrXRYvEUAsmdu+fy2D2WPwU7BcsflTnKLsZsrUDBT/Y1INhuYU7Fw9PqEm+RF7QAa2gD8nMmvfb7QomA==", "requires": { "bitfield": "^0.1.0", "bncode": "^0.5.2", diff --git a/package.json b/package.json index 7c03f53..6725b31 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "real-debrid-api": "git://github.com/TheBeastLT/node-real-debrid.git#935a5c23ae809edbcd2a111526a7f74d6767c50d", "sequelize": "^5.21.5", "sugar-date": "^2.0.6", - "torrent-stream": "^1.1.0", + "torrent-stream": "^1.2.0", "user-agents": "^1.0.559" } } diff --git a/scraper/lib/cache.js b/scraper/lib/cache.js index f904c28..5c6259a 100644 --- a/scraper/lib/cache.js +++ b/scraper/lib/cache.js @@ -7,33 +7,18 @@ const KITSU_ID_PREFIX = `${GLOBAL_KEY_PREFIX}|kitsu_id`; const METADATA_PREFIX = `${GLOBAL_KEY_PREFIX}|metadata`; const RESOLVED_URL_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|moch`; const PROXY_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|proxy`; -const TORRENT_FILES_KEY_PREFIX = `stremio-tpb|files`; +const USER_AGENT_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|agent`; const GLOBAL_TTL = process.env.METADATA_TTL || 7 * 24 * 60 * 60; // 7 days const MEMORY_TTL = process.env.METADATA_TTL || 2 * 60 * 60; // 2 hours const RESOLVED_URL_TTL = 2 * 60; // 2 minutes -const PROXY_TTL = 8 * 60 * 60; // 8 hours +const PROXY_TTL = 60 * 60; // 60 minutes +const USER_AGENT_TTL = 2 * 24 * 60 * 60; // 2 days const MONGO_URI = process.env.MONGODB_URI; const memoryCache = initiateMemoryCache(); const remoteCache = initiateRemoteCache(); -const torrentFilesCache = initiateTorrentFilesCache(); - -function initiateTorrentFilesCache() { - if (MONGO_URI) { - return cacheManager.caching({ - store: mangodbStore, - uri: MONGO_URI, - options: { - collection: 'cacheManager', - useUnifiedTopology: true, - }, - ttl: GLOBAL_TTL, - ignoreCacheErrors: true - }); - } -} function initiateRemoteCache() { if (MONGO_URI) { @@ -62,16 +47,6 @@ function initiateMemoryCache() { }); } -function retrieveTorrentFiles(infoHash) { - return torrentFilesCache.get(`${TORRENT_FILES_KEY_PREFIX}:${infoHash}`) - .then((results) => { - if (!results) { - throw new Error('No cached files found'); - } - return results; - }); -} - function cacheWrap(cache, key, method, options) { return cache.wrap(key, method, options); } @@ -92,16 +67,20 @@ function cacheWrapResolvedUrl(id, method) { return cacheWrap(memoryCache, `${RESOLVED_URL_KEY_PREFIX}:${id}`, method, { ttl: { RESOLVED_URL_TTL } }); } -function cacheWrapOptions(id, method) { +function cacheWrapProxy(id, method) { return cacheWrap(memoryCache, `${PROXY_KEY_PREFIX}:${id}`, method, { ttl: { PROXY_TTL } }); } +function cacheUserAgent(id, method) { + return cacheWrap(memoryCache, `${USER_AGENT_KEY_PREFIX}:${id}`, method, { ttl: { USER_AGENT_TTL } }); +} + module.exports = { cacheWrapImdbId, cacheWrapKitsuId, cacheWrapMetadata, - retrieveTorrentFiles, cacheWrapResolvedUrl, - cacheWrapOptions + cacheWrapProxy, + cacheUserAgent }; diff --git a/scraper/lib/repository.js b/scraper/lib/repository.js index 6e34462..1a79f94 100644 --- a/scraper/lib/repository.js +++ b/scraper/lib/repository.js @@ -28,7 +28,8 @@ const Torrent = database.define('torrent', seeders: { type: Sequelize.SMALLINT }, trackers: { type: Sequelize.STRING(4096) }, languages: { type: Sequelize.STRING(256) }, - resolution: { type: Sequelize.STRING(16) } + resolution: { type: Sequelize.STRING(16) }, + reviewed: { type: Sequelize.BOOLEAN, allowNull: false, defaultValue: false } } ); @@ -118,7 +119,7 @@ function getTorrentsBasedOnTitle(titleQuery, type) { function getTorrentsWithoutSize() { return Torrent.findAll({ where: literal( - 'exists (select 1 from files where files."infoHash" = torrent."infoHash" and files.size = 300000000)'), + 'exists (select 1 from files where files."infoHash" = torrent."infoHash" and files.size = 300000000) and random() < 0.01'), order: [ ['seeders', 'DESC'] ], diff --git a/scraper/lib/torrent.js b/scraper/lib/torrent.js index ac86cc7..43e1896 100644 --- a/scraper/lib/torrent.js +++ b/scraper/lib/torrent.js @@ -38,7 +38,7 @@ module.exports.updateTorrentSize = function (torrent) { .then(result => ({ ...torrent, size: result.size, files: result.files })); }; -module.exports.sizeAndFiles = torrent => filesAndSizeFromTorrentStream(torrent, 20000); +module.exports.sizeAndFiles = torrent => filesAndSizeFromTorrentStream(torrent, 30000); module.exports.torrentFiles = function (torrent) { return getFilesFromObject(torrent) diff --git a/scraper/moch/realdebrid.js b/scraper/moch/realdebrid.js index ec8e8ad..037e6ab 100644 --- a/scraper/moch/realdebrid.js +++ b/scraper/moch/realdebrid.js @@ -1,7 +1,7 @@ const { encode } = require('magnet-uri'); const RealDebridClient = require('real-debrid-api'); const namedQueue = require('named-queue'); -const { cacheWrapResolvedUrl, cacheWrapOptions } = require('../lib/cache'); +const { cacheWrapResolvedUrl, cacheWrapProxy, cacheUserAgent } = require('../lib/cache'); const { getRandomProxy, getRandomUserAgent } = require('../lib/request_helper'); const unrestrictQueue = new namedQueue((task, callback) => task.method() @@ -68,14 +68,16 @@ async function _unrestrictLink(RD, link) { // }); } -function getDefaultOptions(ip) { - const generateOptions = () => ({ - proxy: getRandomProxy(), +async function getDefaultOptions(ip) { + const userAgent = await cacheUserAgent(ip, () => getRandomUserAgent()).catch(() => getRandomUserAgent()); + const proxy = await cacheWrapProxy('realdebrid', () => getRandomProxy()).catch(() => getRandomProxy()); + + return { + proxy: proxy, headers: { - 'User-Agent': getRandomUserAgent() + 'User-Agent': userAgent } - }); - return cacheWrapOptions(ip, generateOptions).catch(() => generateOptions()); + }; } module.exports = { resolve }; \ No newline at end of file