tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] use same cached proxy for rd requests

Browse Source
This commit is contained in:
TheBeastLT
2020-04-04 16:52:12 +02:00
parent 760c577ca7
commit 2eefdc76ec
6 changed files with 104 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

144
package-lock.json generated
View File

@@ -192,11 +192,6 @@
"ms": "2.0.0"
}
},
"minimist": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz",
"integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ="
},
"ms": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
@@ -401,6 +396,31 @@
"lodash.some": "^4.4.0"
}
},
"chrome-dgram": {
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/chrome-dgram/-/chrome-dgram-3.0.4.tgz",
"integrity": "sha512-G8rOANSvSRC4hGny/K/ec1gXtNuZGzryFeoev49u0J4g/qws7H25vMKQlbD9izuedFVHwXFTdKQG62Tf/7Cmwg==",
"requires": {
"inherits": "^2.0.1",
"run-series": "^1.1.2"
}
},
"chrome-dns": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/chrome-dns/-/chrome-dns-1.0.1.tgz",
"integrity": "sha512-HqsYJgIc8ljJJOqOzLphjAs79EUuWSX3nzZi2LNkzlw3GIzAeZbaSektC8iT/tKvLqZq8yl1GJu5o6doA4TRbg==",
"requires": {
"chrome-net": "^3.3.2"
}
},
"chrome-net": {
"version": "3.3.3",
"resolved": "https://registry.npmjs.org/chrome-net/-/chrome-net-3.3.3.tgz",
"integrity": "sha512-11jL8+Ogna8M5TEdyalE8IG6cpaFEU3YcaxAj3YjZKjRM/PeT70pZbrUY+xoGwqiEJZwJE4Td2CvGxUvS9ytKQ==",
"requires": {
"inherits": "^2.0.1"
}
},
"cli-cursor": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz",
@@ -864,11 +884,11 @@
},
"dependencies": {
"mkdirp": {
"version": "0.5.1",
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
"integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=",
"version": "0.5.4",
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.4.tgz",
"integrity": "sha512-iG9AK/dJLtJ0XNgTuDbSyNS3zECqDlAhnQW4CsNxBG3LQJBbHmRX1egw39DmtOdCAqY+dKXV+sgPgilNWUKMVw==",
"requires": {
"minimist": "0.0.8"
"minimist": "^1.2.5"
}
}
}
@@ -897,9 +917,9 @@
}
},
"glob": {
"version": "7.1.3",
"resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz",
"integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==",
"version": "7.1.6",
"resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz",
"integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==",
"requires": {
"fs.realpath": "^1.0.0",
"inflight": "^1.0.4",
@@ -1037,17 +1057,17 @@
"integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo="
},
"ip-set": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/ip-set/-/ip-set-1.0.1.tgz",
"integrity": "sha1-Yztm0L1sjQ3paNBTJjyRINO2cn4=",
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/ip-set/-/ip-set-1.0.2.tgz",
"integrity": "sha512-Mb6kv78bTi4RNAIIWL8Bbre7hXOR2pNUi3j8FaQkLaitf/ZWxkq3/iIwXNYk2ACO3IMfdVdQrOkUtwZblO7uBA==",
"requires": {
"ip": "^1.1.3"
}
},
"ipaddr.js": {
"version": "1.9.0",
"resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.0.tgz",
"integrity": "sha512-M4Sjn6N/+O6/IXSJseKqHoFc+5FdGJ22sXqnjTpdZweHK64MzEPAyQZyEU3R/KRv2GLoa7nNtg/C2Ev6m7z+eA=="
"version": "1.9.1",
"resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz",
"integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="
},
"is-bluebird": {
"version": "1.0.2",
@@ -1149,13 +1169,14 @@
}
},
"k-rpc-socket": {
"version": "1.8.0",
"resolved": "https://registry.npmjs.org/k-rpc-socket/-/k-rpc-socket-1.8.0.tgz",
"integrity": "sha512-f/9TynsO8YYjZ6JjNNtSSH7CJcIHcio1buy3zqByGxb/GX8AWLdL6FZEWTrN8V3/J7W4/E0ZTQQ+Jt2rVq7ELg==",
"version": "1.11.1",
"resolved": "https://registry.npmjs.org/k-rpc-socket/-/k-rpc-socket-1.11.1.tgz",
"integrity": "sha512-8xtA8oqbZ6v1Niryp2/g4GxW16EQh5MvrUylQoOG+zcrDff5CKttON2XUXvMwlIHq4/2zfPVFiinAccJ+WhxoA==",
"requires": {
"bencode": "^2.0.0",
"buffer-equals": "^1.0.4",
"safe-buffer": "^5.1.1"
"chrome-dgram": "^3.0.2",
"chrome-dns": "^1.0.0",
"chrome-net": "^3.3.2"
},
"dependencies": {
"bencode": {
@@ -1391,15 +1412,20 @@
}
},
"minimist": {
"version": "0.0.8",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz",
"integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0="
"version": "1.2.5",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz",
"integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw=="
},
"mkdirp": {
"version": "0.3.5",
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.3.5.tgz",
"integrity": "sha1-3j5fiWHIjHh+4TaN+EmsRBPsqNc="
},
"mkdirp-classic": {
"version": "0.5.2",
"resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.2.tgz",
"integrity": "sha512-ejdnDQcR75gwknmMw/tx02AuRs8jCtqFoFqDZMjiNxsu85sRIJVXDKHuLYvUUPRBUtV2FpSZa9bL1BUa3BdR2g=="
},
"moment": {
"version": "2.24.0",
"resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz",
@@ -1868,9 +1894,9 @@
}
},
"process-nextick-args": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz",
"integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw=="
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
},
"proxy-addr": {
"version": "2.0.4",
@@ -1909,28 +1935,18 @@
"integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA=="
},
"random-access-file": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/random-access-file/-/random-access-file-2.1.0.tgz",
"integrity": "sha512-W2hY3DboLETMclybTVzyqCNVKx1MjqUwZPzkpkkMD2t9mbGEtkV2SKWPqAJ/FTrAtnWB7aGwl0NDUS82da0KdQ==",
"version": "2.1.4",
"resolved": "https://registry.npmjs.org/random-access-file/-/random-access-file-2.1.4.tgz",
"integrity": "sha512-WAcBP5iLhg1pbjZA40WyMenjK7c5gJUY6Pi5HJ3fLJCeVFNSZv3juf20yFMKxBdvcX5GKbX/HZSfFzlLBdGTdQ==",
"requires": {
"mkdirp": "^0.5.1",
"mkdirp-classic": "^0.5.2",
"random-access-storage": "^1.1.1"
},
"dependencies": {
"mkdirp": {
"version": "0.5.1",
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
"integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=",
"requires": {
"minimist": "0.0.8"
}
}
}
},
"random-access-storage": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/random-access-storage/-/random-access-storage-1.3.0.tgz",
"integrity": "sha512-pdS9Mcb9TB7oICypPRALlheaSuszuAKmLVEPKJMuYor7R/zDuHh5ALuQoS+ox31XRwQUL+tDwWH2GPdyspwelA==",
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/random-access-storage/-/random-access-storage-1.4.1.tgz",
"integrity": "sha512-DbCc2TIzOxPaHF6KCbr8zLtiYOJQQQCBHUVNHV/SckUQobCBB2YkDtbLdxGnPwPNpJfEyMWxDAm36A2xkbxxtw==",
"requires": {
"inherits": "^2.0.3"
}
@@ -1980,9 +1996,9 @@
}
},
"re-emitter": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/re-emitter/-/re-emitter-1.1.3.tgz",
"integrity": "sha1-+p4xn/3u6zWycpbvDz03TawvUqc="
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/re-emitter/-/re-emitter-1.1.4.tgz",
"integrity": "sha512-C0SIXdXDSus2yqqvV7qifnb4NoWP7mEBXJq3axci301mXHCZb8Djwm4hrEZo4UeXRaEnfjH98uQ8EBppk2oNWA=="
},
"readable-stream": {
"version": "3.2.0",
@@ -2079,9 +2095,9 @@
}
},
"rimraf": {
"version": "2.6.3",
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz",
"integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==",
"version": "2.7.1",
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz",
"integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==",
"requires": {
"glob": "^7.1.3"
}
@@ -2276,9 +2292,9 @@
"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
},
"readable-stream": {
"version": "2.3.6",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz",
"integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==",
"version": "2.3.7",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
"integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
"requires": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.3",
@@ -2339,9 +2355,9 @@
"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
},
"readable-stream": {
"version": "2.3.6",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz",
"integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==",
"version": "2.3.7",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
"integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
"requires": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.3",
@@ -2488,9 +2504,9 @@
"integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU="
},
"thunky": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/thunky/-/thunky-1.0.3.tgz",
"integrity": "sha512-YwT8pjmNcAXBZqrubu22P4FYsh2D4dxRmnWBOL8Jk8bUcRUtc5326kx32tuTmFDAZtLOGEVNl8POAR8j896Iow=="
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/thunky/-/thunky-1.1.0.tgz",
"integrity": "sha512-eHY7nBftgThBqOyHGVN+l8gF0BucP09fMo0oO/Lb0w1OF80dJv+lDVpXG60WMQvkcxAkNybKsrEIE3ZtKGmPrA=="
},
"toposort-class": {
"version": "1.0.1",
@@ -2532,9 +2548,9 @@
"integrity": "sha512-ElXPyXKKG73o+uziHJ8qlYE9EuyDVxnK2zWL+pW/2bma7RsLpSwFFIJAb8Qui7/tel2hsHQW1z3zBnfQNREpWA=="
},
"torrent-stream": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/torrent-stream/-/torrent-stream-1.1.0.tgz",
"integrity": "sha512-yjKU8l985+/D2CdnAR2+pEpyMX13rlQ1kNYik34EHxcul7BjifW5sMizT+u47suOeBTji3lHBA7eZGhBjpnM6g==",
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/torrent-stream/-/torrent-stream-1.2.0.tgz",
"integrity": "sha512-piQP9/wrXRYvEUAsmdu+fy2D2WPwU7BcsflTnKLsZsrUDBT/Y1INhuYU7Fw9PqEm+RF7QAa2gD8nMmvfb7QomA==",
"requires": {
"bitfield": "^0.1.0",
"bncode": "^0.5.2",

2
package.json
View File

@@ -40,7 +40,7 @@
"real-debrid-api": "git://github.com/TheBeastLT/node-real-debrid.git#935a5c23ae809edbcd2a111526a7f74d6767c50d",
"sequelize": "^5.21.5",
"sugar-date": "^2.0.6",
"torrent-stream": "^1.1.0",
"torrent-stream": "^1.2.0",
"user-agents": "^1.0.559"
}
}

41
scraper/lib/cache.js
View File

@@ -7,33 +7,18 @@ const KITSU_ID_PREFIX = `${GLOBAL_KEY_PREFIX}|kitsu_id`;
const METADATA_PREFIX = `${GLOBAL_KEY_PREFIX}|metadata`;
const RESOLVED_URL_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|moch`;
const PROXY_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|proxy`;
const TORRENT_FILES_KEY_PREFIX = `stremio-tpb|files`;
const USER_AGENT_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|agent`;
const GLOBAL_TTL = process.env.METADATA_TTL || 7 * 24 * 60 * 60; // 7 days
const MEMORY_TTL = process.env.METADATA_TTL || 2 * 60 * 60; // 2 hours
const RESOLVED_URL_TTL = 2 * 60; // 2 minutes
const PROXY_TTL = 8 * 60 * 60; // 8 hours
const PROXY_TTL = 60 * 60; // 60 minutes
const USER_AGENT_TTL = 2 * 24 * 60 * 60; // 2 days
const MONGO_URI = process.env.MONGODB_URI;
const memoryCache = initiateMemoryCache();
const remoteCache = initiateRemoteCache();
const torrentFilesCache = initiateTorrentFilesCache();
function initiateTorrentFilesCache() {
if (MONGO_URI) {
return cacheManager.caching({
store: mangodbStore,
uri: MONGO_URI,
options: {
collection: 'cacheManager',
useUnifiedTopology: true,
},
ttl: GLOBAL_TTL,
ignoreCacheErrors: true
});
}
}
function initiateRemoteCache() {
if (MONGO_URI) {
@@ -62,16 +47,6 @@ function initiateMemoryCache() {
});
}
function retrieveTorrentFiles(infoHash) {
return torrentFilesCache.get(`${TORRENT_FILES_KEY_PREFIX}:${infoHash}`)
.then((results) => {
if (!results) {
throw new Error('No cached files found');
}
return results;
});
}
function cacheWrap(cache, key, method, options) {
return cache.wrap(key, method, options);
}
@@ -92,16 +67,20 @@ function cacheWrapResolvedUrl(id, method) {
return cacheWrap(memoryCache, `${RESOLVED_URL_KEY_PREFIX}:${id}`, method, { ttl: { RESOLVED_URL_TTL } });
}
function cacheWrapOptions(id, method) {
function cacheWrapProxy(id, method) {
return cacheWrap(memoryCache, `${PROXY_KEY_PREFIX}:${id}`, method, { ttl: { PROXY_TTL } });
}
function cacheUserAgent(id, method) {
return cacheWrap(memoryCache, `${USER_AGENT_KEY_PREFIX}:${id}`, method, { ttl: { USER_AGENT_TTL } });
}
module.exports = {
cacheWrapImdbId,
cacheWrapKitsuId,
cacheWrapMetadata,
retrieveTorrentFiles,
cacheWrapResolvedUrl,
cacheWrapOptions
cacheWrapProxy,
cacheUserAgent
};

5
scraper/lib/repository.js
View File

@@ -28,7 +28,8 @@ const Torrent = database.define('torrent',
seeders: { type: Sequelize.SMALLINT },
trackers: { type: Sequelize.STRING(4096) },
languages: { type: Sequelize.STRING(256) },
resolution: { type: Sequelize.STRING(16) }
resolution: { type: Sequelize.STRING(16) },
reviewed: { type: Sequelize.BOOLEAN, allowNull: false, defaultValue: false }
}
);
@@ -118,7 +119,7 @@ function getTorrentsBasedOnTitle(titleQuery, type) {
function getTorrentsWithoutSize() {
return Torrent.findAll({
where: literal(
'exists (select 1 from files where files."infoHash" = torrent."infoHash" and files.size = 300000000)'),
'exists (select 1 from files where files."infoHash" = torrent."infoHash" and files.size = 300000000) and random() < 0.01'),
order: [
['seeders', 'DESC']
],

2
scraper/lib/torrent.js
View File

@@ -38,7 +38,7 @@ module.exports.updateTorrentSize = function (torrent) {
.then(result => ({ ...torrent, size: result.size, files: result.files }));
};
module.exports.sizeAndFiles = torrent => filesAndSizeFromTorrentStream(torrent, 20000);
module.exports.sizeAndFiles = torrent => filesAndSizeFromTorrentStream(torrent, 30000);
module.exports.torrentFiles = function (torrent) {
return getFilesFromObject(torrent)

16
scraper/moch/realdebrid.js
View File

@@ -1,7 +1,7 @@
const { encode } = require('magnet-uri');
const RealDebridClient = require('real-debrid-api');
const namedQueue = require('named-queue');
const { cacheWrapResolvedUrl, cacheWrapOptions } = require('../lib/cache');
const { cacheWrapResolvedUrl, cacheWrapProxy, cacheUserAgent } = require('../lib/cache');
const { getRandomProxy, getRandomUserAgent } = require('../lib/request_helper');
const unrestrictQueue = new namedQueue((task, callback) => task.method()
@@ -68,14 +68,16 @@ async function _unrestrictLink(RD, link) {
// });
}
function getDefaultOptions(ip) {
const generateOptions = () => ({
proxy: getRandomProxy(),
async function getDefaultOptions(ip) {
const userAgent = await cacheUserAgent(ip, () => getRandomUserAgent()).catch(() => getRandomUserAgent());
const proxy = await cacheWrapProxy('realdebrid', () => getRandomProxy()).catch(() => getRandomProxy());
return {
proxy: proxy,
headers: {
'User-Agent': getRandomUserAgent()
'User-Agent': userAgent
}
});
return cacheWrapOptions(ip, generateOptions).catch(() => generateOptions());
};
}
module.exports = { resolve };