tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] cache rd options per ip

Browse Source
This commit is contained in:
TheBeastLT
2020-04-03 13:27:26 +02:00
parent d2eb8d04c0
commit 33d9921179
4 changed files with 25 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
scraper/index.js
View File

@@ -50,7 +50,7 @@ server.get('/', function (req, res) {
server.get('/realdebrid/:apiKey/:infoHash/:cachedFileIds/:fileIndex?', (req, res) => {
const { apiKey, infoHash, cachedFileIds, fileIndex } = req.params;
realDebrid.resolve(apiKey, infoHash, cachedFileIds, isNaN(fileIndex) ? undefined : parseInt(fileIndex))
realDebrid.resolve(req.ip, apiKey, infoHash, cachedFileIds, isNaN(fileIndex) ? undefined : parseInt(fileIndex))
.then(url => {
res.writeHead(301, { Location: url });
res.end();

15
scraper/lib/cache.js
View File

@@ -6,10 +6,12 @@ const IMDB_ID_PREFIX = `${GLOBAL_KEY_PREFIX}|imdb_id`;
const KITSU_ID_PREFIX = `${GLOBAL_KEY_PREFIX}|kitsu_id`;
const METADATA_PREFIX = `${GLOBAL_KEY_PREFIX}|metadata`;
const RESOLVED_URL_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|moch`;
const PROXY_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|proxy`;
const TORRENT_FILES_KEY_PREFIX = `stremio-tpb|files`;
const GLOBAL_TTL = process.env.METADATA_TTL || 7 * 24 * 60 * 60; // 7 days
const MEMORY_TTL = process.env.METADATA_TTL || 2 * 60 * 60; // 2 hours
const PROXY_TTL = 8 * 60 * 60; // 8 hours
const MONGO_URI = process.env.MONGODB_URI;
@@ -89,5 +91,16 @@ function cacheWrapResolvedUrl(id, method) {
return cacheWrap(memoryCache, `${RESOLVED_URL_KEY_PREFIX}:${id}`, method, { ttl: { MEMORY_TTL } });
}
module.exports = { cacheWrapImdbId, cacheWrapKitsuId, cacheWrapMetadata, retrieveTorrentFiles, cacheWrapResolvedUrl };
function cacheWrapProxy(id, method) {
return cacheWrap(memoryCache, `${PROXY_KEY_PREFIX}:${id}`, method, { ttl: { PROXY_TTL } });
}
module.exports = {
cacheWrapImdbId,
cacheWrapKitsuId,
cacheWrapMetadata,
retrieveTorrentFiles,
cacheWrapResolvedUrl,
cacheWrapProxy
};

2
scraper/lib/request_helper.js
View File

@@ -1,6 +1,6 @@
const UserAgent = require('user-agents');
const PROXY_HOSTS = process.env.PROXY_HOST && process.env.PROXY_HOST.split(',');
const PROXY_HOSTS = process.env.PROXY_HOSTS && process.env.PROXY_HOSTS.split(',');
const PROXY_USERNAME = process.env.PROXY_USERNAME;
const PROXY_PASSWORD = process.env.PROXY_PASSWORD;
const userAgent = new UserAgent();

17
scraper/moch/realdebrid.js
View File

@@ -1,28 +1,28 @@
const { encode } = require('magnet-uri');
const RealDebridClient = require('real-debrid-api');
const namedQueue = require('named-queue');
const { cacheWrapResolvedUrl } = require('../lib/cache');
const { cacheWrapResolvedUrl, cacheWrapProxy } = require('../lib/cache');
const { getRandomProxy, getRandomUserAgent } = require('../lib/request_helper');
const unrestrictQueue = new namedQueue((task, callback) => task.method()
.then(result => callback(false, result))
.catch((error => callback(error))));
async function resolve(apiKey, infoHash, cachedFileIds, fileIndex) {
async function resolve(ip, apiKey, infoHash, cachedFileIds, fileIndex) {
if (!apiKey || !infoHash || !cachedFileIds || !cachedFileIds.length) {
return Promise.reject("No valid parameters passed");
}
const id = `${apiKey}_${infoHash}_${fileIndex}`;
const method = () => cacheWrapResolvedUrl(id, () => _unrestrict(apiKey, infoHash, cachedFileIds, fileIndex));
const method = () => cacheWrapResolvedUrl(id, () => _unrestrict(ip, apiKey, infoHash, cachedFileIds, fileIndex));
return new Promise(((resolve, reject) => {
unrestrictQueue.push({ id, method }, (error, result) => result ? resolve(result) : reject(error));
}));
}
async function _unrestrict(apiKey, infoHash, cachedFileIds, fileIndex) {
async function _unrestrict(ip, apiKey, infoHash, cachedFileIds, fileIndex) {
console.log(`Unrestricting ${infoHash} [${fileIndex}]`);
const RD = new RealDebridClient(apiKey, getDefaultOptions());
const RD = await getDefaultOptions(ip).then(options => new RealDebridClient(apiKey, options));
const torrentId = await _createOrFindTorrentId(RD, infoHash, cachedFileIds);
if (torrentId) {
const info = await RD.torrents.info(torrentId);
@@ -67,13 +67,14 @@ async function _unrestrictLink(RD, link) {
// });
}
function getDefaultOptions() {
return {
function getDefaultOptions(ip) {
const generateOptions = () => ({
proxy: getRandomProxy(),
headers: {
'User-Agent': getRandomUserAgent()
}
};
});
return cacheWrapProxy(ip, generateOptions).catch(() => generateOptions());
}
module.exports = { resolve };