tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] removes moch resolver from scraper

Browse Source
This commit is contained in:
TheBeastLT
2020-04-09 13:35:11 +02:00
parent fed5098b88
commit cf7b4411d2
5 changed files with 123 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
scraper/index.js
View File

@@ -3,7 +3,6 @@ const express = require("express");
const server = express();
const schedule = require('node-schedule');
const { connect, getUpdateSeedersTorrents } = require('./lib/repository');
const realDebrid = require('./moch/realdebrid');
const thepiratebayScraper = require('./scrapers/thepiratebay/thepiratebay_scraper');
const horribleSubsScraper = require('./scrapers/horriblesubs/horriblesubs_scraper');
const leetxScraper = require('./scrapers/1337x/1337x_scraper');
@@ -62,20 +61,6 @@ server.get('/', function (req, res) {
res.sendStatus(200);
});
server.get('/realdebrid/:apiKey/:infoHash/:cachedFileIds/:fileIndex?', (req, res) => {
const { apiKey, infoHash, cachedFileIds, fileIndex } = req.params;
realDebrid.resolve(req.ip, apiKey, infoHash, cachedFileIds, isNaN(fileIndex) ? undefined : parseInt(fileIndex))
.then(url => {
res.writeHead(301, { Location: url });
res.end();
})
.catch(error => {
console.log(error);
res.statusCode = 404;
res.end();
});
});
server.listen(process.env.PORT || 7000, async () => {
await connect();
console.log('Scraper started');

21
scraper/lib/cache.js
View File

@@ -5,16 +5,10 @@ const GLOBAL_KEY_PREFIX = 'stremio-torrentio';
const IMDB_ID_PREFIX = `${GLOBAL_KEY_PREFIX}|imdb_id`;
const KITSU_ID_PREFIX = `${GLOBAL_KEY_PREFIX}|kitsu_id`;
const METADATA_PREFIX = `${GLOBAL_KEY_PREFIX}|metadata`;
const RESOLVED_URL_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|moch`;
const PROXY_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|proxy`;
const USER_AGENT_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|agent`;
const TRACKERS_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|trackers`;
const GLOBAL_TTL = process.env.METADATA_TTL || 7 * 24 * 60 * 60; // 7 days
const MEMORY_TTL = process.env.METADATA_TTL || 2 * 60 * 60; // 2 hours
const RESOLVED_URL_TTL = 2 * 60; // 2 minutes
const PROXY_TTL = 60 * 60; // 60 minutes
const USER_AGENT_TTL = 2 * 24 * 60 * 60; // 2 days
const TRACKERS_TTL = 2 * 24 * 60 * 60; // 2 days
const MONGO_URI = process.env.MONGODB_URI;
@@ -65,18 +59,6 @@ function cacheWrapMetadata(id, method) {
return cacheWrap(memoryCache, `${METADATA_PREFIX}:${id}`, method, { ttl: MEMORY_TTL });
}
function cacheWrapResolvedUrl(id, method) {
return cacheWrap(memoryCache, `${RESOLVED_URL_KEY_PREFIX}:${id}`, method, { ttl: { RESOLVED_URL_TTL } });
}
function cacheWrapProxy(id, method) {
return cacheWrap(memoryCache, `${PROXY_KEY_PREFIX}:${id}`, method, { ttl: { PROXY_TTL } });
}
function cacheUserAgent(id, method) {
return cacheWrap(memoryCache, `${USER_AGENT_KEY_PREFIX}:${id}`, method, { ttl: { USER_AGENT_TTL } });
}
function cacheTrackers(method) {
return cacheWrap(memoryCache, `${TRACKERS_KEY_PREFIX}`, method, { ttl: { TRACKERS_TTL } });
}
@@ -85,9 +67,6 @@ module.exports = {
cacheWrapImdbId,
cacheWrapKitsuId,
cacheWrapMetadata,
cacheWrapResolvedUrl,
cacheWrapProxy,
cacheUserAgent,
cacheTrackers
};

83
scraper/moch/realdebrid.js
View File

@@ -1,83 +0,0 @@
const { encode } = require('magnet-uri');
const RealDebridClient = require('real-debrid-api');
const namedQueue = require('named-queue');
const { cacheWrapResolvedUrl, cacheWrapProxy, cacheUserAgent } = require('../lib/cache');
const { getRandomProxy, getRandomUserAgent } = require('../lib/request_helper');
const unrestrictQueue = new namedQueue((task, callback) => task.method()
.then(result => callback(false, result))
.catch((error => callback(error))));
async function resolve(ip, apiKey, infoHash, cachedFileIds, fileIndex) {
if (!apiKey || !infoHash || !cachedFileIds || !cachedFileIds.length) {
return Promise.reject("No valid parameters passed");
}
const id = `${apiKey}_${infoHash}_${fileIndex}`;
const method = () => cacheWrapResolvedUrl(id, () => _unrestrict(ip, apiKey, infoHash, cachedFileIds, fileIndex));
return new Promise(((resolve, reject) => {
unrestrictQueue.push({ id, method }, (error, result) => result ? resolve(result) : reject(error));
}));
}
async function _unrestrict(ip, apiKey, infoHash, cachedFileIds, fileIndex) {
console.log(`Unrestricting ${infoHash} [${fileIndex}]`);
const options = await getDefaultOptions(ip);
const RD = new RealDebridClient(apiKey, options);
const torrentId = await _createOrFindTorrentId(RD, infoHash, cachedFileIds);
if (torrentId) {
const info = await RD.torrents.info(torrentId);
const targetFile = info.files.find(file => file.id === fileIndex + 1)
|| info.files.filter(file => file.selected).sort((a, b) => b.bytes - a.bytes)[0];
const selectedFiles = info.files.filter(file => file.selected);
const fileLink = info.links.length === 1
? info.links[0]
: info.links[selectedFiles.indexOf(targetFile)];
const unrestrictedLink = await _unrestrictLink(RD, fileLink);
console.log(`Unrestricted ${infoHash} [${fileIndex}] to ${unrestrictedLink}`);
return unrestrictedLink;
}
return Promise.reject("Failed adding torrent");
}
async function _createOrFindTorrentId(RD, infoHash, cachedFileIds) {
return RD.torrents.get(0, 1)
.then(torrents => torrents.find(torrent => torrent.hash.toLowerCase() === infoHash))
.then(torrent => torrent && torrent.id || Promise.reject('No recent torrent found'))
.catch((error) => RD.torrents.addMagnet(encode({ infoHash }))
.then(response => RD.torrents.selectFiles(response.id, cachedFileIds)
.then((() => response.id))))
.catch(error => {
console.warn('Failed RealDebrid torrent retrieval', error);
return undefined;
});
}
async function _unrestrictLink(RD, link) {
if (!link || !link.length) {
return Promise.reject("No available links found");
}
return RD.unrestrict.link(link)
.then(unrestrictedLink => unrestrictedLink.download);
// .then(unrestrictedLink => RD.streaming.transcode(unrestrictedLink.id))
// .then(transcodedLink => {
// const url = transcodedLink.apple && transcodedLink.apple.full
// || transcodedLink[Object.keys(transcodedLink)[0]].full;
// console.log(`Unrestricted ${link} to ${url}`);
// return url;
// });
}
async function getDefaultOptions(ip) {
const userAgent = await cacheUserAgent(ip, () => getRandomUserAgent()).catch(() => getRandomUserAgent());
const proxy = await cacheWrapProxy('realdebrid', () => getRandomProxy()).catch(() => getRandomProxy());
return {
proxy: proxy,
headers: {
'User-Agent': userAgent
}
};
}
module.exports = { resolve };