tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] updates samples and redundant video filtering

Browse Source
This commit is contained in:
TheBeastLT
2020-05-06 08:32:52 +02:00
parent 6c9313a0f4
commit 0ad7324a6a
1 changed files with 16 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
scraper/lib/torrent.js
View File

@@ -11,7 +11,7 @@ const TRACKERS_URL = 'https://ngosang.github.io/trackerslist/trackers_best.txt';
const MAX_PEER_CONNECTIONS = process.env.MAX_PEER_CONNECTIONS || 20; const MAX_PEER_CONNECTIONS = process.env.MAX_PEER_CONNECTIONS || 20;
const SEEDS_CHECK_TIMEOUT = process.env.SEEDS_CHECK_TIMEOUT || 10 * 1000; // 10 secs const SEEDS_CHECK_TIMEOUT = process.env.SEEDS_CHECK_TIMEOUT || 10 * 1000; // 10 secs
module.exports.updateCurrentSeeders = function (torrent) { async function updateCurrentSeeders(torrent) {
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
if (!torrent.magnetLink && !torrent.infoHash) { if (!torrent.magnetLink && !torrent.infoHash) {
return resolve(0); return resolve(0);
@@ -36,20 +36,22 @@ module.exports.updateCurrentSeeders = function (torrent) {
torrent.seeders = seeders; torrent.seeders = seeders;
return torrent; return torrent;
}); });
}; }
module.exports.updateTorrentSize = function (torrent) { async function updateTorrentSize(torrent) {
return filesAndSizeFromTorrentStream(torrent, SEEDS_CHECK_TIMEOUT) return filesAndSizeFromTorrentStream(torrent, SEEDS_CHECK_TIMEOUT)
.then(result => { .then(result => {
torrent.size = result.size; torrent.size = result.size;
torrent.files = result.files; torrent.files = result.files;
return torrent; return torrent;
}); });
}; }
module.exports.sizeAndFiles = torrent => filesAndSizeFromTorrentStream(torrent, 30000); async function sizeAndFiles(torrent) {
return filesAndSizeFromTorrentStream(torrent, 30000);
}
module.exports.torrentFiles = function (torrent, timeout) { async function torrentFiles(torrent, timeout) {
return getFilesFromObject(torrent) return getFilesFromObject(torrent)
.catch(() => filesFromTorrentFile(torrent)) .catch(() => filesFromTorrentFile(torrent))
.catch(() => filesFromTorrentStream(torrent, timeout)) .catch(() => filesFromTorrentStream(torrent, timeout))
@@ -58,7 +60,7 @@ module.exports.torrentFiles = function (torrent, timeout) {
videos: filterVideos(files), videos: filterVideos(files),
subtitles: filterSubtitles(files) subtitles: filterSubtitles(files)
})); }));
}; }
function getFilesFromObject(torrent) { function getFilesFromObject(torrent) {
if (torrent.files && torrent.files.length) { if (torrent.files && torrent.files.length) {
@@ -122,12 +124,16 @@ function filesAndSizeFromTorrentStream(torrent, timeout = 30000) {
function filterVideos(files) { function filterVideos(files) {
const maxSize = Math.max(...files.map(file => file.size)); const maxSize = Math.max(...files.map(file => file.size));
const isSample = file => file.name.match(/sample/i) && maxSize / parseInt(file.size) > 10; const minSampleRatio = files.length <= 3 ? 5 : 10;
const minRedundantRatio = files.length <= 3 ? 30 : Number.MAX_VALUE;
const isSample = file => file.path.match(/sample/i) && maxSize / parseInt(file.size) > minSampleRatio;
const isRedundant = file => maxSize / parseInt(file.size) > minRedundantRatio;
const isExtra = file => file.path.match(/extras?\//i); const isExtra = file => file.path.match(/extras?\//i);
return files return files
.filter(file => isVideo(file.path)) .filter(file => isVideo(file.path))
.filter(file => !isSample(file)) .filter(file => !isSample(file))
.filter(file => !isExtra(file)); .filter(file => !isExtra(file))
.filter(file => !isRedundant(file));
} }
function filterSubtitles(files) { function filterSubtitles(files) {
@@ -140,3 +146,4 @@ async function getDefaultTrackers() {
.then(body => body && body.split('\n\n') || [])); .then(body => body && body.split('\n\n') || []));
} }
module.exports = { updateCurrentSeeders, updateTorrentSize, sizeAndFiles, torrentFiles }