tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] revert seeders delay reduce

Browse Source
This commit is contained in:
TheBeastLT
2021-02-09 13:45:24 +01:00
parent 849e308e75
commit 8a10ae13e2
2 changed files with 13 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
scraper/lib/torrent.js
View File

@@ -8,9 +8,9 @@ const { Type } = require('./types');
const { isVideo, isSubtitle } = require('./extension'); const { isVideo, isSubtitle } = require('./extension');
const { cacheTrackers } = require('./cache'); const { cacheTrackers } = require('./cache');
const TRACKERS_URL = 'https://ngosang.github.io/trackerslist/trackers_all_udp.txt'; const TRACKERS_URL = 'https://ngosang.github.io/trackerslist/trackers_all.txt';
const MAX_PEER_CONNECTIONS = process.env.MAX_PEER_CONNECTIONS || 20; const MAX_PEER_CONNECTIONS = process.env.MAX_PEER_CONNECTIONS || 20;
const SEEDS_CHECK_TIMEOUT = 20 * 1000; // 30 secs const SEEDS_CHECK_TIMEOUT = 15 * 1000; // 30 secs
const ADDITIONAL_TRACKERS = [ const ADDITIONAL_TRACKERS = [
'http://tracker.trackerfix.com:80/announce', 'http://tracker.trackerfix.com:80/announce',
'udp://9.rarbg.me:2780', 'udp://9.rarbg.me:2780',
@@ -39,12 +39,16 @@ async function updateCurrentSeeders(torrentsInput) {
setTimeout(callback, SEEDS_CHECK_TIMEOUT); setTimeout(callback, SEEDS_CHECK_TIMEOUT);
async.each(Object.keys(perTrackerInfoHashes), function (tracker, ready) { async.each(Object.keys(perTrackerInfoHashes), function (tracker, ready) {
BTClient.scrape({ infoHash: perTrackerInfoHashes[tracker], announce: tracker }, (_, results) => { BTClient.scrape({ infoHash: perTrackerInfoHashes[tracker], announce: tracker }, (error, results) => {
if (results) { if (results) {
Object.entries(results) Object.entries(results)
.filter(([infoHash]) => perTorrentResults[infoHash]) .filter(([infoHash]) => perTorrentResults[infoHash])
.forEach(([infoHash, seeders]) => .forEach(([infoHash, seeders]) =>
perTorrentResults[infoHash][tracker] = [seeders.complete, seeders.incomplete]) perTorrentResults[infoHash][tracker] = [seeders.complete, seeders.incomplete])
} else if (error) {
perTrackerInfoHashes[tracker]
.filter(infoHash => perTorrentResults[infoHash])
.forEach(infoHash => perTorrentResults[infoHash][tracker] = [0, 0, error.message])
} }
ready(); ready();
}) })
@@ -171,9 +175,10 @@ function filterSubtitles(files) {
} }
async function getTorrentTrackers(torrent) { async function getTorrentTrackers(torrent) {
const magnetTrackers = torrent.magnetLink && decode(torrent.magnetLink).tr; const magnetTrackers = torrent.magnetLink && decode(torrent.magnetLink).tr || [];
const torrentTrackers = torrent.trackers && torrent.trackers.split(','); const torrentTrackers = torrent.trackers && torrent.trackers.split(',') || [];
return magnetTrackers || torrentTrackers || getDefaultTrackers(torrent); const defaultTrackers = await getDefaultTrackers(torrent);
return Array.from(new Set([].concat(magnetTrackers).concat(torrentTrackers).concat(defaultTrackers)));
} }
async function getDefaultTrackers(torrent) { async function getDefaultTrackers(torrent) {
@@ -181,8 +186,7 @@ async function getDefaultTrackers(torrent) {
.then(response => response.body && response.body.trim()) .then(response => response.body && response.body.trim())
.then(body => body && body.split('\n\n') || [])) .then(body => body && body.split('\n\n') || []))
.then(trackers => trackers.concat(ADDITIONAL_TRACKERS)) .then(trackers => trackers.concat(ADDITIONAL_TRACKERS))
.then(trackers => torrent.type === Type.ANIME ? trackers.concat(ANIME_TRACKERS) : trackers) .then(trackers => torrent.type === Type.ANIME ? trackers.concat(ANIME_TRACKERS) : trackers);
.then(trackers => Array.from(new Set(trackers)));
} }
module.exports = { updateCurrentSeeders, updateTorrentSize, sizeAndFiles, torrentFiles } module.exports = { updateCurrentSeeders, updateTorrentSize, sizeAndFiles, torrentFiles }

2
scraper/scheduler/seeders.js
View File

@@ -10,7 +10,7 @@ const statistics = {};
function scheduleUpdateSeeders() { function scheduleUpdateSeeders() {
console.log('Starting seeders update...') console.log('Starting seeders update...')
return repository.getUpdateSeedersTorrents(100) return repository.getUpdateSeedersTorrents(50)
.then(torrents => updateCurrentSeeders(torrents)) .then(torrents => updateCurrentSeeders(torrents))
.then(updatedTorrents => Promise.all( .then(updatedTorrents => Promise.all(
updatedTorrents.map(updated => updateLimiter.schedule(() => updateTorrentSeeders(updated))))) updatedTorrents.map(updated => updateLimiter.schedule(() => updateTorrentSeeders(updated)))))