tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] adds timeout to seeders updating

Browse Source
This commit is contained in:
TheBeastLT
2020-09-17 21:28:47 +02:00
parent ea9a22ce63
commit 2debc4ee8c
2 changed files with 21 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
scraper/scheduler/seeders.js
View File

@@ -1,11 +1,12 @@
const Bottleneck = require('bottleneck');
const scrapers = require('./scrapers');
const repository = require('../lib/repository')
const { delay } = require('../lib/promises')
const { delay, timeout } = require('../lib/promises')
const { updateCurrentSeeders } = require('../lib/torrent')
const { updateTorrentSeeders } = require('../lib/torrentEntries')
const DELAY = 15 * 1000; // 15 seconds
const DELAY_MS = 15 * 1000; // 15 seconds
const TIMEOUT_MS = 30 * 1000 // 30 seconds
const limiter = new Bottleneck({ maxConcurrent: 20, minTime: 250 });
const updateLimiter = new Bottleneck({ maxConcurrent: 5 });
const forceSeedersLimiter = new Bottleneck({ maxConcurrent: 5 });
@@ -14,11 +15,12 @@ const statistics = {};
function scheduleUpdateSeeders() {
console.log('Starting seeders update...')
return repository.getUpdateSeedersTorrents()
.then(torrents => Promise.all(torrents.map(torrent => limiter.schedule(() => _updateSeeders(torrent)))))
.then(torrents => Promise.all(torrents.map(torrent => limiter.schedule(() =>
timeout(TIMEOUT_MS, _updateSeeders(torrent), `Failed [${torrent.infoHash}] torrent seeders update`)))))
.then(torrents => updateStatistics(torrents))
.then(() => console.log('Finished seeders update:', statistics))
.catch(error => console.warn('Failed seeders update:', error))
.then(() => delay(DELAY))
.then(() => delay(DELAY_MS))
.then(() => scheduleUpdateSeeders());
}