tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] track updated seeders statistics

Browse Source
This commit is contained in:
TheBeastLT
2020-04-27 18:24:12 +02:00
parent a6f3e85c34
commit 95c2893523
1 changed files with 10 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
scraper/scheduler/seeders.js
View File

@@ -9,12 +9,14 @@ const DELAY = 15 * 1000; // 15 seconds
const limiter = new Bottleneck({ maxConcurrent: 20, minTime: 250 }); const limiter = new Bottleneck({ maxConcurrent: 20, minTime: 250 });
const updateLimiter = new Bottleneck({ maxConcurrent: 20 }); const updateLimiter = new Bottleneck({ maxConcurrent: 20 });
const forceSeedersLimiter = new Bottleneck({ maxConcurrent: 5 }); const forceSeedersLimiter = new Bottleneck({ maxConcurrent: 5 });
const statistics = {};
function scheduleUpdateSeeders() { function scheduleUpdateSeeders() {
console.log('Starting seeders update...') console.log('Starting seeders update...')
return repository.getUpdateSeedersTorrents() return repository.getUpdateSeedersTorrents()
.then(torrents => Promise.all(torrents.map(torrent => limiter.schedule(() => _updateSeeders(torrent))))) .then(torrents => Promise.all(torrents.map(torrent => limiter.schedule(() => _updateSeeders(torrent)))))
.then(() => console.log('Finished seeders update')) .then(torrents => updateStatistics(torrents))
.then(() => console.log('Finished seeders update: ', statistics))
.catch(error => console.warn('Failed seeders update: ', error)) .catch(error => console.warn('Failed seeders update: ', error))
.then(() => delay(DELAY)) .then(() => delay(DELAY))
.then(() => scheduleUpdateSeeders()); .then(() => scheduleUpdateSeeders());
@@ -24,7 +26,7 @@ async function _updateSeeders(torrent) {
const provider = await scrapers.find(provider => provider.name === torrent.provider); const provider = await scrapers.find(provider => provider.name === torrent.provider);
if (!provider) { if (!provider) {
console.log(`No provider found for ${torrent.provider} [${torrent.infoHash}]`) console.log(`No provider found for ${torrent.provider} [${torrent.infoHash}]`)
return Promise.resolve(); return Promise.resolve([]);
} }
const updatedTorrents = await provider.scraper.updateSeeders(torrent, getImdbIdsMethod(torrent)) const updatedTorrents = await provider.scraper.updateSeeders(torrent, getImdbIdsMethod(torrent))
.then(updated => Array.isArray(updated) ? updated : [updated]) .then(updated => Array.isArray(updated) ? updated : [updated])
@@ -47,4 +49,10 @@ function getImdbIdsMethod(torrent) {
.then(ids => Array.from(new Set(ids))); .then(ids => Array.from(new Set(ids)));
} }
function updateStatistics(updatedTorrents) {
const totalTorrents = updatedTorrents.map(nested => nested.length).reduce((a, b) => a + b, 0);
const date = new Date().toISOString().replace(/T.*/, '');
statistics[date] = (statistics[date] || 0) + totalTorrents;
}
module.exports = { scheduleUpdateSeeders } module.exports = { scheduleUpdateSeeders }