tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

update seeders for new torrents

Browse Source
This commit is contained in:
TheBeastLT
2021-09-17 20:37:09 +02:00
parent b0029dcf5a
commit e4c4259ed8
4 changed files with 45 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
scraper/scheduler/seeders.js
View File

@@ -7,29 +7,48 @@ const { updateTorrentSeeders } = require('../lib/torrentEntries')
const DELAY_MS = 0; // 0 seconds
const updateLimiter = new Bottleneck({ maxConcurrent: 5 });
const statistics = {};
const statisticsNew = {};
function scheduleUpdateSeeders() {
console.log('Starting seeders update...')
return getTorrents()
getTorrents()
.then(torrents => updateCurrentSeeders(torrents))
.then(updatedTorrents => Promise.all(
updatedTorrents.map(updated => updateLimiter.schedule(() => updateTorrentSeeders(updated)))))
.then(torrents => updateStatistics(torrents))
.then(torrents => updateStatistics(torrents, statistics))
.then(() => console.log('Finished seeders update:', statistics))
.catch(error => console.warn('Failed seeders update:', error))
.then(() => delay(DELAY_MS))
.then(() => scheduleUpdateSeeders());
}
function scheduleUpdateSeedersForNewTorrents() {
console.log('Starting seeders update for new torrents...')
getNewTorrents()
.then(torrents => updateCurrentSeeders(torrents))
.then(updatedTorrents => Promise.all(
updatedTorrents.map(updated => updateLimiter.schedule(() => updateTorrentSeeders(updated)))))
.then(torrents => updateStatistics(torrents, statisticsNew))
.then(() => console.log('Finished seeders update for new torrents:', statisticsNew))
.catch(error => console.warn('Failed seeders update for new torrents:', error))
.then(() => delay(30_000))
.then(() => scheduleUpdateSeedersForNewTorrents());
}
async function getTorrents() {
return repository.getUpdateSeedersTorrents(50)
return repository.getUpdateSeedersTorrents()
.catch(() => delay(5000).then(() => getTorrents()))
}
function updateStatistics(updatedTorrents) {
const totalTorrents = updatedTorrents.map(nested => nested.length).reduce((a, b) => a + b, 0);
const date = new Date().toISOString().replace(/T.*/, '');
statistics[date] = (statistics[date] || 0) + totalTorrents;
async function getNewTorrents() {
return repository.getUpdateSeedersNewTorrents()
.catch(() => delay(5000).then(() => getNewTorrents()))
}
module.exports = { scheduleUpdateSeeders }
function updateStatistics(updatedTorrents, statisticsObject) {
const totalTorrents = updatedTorrents.map(nested => nested.length).reduce((a, b) => a + b, 0);
const date = new Date().toISOString().replace(/T.*/, '');
statisticsObject[date] = (statisticsObject[date] || 0) + totalTorrents;
}
module.exports = { scheduleUpdateSeeders, scheduleUpdateSeedersForNewTorrents }