[scraper] fixes seeders update for imdb search providers

This commit is contained in:
TheBeastLT
2020-04-23 20:49:04 +02:00
parent a6b3b164aa
commit d0e0d7b581
3 changed files with 23 additions and 6 deletions

View File

@@ -27,7 +27,10 @@ async function _updateSeeders(torrent) {
}
const updatedTorrents = await provider.scraper.updateSeeders(torrent, getImdbIdsMethod(torrent))
.then(updated => Array.isArray(updated) ? updated : [updated])
.catch(() => []);
.catch(error => {
console.warn(`Failed seeders update ${torrent.provider} [${torrent.infoHash}]: `, error)
return []
});
if (!updatedTorrents.find(updated => updated.infoHash === torrent.infoHash)) {
await forceSeedersLimiter.schedule(() => updateCurrentSeeders(torrent))
@@ -37,7 +40,7 @@ async function _updateSeeders(torrent) {
return Promise.all(updatedTorrents.map(updated => updateTorrentSeeders(updated)))
}
async function getImdbIdsMethod(torrent) {
function getImdbIdsMethod(torrent) {
return () => repository.getFiles(torrent)
.then(files => files.map(file => file.imdbId).filter(id => id))
.then(ids => Array.from(new Set(ids)));

View File

@@ -25,8 +25,9 @@ async function scrape() {
}
async function updateSeeders(torrent, getImdbIdsMethod) {
return getImdbIdsMethod().then(imdbIds => Promises.sequence(imdbIds
.map(imdbId => limiter.schedule(() => eztv.search(imdbId)))));
return getImdbIdsMethod()
.then(imdbIds => Promise.all(imdbIds.map(imdbId => limiter.schedule(() => eztv.search(imdbId)))))
.then(results => results.reduce((a, b) => a.concat(b), []));
}
async function scrapeLatestTorrents() {

View File

@@ -27,8 +27,9 @@ async function scrape() {
}
async function updateSeeders(torrent, getImdbIdsMethod) {
return getImdbIdsMethod().then(imdbIds => Promises.sequence(imdbIds
.map(imdbId => limiter.schedule(() => rarbg.search(imdbId, SEARCH_OPTIONS, 'imdb')))));
return getImdbIdsMethod()
.then(imdbIds => Promise.all(imdbIds.map(imdbId => limiter.schedule(() => search(imdbId)))))
.then(results => results.reduce((a, b) => a.concat(b), []));
}
async function scrapeLatestTorrents() {
@@ -88,6 +89,18 @@ async function processTorrentRecord(record) {
return createTorrentEntry(torrent);
}
async function search(imdbId, retries = 5) {
return rarbg.search(imdbId, SEARCH_OPTIONS, 'imdb')
.then(results => results.map(result => toTorrent(result)))
.catch(error => {
if (retries > 0) {
console.log(`Retrying ${imdbId} search...`);
return search(imdbId, retries - 1);
}
return Promise.reject(error);
});
}
function toTorrent(result) {
return {
title: result.title,