tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] fixes seeders update for imdb search providers

Browse Source
This commit is contained in:
TheBeastLT
2020-04-23 20:49:04 +02:00
parent a6b3b164aa
commit d0e0d7b581
3 changed files with 23 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
scraper/scrapers/eztv/eztv_scraper.js
View File

@@ -25,8 +25,9 @@ async function scrape() {
}
async function updateSeeders(torrent, getImdbIdsMethod) {
return getImdbIdsMethod().then(imdbIds => Promises.sequence(imdbIds
.map(imdbId => limiter.schedule(() => eztv.search(imdbId)))));
return getImdbIdsMethod()
.then(imdbIds => Promise.all(imdbIds.map(imdbId => limiter.schedule(() => eztv.search(imdbId)))))
.then(results => results.reduce((a, b) => a.concat(b), []));
}
async function scrapeLatestTorrents() {

17
scraper/scrapers/rarbg/rarbg_scraper.js
View File

@@ -27,8 +27,9 @@ async function scrape() {
}
async function updateSeeders(torrent, getImdbIdsMethod) {
return getImdbIdsMethod().then(imdbIds => Promises.sequence(imdbIds
.map(imdbId => limiter.schedule(() => rarbg.search(imdbId, SEARCH_OPTIONS, 'imdb')))));
return getImdbIdsMethod()
.then(imdbIds => Promise.all(imdbIds.map(imdbId => limiter.schedule(() => search(imdbId)))))
.then(results => results.reduce((a, b) => a.concat(b), []));
}
async function scrapeLatestTorrents() {
@@ -88,6 +89,18 @@ async function processTorrentRecord(record) {
return createTorrentEntry(torrent);
}
async function search(imdbId, retries = 5) {
return rarbg.search(imdbId, SEARCH_OPTIONS, 'imdb')
.then(results => results.map(result => toTorrent(result)))
.catch(error => {
if (retries > 0) {
console.log(`Retrying ${imdbId} search...`);
return search(imdbId, retries - 1);
}
return Promise.reject(error);
});
}
function toTorrent(result) {
return {
title: result.title,