tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] fixes seeders update for imdb search providers

Browse Source
This commit is contained in:
TheBeastLT
2020-04-23 20:49:04 +02:00
parent a6b3b164aa
commit d0e0d7b581
3 changed files with 23 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
scraper/scheduler/seeders.js
View File

@@ -27,7 +27,10 @@ async function _updateSeeders(torrent) {
} }
const updatedTorrents = await provider.scraper.updateSeeders(torrent, getImdbIdsMethod(torrent)) const updatedTorrents = await provider.scraper.updateSeeders(torrent, getImdbIdsMethod(torrent))
.then(updated => Array.isArray(updated) ? updated : [updated]) .then(updated => Array.isArray(updated) ? updated : [updated])
.catch(() => []); .catch(error => {
console.warn(`Failed seeders update ${torrent.provider} [${torrent.infoHash}]: `, error)
return []
});
if (!updatedTorrents.find(updated => updated.infoHash === torrent.infoHash)) { if (!updatedTorrents.find(updated => updated.infoHash === torrent.infoHash)) {
await forceSeedersLimiter.schedule(() => updateCurrentSeeders(torrent)) await forceSeedersLimiter.schedule(() => updateCurrentSeeders(torrent))
@@ -37,7 +40,7 @@ async function _updateSeeders(torrent) {
return Promise.all(updatedTorrents.map(updated => updateTorrentSeeders(updated))) return Promise.all(updatedTorrents.map(updated => updateTorrentSeeders(updated)))
} }
async function getImdbIdsMethod(torrent) { function getImdbIdsMethod(torrent) {
return () => repository.getFiles(torrent) return () => repository.getFiles(torrent)
.then(files => files.map(file => file.imdbId).filter(id => id)) .then(files => files.map(file => file.imdbId).filter(id => id))
.then(ids => Array.from(new Set(ids))); .then(ids => Array.from(new Set(ids)));

5
scraper/scrapers/eztv/eztv_scraper.js
View File

@@ -25,8 +25,9 @@ async function scrape() {
} }
async function updateSeeders(torrent, getImdbIdsMethod) { async function updateSeeders(torrent, getImdbIdsMethod) {
return getImdbIdsMethod().then(imdbIds => Promises.sequence(imdbIds return getImdbIdsMethod()
.map(imdbId => limiter.schedule(() => eztv.search(imdbId))))); .then(imdbIds => Promise.all(imdbIds.map(imdbId => limiter.schedule(() => eztv.search(imdbId)))))
.then(results => results.reduce((a, b) => a.concat(b), []));
} }
async function scrapeLatestTorrents() { async function scrapeLatestTorrents() {

17
scraper/scrapers/rarbg/rarbg_scraper.js
View File

@@ -27,8 +27,9 @@ async function scrape() {
} }
async function updateSeeders(torrent, getImdbIdsMethod) { async function updateSeeders(torrent, getImdbIdsMethod) {
return getImdbIdsMethod().then(imdbIds => Promises.sequence(imdbIds return getImdbIdsMethod()
.map(imdbId => limiter.schedule(() => rarbg.search(imdbId, SEARCH_OPTIONS, 'imdb'))))); .then(imdbIds => Promise.all(imdbIds.map(imdbId => limiter.schedule(() => search(imdbId)))))
.then(results => results.reduce((a, b) => a.concat(b), []));
} }
async function scrapeLatestTorrents() { async function scrapeLatestTorrents() {
@@ -88,6 +89,18 @@ async function processTorrentRecord(record) {
return createTorrentEntry(torrent); return createTorrentEntry(torrent);
} }
async function search(imdbId, retries = 5) {
return rarbg.search(imdbId, SEARCH_OPTIONS, 'imdb')
.then(results => results.map(result => toTorrent(result)))
.catch(error => {
if (retries > 0) {
console.log(`Retrying ${imdbId} search...`);
return search(imdbId, retries - 1);
}
return Promise.reject(error);
});
}
function toTorrent(result) { function toTorrent(result) {
return { return {
title: result.title, title: result.title,