tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] updates rarbg dum scraper with better logging

Browse Source
This commit is contained in:
TheBeastLT
2020-03-13 09:48:01 +01:00
parent bcd3715612
commit 972d4b8a43
1 changed files with 10 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
scraper/scrapers/rarbg/rarbg_dump_scraper.js
View File

@@ -20,8 +20,8 @@ async function scrape() {
const seriesImdbIds = require('./rargb_series_imdb_ids_2020-03-09.json'); const seriesImdbIds = require('./rargb_series_imdb_ids_2020-03-09.json');
const allImdbIds = [].concat(movieImdbIds).concat(seriesImdbIds); const allImdbIds = [].concat(movieImdbIds).concat(seriesImdbIds);
return Promise.all(allImdbIds.map(imdbId => limiter.schedule(() => getTorrentsForImdbId(imdbId)) return Promise.all(allImdbIds.map(imdbId => limiter.schedule(() => getTorrentsForImdbId(imdbId)
.then(torrents => Promise.all(torrents.map(t => entryLimiter.schedule(() => processTorrentRecord(t))))))) .then(torrents => Promise.all(torrents.map(t => entryLimiter.schedule(() => processTorrentRecord(t))))))))
.then(() => console.log(`[${moment()}] finished ${NAME} dump scrape`)); .then(() => console.log(`[${moment()}] finished ${NAME} dump scrape`));
} }
@@ -38,7 +38,14 @@ async function getTorrentsForImdbId(imdbId) {
uploadDate: new Date(torrent.pubdate), uploadDate: new Date(torrent.pubdate),
imdbId: torrent.episode_info && torrent.episode_info.imdb imdbId: torrent.episode_info && torrent.episode_info.imdb
}))) })))
.catch((err) => []); .then(torrents => {
console.log(`Completed ${imdbId} request`);
return torrents;
})
.catch(error => {
console.warn(`Failed ${NAME} request for ${imdbId}: `, error);
return [];
});
} }
async function processTorrentRecord(record) { async function processTorrentRecord(record) {