tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] fixes periodic hs all shows scraping with disabled seeders updating

Browse Source
This commit is contained in:
TheBeastLT
2020-04-17 19:05:55 +02:00
parent da2c1e5bbe
commit 3299bc2d06
1 changed files with 9 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
scraper/scrapers/horriblesubs/horriblesubs_scraper.js
View File

@@ -19,7 +19,7 @@ const entryLimiter = new Bottleneck({ maxConcurrent: 10 });
async function scrape() { async function scrape() {
const scrapeStart = moment(); const scrapeStart = moment();
const lastScrape = await repository.getProvider({ name: NAME }); const lastScrape = await repository.getProvider({ name: NAME });
const lastScraped = lastScrape.lastScraped && moment.unix(lastScrape.lastScraped); const lastScraped = lastScrape.lastScraped && moment(lastScrape.lastScraped);
if (!lastScraped || lastScraped.add(NEXT_FULL_SCRAPE_OFFSET, 'seconds') < scrapeStart) { if (!lastScraped || lastScraped.add(NEXT_FULL_SCRAPE_OFFSET, 'seconds') < scrapeStart) {
console.log(`[${scrapeStart}] scrapping all ${NAME} shows...`); console.log(`[${scrapeStart}] scrapping all ${NAME} shows...`);
@@ -54,7 +54,7 @@ async function _scrapeAllShows() {
return Promise.all(shows return Promise.all(shows
.map((show) => limiter.schedule(() => horriblesubs.showData(show) .map((show) => limiter.schedule(() => horriblesubs.showData(show)
.then((showData) => _parseShowData(showData)) .then((showData) => _parseShowData(showData, false))
.catch((err) => console.log(err))))); .catch((err) => console.log(err)))));
} }
@@ -110,7 +110,7 @@ async function enrichShow(show) {
} }
} }
async function _parseShowData(showData) { async function _parseShowData(showData, updateSeeders = true) {
console.log(`${NAME}: scrapping ${showData.title} data...`); console.log(`${NAME}: scrapping ${showData.title} data...`);
const showMapping = showMappings[showData.title]; const showMapping = showMappings[showData.title];
const kitsuId = showMapping && showMapping.kitsu_id; const kitsuId = showMapping && showMapping.kitsu_id;
@@ -164,15 +164,18 @@ async function _parseShowData(showData) {
}))) })))
.reduce((a, b) => a.concat(b), []) .reduce((a, b) => a.concat(b), [])
.filter(torrent => torrent.kitsuId) .filter(torrent => torrent.kitsuId)
.map(torrent => entryLimiter.schedule(() => processTorrentRecord(torrent)))) .map(torrent => entryLimiter.schedule(() => processTorrentRecord(torrent, updateSeeders))))
.then(() => console.log(`${NAME}: finished scrapping ${showData.title} data`)); .then(() => console.log(`${NAME}: finished scrapping ${showData.title} data`));
} }
async function processTorrentRecord(torrent) { async function processTorrentRecord(torrent, updateSeeders = true) {
const existingTorrent = await repository.getTorrent(torrent).catch(() => undefined); const existingTorrent = await repository.getTorrent(torrent).catch(() => undefined);
if (existingTorrent && existingTorrent.provider === NAME) { if (existingTorrent && existingTorrent.provider === NAME) {
return updateCurrentSeeders(torrent).then(updatedSeeders => updateTorrentSeeders(updatedSeeders)) if (updateSeeders) {
return updateCurrentSeeders(torrent).then(updatedSeeders => updateTorrentSeeders(updatedSeeders))
}
return Promise.resolve(torrent)
} }
return updateTorrentSize(torrent) return updateTorrentSize(torrent)