diff --git a/scraper/index.js b/scraper/index.js index 4faf7b5..62282c6 100644 --- a/scraper/index.js +++ b/scraper/index.js @@ -26,17 +26,27 @@ async function scrape() { return PROVIDERS .reduce(async (previousPromise, nextProvider) => { await previousPromise; - return nextProvider.scrape().catch(() => Promise.resolve()); + return nextProvider.scrape().catch(error => { + console.warn(`Failed ${nextProvider.NAME} scraping due: `, error); + return Promise.resolve() + }); }, Promise.resolve()); } +function enableScheduling() { + if (process.env.ENABLE_SCHEDULING) { + schedule.scheduleJob(SCRAPE_CRON, () => scrape()); + } else { + scrape(); + } +} + server.get('/', function (req, res) { res.send(200); }); server.listen(process.env.PORT || 7000, async function () { await connect(); - // schedule.scheduleJob(SCRAPE_CRON, () => scrape()); console.log('Scraper started'); - scrape(); + enableScheduling(); }); \ No newline at end of file diff --git a/scraper/scrapers/1337x/1337x_scraper.js b/scraper/scrapers/1337x/1337x_scraper.js index 5aa6e26..e0f7067 100644 --- a/scraper/scrapers/1337x/1337x_scraper.js +++ b/scraper/scrapers/1337x/1337x_scraper.js @@ -86,4 +86,4 @@ function typeMapping() { return mapping; } -module.exports = { scrape }; \ No newline at end of file +module.exports = { scrape, NAME }; \ No newline at end of file diff --git a/scraper/scrapers/horriblesubs/horriblesubs_scraper.js b/scraper/scrapers/horriblesubs/horriblesubs_scraper.js index 1ddd73c..94e8e85 100644 --- a/scraper/scrapers/horriblesubs/horriblesubs_scraper.js +++ b/scraper/scrapers/horriblesubs/horriblesubs_scraper.js @@ -193,4 +193,4 @@ async function checkIfExists(torrent) { return { ...torrent, size: existingTorrent.size, seeders: existingTorrent.seeders }; } -module.exports = { scrape }; \ No newline at end of file +module.exports = { scrape, NAME }; \ No newline at end of file diff --git a/scraper/scrapers/kickass/kickass_dump_scraper.js b/scraper/scrapers/kickass/kickass_dump_scraper.js index af947a1..6e8dc5e 100644 --- a/scraper/scrapers/kickass/kickass_dump_scraper.js +++ b/scraper/scrapers/kickass/kickass_dump_scraper.js @@ -95,4 +95,4 @@ async function findTorrent(record) { return Promise.reject("not found"); } -module.exports = { scrape }; \ No newline at end of file +module.exports = { scrape, NAME }; \ No newline at end of file diff --git a/scraper/scrapers/kickass/kickass_scraper.js b/scraper/scrapers/kickass/kickass_scraper.js index 6a8317d..a23fb64 100644 --- a/scraper/scrapers/kickass/kickass_scraper.js +++ b/scraper/scrapers/kickass/kickass_scraper.js @@ -84,4 +84,4 @@ function typeMapping() { return mapping; } -module.exports = { scrape }; \ No newline at end of file +module.exports = { scrape, NAME }; \ No newline at end of file diff --git a/scraper/scrapers/rarbg/rarbg_dump_scraper.js b/scraper/scrapers/rarbg/rarbg_dump_scraper.js index 4635c8d..5bb3ed9 100644 --- a/scraper/scrapers/rarbg/rarbg_dump_scraper.js +++ b/scraper/scrapers/rarbg/rarbg_dump_scraper.js @@ -73,4 +73,4 @@ function getType(category) { return Type.MOVIE; } -module.exports = { scrape }; \ No newline at end of file +module.exports = { scrape, NAME }; \ No newline at end of file diff --git a/scraper/scrapers/rarbg/rarbg_scraper.js b/scraper/scrapers/rarbg/rarbg_scraper.js index c61cf82..5e57827 100644 --- a/scraper/scrapers/rarbg/rarbg_scraper.js +++ b/scraper/scrapers/rarbg/rarbg_scraper.js @@ -100,4 +100,4 @@ function getType(category) { return Type.MOVIE; } -module.exports = { scrape }; \ No newline at end of file +module.exports = { scrape, NAME }; \ No newline at end of file diff --git a/scraper/scrapers/thepiratebay/thepiratebay_dump_scraper.js b/scraper/scrapers/thepiratebay/thepiratebay_dump_scraper.js index 32d808f..67eddd2 100644 --- a/scraper/scrapers/thepiratebay/thepiratebay_dump_scraper.js +++ b/scraper/scrapers/thepiratebay/thepiratebay_dump_scraper.js @@ -178,4 +178,4 @@ function downloadDump(dump) { }) } -module.exports = { scrape }; \ No newline at end of file +module.exports = { scrape, NAME }; \ No newline at end of file diff --git a/scraper/scrapers/thepiratebay/thepiratebay_scraper.js b/scraper/scrapers/thepiratebay/thepiratebay_scraper.js index d490606..565e61d 100644 --- a/scraper/scrapers/thepiratebay/thepiratebay_scraper.js +++ b/scraper/scrapers/thepiratebay/thepiratebay_scraper.js @@ -82,4 +82,4 @@ async function processTorrentRecord(record) { return createTorrentEntry(torrent); } -module.exports = { scrape }; \ No newline at end of file +module.exports = { scrape, NAME }; \ No newline at end of file diff --git a/scraper/scrapers/thepiratebay/thepiratebay_unofficial_dump_scraper.js b/scraper/scrapers/thepiratebay/thepiratebay_unofficial_dump_scraper.js index 99ec125..6cda1f8 100644 --- a/scraper/scrapers/thepiratebay/thepiratebay_unofficial_dump_scraper.js +++ b/scraper/scrapers/thepiratebay/thepiratebay_unofficial_dump_scraper.js @@ -109,4 +109,4 @@ async function processTorrentRecord(record) { return createTorrentEntry(torrent); } -module.exports = { scrape }; \ No newline at end of file +module.exports = { scrape, NAME }; \ No newline at end of file