tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] adds scheduling flag to enable it

Browse Source
This commit is contained in:
TheBeastLT
2020-03-12 09:02:50 +01:00
parent 4a0a5c4260
commit c6e0c01850
10 changed files with 22 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
scraper/index.js
View File

@@ -26,17 +26,27 @@ async function scrape() {
return PROVIDERS return PROVIDERS
.reduce(async (previousPromise, nextProvider) => { .reduce(async (previousPromise, nextProvider) => {
await previousPromise; await previousPromise;
return nextProvider.scrape().catch(() => Promise.resolve()); return nextProvider.scrape().catch(error => {
console.warn(`Failed ${nextProvider.NAME} scraping due: `, error);
return Promise.resolve()
});
}, Promise.resolve()); }, Promise.resolve());
} }
function enableScheduling() {
if (process.env.ENABLE_SCHEDULING) {
schedule.scheduleJob(SCRAPE_CRON, () => scrape());
} else {
scrape();
}
}
server.get('/', function (req, res) { server.get('/', function (req, res) {
res.send(200); res.send(200);
}); });
server.listen(process.env.PORT || 7000, async function () { server.listen(process.env.PORT || 7000, async function () {
await connect(); await connect();
// schedule.scheduleJob(SCRAPE_CRON, () => scrape());
console.log('Scraper started'); console.log('Scraper started');
scrape(); enableScheduling();
}); });

2
scraper/scrapers/1337x/1337x_scraper.js
View File

@@ -86,4 +86,4 @@ function typeMapping() {
return mapping; return mapping;
} }
module.exports = { scrape }; module.exports = { scrape, NAME };

2
scraper/scrapers/horriblesubs/horriblesubs_scraper.js
View File

@@ -193,4 +193,4 @@ async function checkIfExists(torrent) {
return { ...torrent, size: existingTorrent.size, seeders: existingTorrent.seeders }; return { ...torrent, size: existingTorrent.size, seeders: existingTorrent.seeders };
} }
module.exports = { scrape }; module.exports = { scrape, NAME };

2
scraper/scrapers/kickass/kickass_dump_scraper.js
View File

@@ -95,4 +95,4 @@ async function findTorrent(record) {
return Promise.reject("not found"); return Promise.reject("not found");
} }
module.exports = { scrape }; module.exports = { scrape, NAME };

2
scraper/scrapers/kickass/kickass_scraper.js
View File

@@ -84,4 +84,4 @@ function typeMapping() {
return mapping; return mapping;
} }
module.exports = { scrape }; module.exports = { scrape, NAME };

2
scraper/scrapers/rarbg/rarbg_dump_scraper.js
View File

@@ -73,4 +73,4 @@ function getType(category) {
return Type.MOVIE; return Type.MOVIE;
} }
module.exports = { scrape }; module.exports = { scrape, NAME };

2
scraper/scrapers/rarbg/rarbg_scraper.js
View File

@@ -100,4 +100,4 @@ function getType(category) {
return Type.MOVIE; return Type.MOVIE;
} }
module.exports = { scrape }; module.exports = { scrape, NAME };

2
scraper/scrapers/thepiratebay/thepiratebay_dump_scraper.js
View File

@@ -178,4 +178,4 @@ function downloadDump(dump) {
}) })
} }
module.exports = { scrape }; module.exports = { scrape, NAME };

2
scraper/scrapers/thepiratebay/thepiratebay_scraper.js
View File

@@ -82,4 +82,4 @@ async function processTorrentRecord(record) {
return createTorrentEntry(torrent); return createTorrentEntry(torrent);
} }
module.exports = { scrape }; module.exports = { scrape, NAME };

2
scraper/scrapers/thepiratebay/thepiratebay_unofficial_dump_scraper.js
View File

@@ -109,4 +109,4 @@ async function processTorrentRecord(record) {
return createTorrentEntry(torrent); return createTorrentEntry(torrent);
} }
module.exports = { scrape }; module.exports = { scrape, NAME };