tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] updates scapers limiter usage

Browse Source
This commit is contained in:
TheBeastLT
2020-03-13 22:12:18 +01:00
parent 19052cc366
commit 542c96a2c3
8 changed files with 114 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
scraper/scrapers/kickass/kickass_scraper.js
View File

@@ -3,15 +3,11 @@ const Bottleneck = require('bottleneck');
const kickass = require('./kickass_api');
const { Type } = require('../../lib/types');
const repository = require('../../lib/repository');
const {
createTorrentEntry,
createSkipTorrentEntry,
getStoredTorrentEntry,
updateTorrentSeeders
} = require('../../lib/torrentEntries');
const Promises = require('../../lib/promises');
const { createTorrentEntry, getStoredTorrentEntry, updateTorrentSeeders } = require('../../lib/torrentEntries');
const NAME = 'KickassTorrents';
const UNTIL_PAGE = 1;
const UNTIL_PAGE = 10;
const TYPE_MAPPING = typeMapping();
const limiter = new Bottleneck({ maxConcurrent: 40 });
@@ -21,33 +17,36 @@ async function scrape() {
const lastScrape = await repository.getProvider({ name: NAME });
console.log(`[${scrapeStart}] starting ${NAME} scrape...`);
const latestTorrents = await getLatestTorrents();
return Promise.all(latestTorrents.map(torrent => limiter.schedule(() => processTorrentRecord(torrent))))
return scrapeLatestTorrents()
.then(() => {
lastScrape.lastScraped = scrapeStart;
lastScrape.lastScrapedId = latestTorrents.length && latestTorrents[latestTorrents.length - 1].torrentId;
return repository.updateProvider(lastScrape);
})
.then(() => console.log(`[${moment()}] finished ${NAME} scrape`));
}
async function getLatestTorrents() {
async function scrapeLatestTorrents() {
const allowedCategories = [
kickass.Categories.MOVIE,
kickass.Categories.TV,
kickass.Categories.ANIME,
];
return Promise.all(allowedCategories.map(category => getLatestTorrentsForCategory(category)))
return Promises.sequence(allowedCategories.map(category => () => scrapeLatestTorrentsForCategory(category)))
.then(entries => entries.reduce((a, b) => a.concat(b), []));
}
async function getLatestTorrentsForCategory(category, page = 1) {
async function scrapeLatestTorrentsForCategory(category, page = 1) {
console.log(`Scrapping ${NAME} ${category} category page ${page}`);
return kickass.browse(({ category, page }))
.then(torrents => torrents.length && page < UNTIL_PAGE
? getLatestTorrents(category, page + 1).then(nextTorrents => torrents.concat(nextTorrents))
: torrents)
.catch(() => []);
.then(torrents => Promise.all(torrents.map(torrent => limiter.schedule(() => processTorrentRecord(torrent)))))
.then(resolved => resolved.length > 0 && page < UNTIL_PAGE
? scrapeLatestTorrentsForCategory(category, page + 1)
: Promise.resolve())
.catch(error => {
console.warn(`Failed ${NAME} scrapping for [${page}] ${category} due: `, error);
return Promise.resolve();
});
}
async function processTorrentRecord(record) {
@@ -58,7 +57,7 @@ async function processTorrentRecord(record) {
const torrentFound = await kickass.torrent(record.torrentId).catch(() => undefined);
if (!torrentFound || !TYPE_MAPPING[torrentFound.category]) {
return createSkipTorrentEntry(record);
return Promise.resolve('Invalid torrent record');
}
const torrent = {
@@ -73,7 +72,7 @@ async function processTorrentRecord(record) {
seeders: torrentFound.seeders,
};
return createTorrentEntry(torrent);
return createTorrentEntry(torrent).then(() => torrent);
}
function typeMapping() {