tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

remove scraper module

Browse Source
This commit is contained in:
TheBeastLT
2022-06-16 17:44:38 +03:00
parent a253e8e0b0
commit 554c07d636
69 changed files with 0 additions and 61771 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
scraper/scrapers/torrent9/torrent9_scraper.js
View File

@@ -1,104 +0,0 @@
const moment = require('moment');
const Bottleneck = require('bottleneck');
const torrent9 = require('./torrent9_api');
const { Type } = require('../../lib/types');
const repository = require('../../lib/repository');
const Promises = require('../../lib/promises');
const { createTorrentEntry, checkAndUpdateTorrent } = require('../../lib/torrentEntries');
const NAME = 'Torrent9';
const TYPE_MAPPING = typeMapping();
const api_limiter = new Bottleneck({ maxConcurrent: 1, minTime: 5000 });
const limiter = new Bottleneck({ maxConcurrent: 10 });
const allowedCategories = [
torrent9.Categories.MOVIE,
torrent9.Categories.TV,
];
const clients = [
torrent9
];
async function scrape() {
const scrapeStart = moment();
const lastScrape = await repository.getProvider({ name: NAME });
console.log(`[${scrapeStart}] starting ${NAME} scrape...`);
return scrapeLatestTorrents()
.then(() => {
lastScrape.lastScraped = scrapeStart;
return lastScrape.save();
})
.then(() => console.log(`[${moment()}] finished ${NAME} scrape`));
}
async function scrapeLatestTorrents() {
const scrapeFunctions = allowedCategories
.map(category => clients.map(client => () => scrapeLatestTorrentsForCategory(client, category)))
.reduce((a, b) => a.concat(b), []);
return Promises.sequence(scrapeFunctions)
.then(entries => entries.reduce((a, b) => a.concat(b), []));
}
async function scrapeLatestTorrentsForCategory(client, category, page = 1) {
console.log(`Scrapping ${NAME} ${category} category page ${page}`);
return api_limiter.schedule(() => client.browse({ category, page }))
.catch(error => {
console.warn(`Failed ${NAME} scrapping for [${page}] ${category} due: `, error);
return Promise.resolve([]);
})
.then(results => Promise.all(results.map(r => limiter.schedule(() => processTorrentRecord(client, r)))))
.then(resolved => resolved.length > 0 && page < getUntilPage(category)
? scrapeLatestTorrentsForCategory(client, category, page + 1)
: Promise.resolve([]));
}
async function processTorrentRecord(client, record) {
if (await checkAndUpdateTorrent({ provider: NAME, torrentId: record.torrentId })) {
return record;
}
const foundTorrent = await api_limiter.schedule(() => client.torrent(record.torrentId)).catch(() => undefined);
if (!foundTorrent || !foundTorrent.infoHash) {
console.warn(`Failed retrieving torrent ${record.torrentId}`);
return record;
}
const torrent = {
provider: NAME,
infoHash: foundTorrent.infoHash,
magnetLink: foundTorrent.magnetLink,
torrentLink: foundTorrent.torrentLink,
torrentId: foundTorrent.torrentId,
title: foundTorrent.title,
type: TYPE_MAPPING[foundTorrent.category],
size: foundTorrent.size,
seeders: foundTorrent.seeders,
uploadDate: foundTorrent.uploadDate,
imdbId: foundTorrent.imdbId,
languages: foundTorrent.languages
};
if (await checkAndUpdateTorrent(torrent)) {
console.info(`Skipping torrent ${torrent.torrentId} - [${torrent.infoHash}] ${torrent.title}`);
return torrent;
}
return createTorrentEntry(torrent).then(() => torrent);
}
function typeMapping() {
const mapping = {};
mapping[torrent9.Categories.MOVIE] = Type.MOVIE;
mapping[torrent9.Categories.TV] = Type.SERIES;
return mapping;
}
function getUntilPage(category) {
if (category === torrent9.Categories.TV) {
return 2;
}
return 1;
}
module.exports = { scrape, NAME };