tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

format comando provider

Browse Source
This commit is contained in:
TheBeastLT
2021-09-14 15:10:28 +02:00
committed by TheBeastLT
parent 4b4cefb3c2
commit 58a0e8286f
2 changed files with 80 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

146
scraper/scrapers/comando/comando_scraper.js
View File

@@ -1,6 +1,6 @@
const moment = require("moment");
const Bottleneck = require("bottleneck");
const leetx = require("./comando_api");
const comando = require("./comando_api");
const { Type } = require("../../lib/types");
const repository = require("../../lib/repository");
const Promises = require("../../lib/promises");
@@ -20,114 +20,90 @@ async function scrape() {
console.log(`[${scrapeStart}] starting ${NAME} scrape...`);
return scrapeLatestTorrents()
.then(() => {
lastScrape.lastScraped = scrapeStart;
return lastScrape.save();
})
.then(() => console.log(`[${moment()}] finished ${NAME} scrape`));
.then(() => {
lastScrape.lastScraped = scrapeStart;
return lastScrape.save();
})
.then(() => console.log(`[${moment()}] finished ${NAME} scrape`));
}
async function updateSeeders(torrent) {
return limiter.schedule(() => leetx.torrent(torrent.torrentId));
return limiter.schedule(() => comando.torrent(torrent.torrentId));
}
async function scrapeLatestTorrents() {
const allowedCategories = [
leetx.Categories.MOVIE,
leetx.Categories.TV,
leetx.Categories.ANIME,
// comando.Categories.MOVIE,
comando.Categories.TV
];
return Promises.sequence(
allowedCategories.map(
(category) => () => scrapeLatestTorrentsForCategory(category)
)
).then((entries) => entries.reduce((a, b) => a.concat(b), []));
return Promises.sequence(allowedCategories
.map((category) => () => scrapeLatestTorrentsForCategory(category)))
.then((entries) => entries.reduce((a, b) => a.concat(b), []));
}
async function scrapeLatestTorrentsForCategory(category, page = 1) {
console.log({Scraper: `Scrapping ${NAME} ${category} category page ${page}`});
return leetx
.browse({ category, page })
.catch((error) => {
console.warn(
`Failed ${NAME} scrapping for [${page}] ${category} due: `,
error
);
return Promise.resolve([]);
})
.then((torrents) => Promise.all(torrents.map((torrent) => limiter.schedule(() => processTorrentRecord(torrent)))))
.then((resolved) => resolved.length > 0 && page < untilPage(category) ? scrapeLatestTorrentsForCategory(category, page + 1) : Promise.resolve());
console.log(`Scrapping ${NAME} ${category} category page ${page}`);
return comando.browse({ category, page })
.catch((error) => {
console.warn(`Failed ${NAME} scrapping for [${page}] ${category} due: `, error);
return Promise.resolve([]);
})
.then((torrents) => Promise.all(torrents.map((torrent) => limiter.schedule(() => processEntry(torrent)))))
.then((resolved) => resolved.length > 0 && page < untilPage(category)
? scrapeLatestTorrentsForCategory(category, page + 1)
: Promise.resolve());
}
async function processTorrentRecord(record) {
if (await checkAndUpdateTorrent({ provider: NAME, ...record })) {
return record;
async function processEntry(entry) {
return comando.torrent(entry.torrentId)
.then(records => Promises.sequence(records.map(record => () => processTorrentRecord(record))))
.catch(() => undefined);
}
async function processTorrentRecord(foundTorrent) {
if (await checkAndUpdateTorrent({ provider: NAME, ...foundTorrent })) {
return foundTorrent;
}
const torrentEntrys = await leetx
.torrent(record.torrentId)
.catch(() => undefined);
if (torrentEntrys === undefined) {
return Promise.resolve([])
if (!foundTorrent.size) {
await updateTorrentSize(foundTorrent);
}
return await Promise.allSettled(
torrentEntrys.map(async (torrentFound) => {
if (!torrentFound || !TYPE_MAPPING[torrentFound.category]) {
return Promise.resolve("Invalid torrent record");
}
if (isNaN(torrentFound.uploadDate)) {
console.warn(
`Incorrect upload date for [${torrentFound.infoHash}] ${torrentFound.name}`
);
return;
}
if (await checkAndUpdateTorrent(torrentFound)) {
return torrentFound;
}
if (!torrentFound.size) {
await updateTorrentSize(torrentFound)
.catch((err) => Promise.resolve(err))
}
if (!torrentFound.seeders) {
await updateCurrentSeeders(torrentFound)
.then(response => response.seeders === 0 ? delete response.seeders : response)
}
if (!torrentFound.imdbId) {
torrentFound.imdbId = await getImdbId(torrentFound.original_name, torrentFound.year, TYPE_MAPPING[torrentFound.category])
}
const torrent = {
infoHash: torrentFound.infoHash,
provider: NAME,
torrentId: torrentFound.torrentId,
name: torrentFound.original_name,
title: torrentFound.name.replace(/\t|\s+/g, " ").trim(),
type: TYPE_MAPPING[torrentFound.category],
year: torrentFound.year,
imdbId: torrentFound.imdbId,
uploadDate: torrentFound.uploadDate,
seeders: torrentFound.seeders,
size: torrentFound.size,
files: torrentFound.files
};
return createTorrentEntry(torrent);
})
);
if (!Number.isInteger(foundTorrent.seeders)) {
await updateCurrentSeeders(foundTorrent);
}
if (!foundTorrent.imdbId) {
const info = { title: foundTorrent.originalName, year: foundTorrent.year };
foundTorrent.imdbId = await getImdbId(info, TYPE_MAPPING[foundTorrent.category]).catch(() => undefined);
}
const torrent = {
infoHash: foundTorrent.infoHash,
provider: NAME,
torrentId: foundTorrent.torrentId,
title: foundTorrent.title,
type: TYPE_MAPPING[foundTorrent.category],
imdbId: foundTorrent.imdbId,
uploadDate: foundTorrent.uploadDate,
seeders: foundTorrent.seeders,
size: foundTorrent.size,
files: foundTorrent.files,
languages: foundTorrent.languages
};
return createTorrentEntry(torrent);
}
function typeMapping() {
const mapping = {};
mapping[leetx.Categories.MOVIE] = Type.MOVIE;
mapping[leetx.Categories.DOCUMENTARIES] = Type.SERIES;
mapping[leetx.Categories.TV] = Type.SERIES;
mapping[leetx.Categories.ANIME] = Type.ANIME;
mapping[comando.Categories.MOVIE] = Type.MOVIE;
mapping[comando.Categories.DOCUMENTARIES] = Type.SERIES;
mapping[comando.Categories.TV] = Type.SERIES;
mapping[comando.Categories.ANIME] = Type.ANIME;
return mapping;
}
function untilPage(category) {
if (leetx.Categories.ANIME === category) {
return 5;
}
if (leetx.Categories.TV === category) {
if (comando.Categories.TV === category) {
return 5;
}
return UNTIL_PAGE;