137 lines
4.4 KiB
JavaScript
137 lines
4.4 KiB
JavaScript
const moment = require("moment");
|
|
const Bottleneck = require("bottleneck");
|
|
const leetx = require("./comando_api");
|
|
const { Type } = require("../../lib/types");
|
|
const repository = require("../../lib/repository");
|
|
const Promises = require("../../lib/promises");
|
|
const { updateCurrentSeeders, updateTorrentSize } = require("../../lib/torrent");
|
|
const { createTorrentEntry, checkAndUpdateTorrent } = require("../../lib/torrentEntries");
|
|
const { getImdbId } = require("../../lib/metadata");
|
|
|
|
const NAME = "Comando";
|
|
const UNTIL_PAGE = 5;
|
|
const TYPE_MAPPING = typeMapping();
|
|
|
|
const limiter = new Bottleneck({ maxConcurrent: 5 });
|
|
|
|
async function scrape() {
|
|
const scrapeStart = moment();
|
|
const lastScrape = await repository.getProvider({ name: NAME });
|
|
console.log(`[${scrapeStart}] starting ${NAME} scrape...`);
|
|
|
|
return scrapeLatestTorrents()
|
|
.then(() => {
|
|
lastScrape.lastScraped = scrapeStart;
|
|
return lastScrape.save();
|
|
})
|
|
.then(() => console.log(`[${moment()}] finished ${NAME} scrape`));
|
|
}
|
|
|
|
async function updateSeeders(torrent) {
|
|
return limiter.schedule(() => leetx.torrent(torrent.torrentId));
|
|
}
|
|
|
|
async function scrapeLatestTorrents() {
|
|
const allowedCategories = [
|
|
leetx.Categories.MOVIE,
|
|
leetx.Categories.TV,
|
|
leetx.Categories.ANIME,
|
|
];
|
|
|
|
return Promises.sequence(
|
|
allowedCategories.map(
|
|
(category) => () => scrapeLatestTorrentsForCategory(category)
|
|
)
|
|
).then((entries) => entries.reduce((a, b) => a.concat(b), []));
|
|
}
|
|
|
|
async function scrapeLatestTorrentsForCategory(category, page = 1) {
|
|
console.log({Scraper: `Scrapping ${NAME} ${category} category page ${page}`});
|
|
return leetx
|
|
.browse({ category, page })
|
|
.catch((error) => {
|
|
console.warn(
|
|
`Failed ${NAME} scrapping for [${page}] ${category} due: `,
|
|
error
|
|
);
|
|
return Promise.resolve([]);
|
|
})
|
|
.then((torrents) => Promise.all(torrents.map((torrent) => limiter.schedule(() => processTorrentRecord(torrent)))))
|
|
.then((resolved) => resolved.length > 0 && page < untilPage(category) ? scrapeLatestTorrentsForCategory(category, page + 1) : Promise.resolve());
|
|
}
|
|
|
|
async function processTorrentRecord(record) {
|
|
if (await checkAndUpdateTorrent({ provider: NAME, ...record })) {
|
|
return record;
|
|
}
|
|
const torrentEntrys = await leetx
|
|
.torrent(record.torrentId)
|
|
.catch(() => undefined);
|
|
if (torrentEntrys === undefined) {
|
|
return Promise.resolve([])
|
|
}
|
|
return await Promise.allSettled(
|
|
torrentEntrys.map(async (torrentFound) => {
|
|
if (!torrentFound || !TYPE_MAPPING[torrentFound.category]) {
|
|
return Promise.resolve("Invalid torrent record");
|
|
}
|
|
if (isNaN(torrentFound.uploadDate)) {
|
|
console.warn(
|
|
`Incorrect upload date for [${torrentFound.infoHash}] ${torrentFound.name}`
|
|
);
|
|
return;
|
|
}
|
|
if (await checkAndUpdateTorrent(torrentFound)) {
|
|
return torrentFound;
|
|
}
|
|
if (!torrentFound.size) {
|
|
await updateTorrentSize(torrentFound)
|
|
.catch((err) => Promise.resolve(err))
|
|
}
|
|
if (!torrentFound.seeders) {
|
|
await updateCurrentSeeders(torrentFound)
|
|
.then(response => response.seeders === 0 ? delete response.seeders : response)
|
|
}
|
|
if (!torrentFound.imdbId) {
|
|
torrentFound.imdbId = await getImdbId(torrentFound.original_name, torrentFound.year, TYPE_MAPPING[torrentFound.category])
|
|
}
|
|
const torrent = {
|
|
infoHash: torrentFound.infoHash,
|
|
provider: NAME,
|
|
torrentId: torrentFound.torrentId,
|
|
name: torrentFound.original_name,
|
|
title: torrentFound.name.replace(/\t|\s+/g, " ").trim(),
|
|
type: TYPE_MAPPING[torrentFound.category],
|
|
year: torrentFound.year,
|
|
imdbId: torrentFound.imdbId,
|
|
uploadDate: torrentFound.uploadDate,
|
|
seeders: torrentFound.seeders,
|
|
size: torrentFound.size,
|
|
files: torrentFound.files
|
|
};
|
|
return createTorrentEntry(torrent);
|
|
})
|
|
);
|
|
}
|
|
|
|
function typeMapping() {
|
|
const mapping = {};
|
|
mapping[leetx.Categories.MOVIE] = Type.MOVIE;
|
|
mapping[leetx.Categories.DOCUMENTARIES] = Type.SERIES;
|
|
mapping[leetx.Categories.TV] = Type.SERIES;
|
|
mapping[leetx.Categories.ANIME] = Type.ANIME;
|
|
return mapping;
|
|
}
|
|
|
|
function untilPage(category) {
|
|
if (leetx.Categories.ANIME === category) {
|
|
return 5;
|
|
}
|
|
if (leetx.Categories.TV === category) {
|
|
return 5;
|
|
}
|
|
return UNTIL_PAGE;
|
|
}
|
|
|
|
module.exports = { scrape, updateSeeders, NAME };
|