brock/torrentio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

format lapumia provider

Browse Source
This commit is contained in:
TheBeastLT
2021-09-14 15:15:02 +02:00
committed by TheBeastLT
parent c17627e583
commit 3379133ad2
2 changed files with 82 additions and 126 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
scraper/scrapers/lapumia/lapumia_api.js
View File

@@ -4,8 +4,8 @@ const cheerio = require("cheerio");
const decode = require('magnet-uri'); const decode = require('magnet-uri');
const Promises = require('../../lib/promises'); const Promises = require('../../lib/promises');
const { escapeHTML } = require('../../lib/metadata'); const { escapeHTML } = require('../../lib/metadata');
const { getRandomUserAgent } = require("../../lib/requestHelper"); const { getRandomUserAgent } = require('../../lib/requestHelper');
moment.locale("pt-br"); const { isPtDubbed, sanitizePtName, sanitizePtOriginalName, sanitizePtLanguages } = require('../scraperHelper')
const defaultTimeout = 10000; const defaultTimeout = 10000;
const maxSearchPage = 50 const maxSearchPage = 50
@@ -27,7 +27,7 @@ function torrent(torrentId, config = {}, retries = 2) {
const proxyList = config.proxyList || defaultProxies; const proxyList = config.proxyList || defaultProxies;
const slug = torrentId.split('?p=')[1]; const slug = torrentId.split('?p=')[1];
return Promises.first(proxyList return Promises.first(proxyList
.map((proxyUrl) => singleRequest(`${proxyUrl}/?p=${slug}`, config))) .map((proxyUrl) => singleRequest(`${proxyUrl}/?p=${slug}`, config)))
.then((body) => parseTorrentPage(body)) .then((body) => parseTorrentPage(body))
.then((torrent) => torrent.map(el => ({ torrentId: slug, ...el }))) .then((torrent) => torrent.map(el => ({ torrentId: slug, ...el })))
.catch((err) => torrent(slug, config, retries - 1)); .catch((err) => torrent(slug, config, retries - 1));
@@ -43,7 +43,7 @@ function search(keyword, config = {}, retries = 2) {
const requestUrl = proxyUrl => `${proxyUrl}/page/${page}/?s=${keyword}` const requestUrl = proxyUrl => `${proxyUrl}/page/${page}/?s=${keyword}`
return Promises.first(proxyList return Promises.first(proxyList
.map(proxyUrl => singleRequest(requestUrl(proxyUrl), config))) .map(proxyUrl => singleRequest(requestUrl(proxyUrl), config)))
.then(body => parseTableBody(body)) .then(body => parseTableBody(body))
.then(torrents => torrents.length === 10 && page < extendToPage .then(torrents => torrents.length === 10 && page < extendToPage
? search(keyword, { ...config, page: page + 1 }).catch(() => []) ? search(keyword, { ...config, page: page + 1 }).catch(() => [])
@@ -62,7 +62,7 @@ function browse(config = {}, retries = 2) {
const requestUrl = proxyUrl => category ? `${proxyUrl}/${category}/page/${page}/` : `${proxyUrl}/page/${page}/` const requestUrl = proxyUrl => category ? `${proxyUrl}/${category}/page/${page}/` : `${proxyUrl}/page/${page}/`
return Promises.first(proxyList return Promises.first(proxyList
.map((proxyUrl) => singleRequest(requestUrl(proxyUrl), config))) .map((proxyUrl) => singleRequest(requestUrl(proxyUrl), config)))
.then((body) => parseTableBody(body)) .then((body) => parseTableBody(body))
.catch((err) => browse(config, retries - 1)); .catch((err) => browse(config, retries - 1));
} }
@@ -103,7 +103,7 @@ function parseTableBody(body) {
}); });
resolve(torrents); resolve(torrents);
}); });
} }
function parseTorrentPage(body) { function parseTorrentPage(body) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
@@ -117,58 +117,36 @@ function parseTorrentPage(body) {
let magnet = $(section).attr("href"); let magnet = $(section).attr("href");
magnets.push(magnet); magnets.push(magnet);
}); });
const categorys = $('div.category').html(); const category = parseCategory($('div.category').html());
const details = $('div.content') const details = $('div.content')
const isAnime = parseCategory(categorys) === Categories.ANIME const isAnime = category === Categories.ANIME
const torrent = magnets.map(magnetLink => { const torrent = magnets.map(magnetLink => {
const name = escapeHTML(decode(magnetLink).name.replace(/\+/g, ' ')) const name = escapeHTML(decode(magnetLink).name.replace(/\+/g, ' '))
if(isDubled(name) || isAnime) { if (isPtDubbed(name) || isAnime) {
return { return {
name: escapeHTML(decode(magnetLink).name.replace(/\+/g, ' ')), title: sanitizePtName(name),
original_name: parseName(details.find('b:contains(\'Titulo Original:\')')[0].nextSibling.nodeValue), originalName: sanitizePtOriginalName(
details.find('b:contains(\'Titulo Original:\')')[0].nextSibling.nodeValue),
year: details.find('b:contains(\'Ano de Lançamento:\')')[0].nextSibling.nodeValue.trim(), year: details.find('b:contains(\'Ano de Lançamento:\')')[0].nextSibling.nodeValue.trim(),
infoHash: decode(magnetLink).infoHash, infoHash: decode(magnetLink).infoHash,
magnetLink: magnetLink, magnetLink: magnetLink,
category: parseCategory(categorys), category: category,
uploadDate: new Date(moment($('div.infos').text().split('•')[0].trim(), 'LL', true).format()), uploadDate: new Date(moment($('div.infos').text().split('•')[0].trim(), 'LL', 'pt-br').format()),
imdbId: $('.imdbRatingPlugin').attr('data-title') || null imdbId: $('.imdbRatingPlugin').attr('data-title') || null,
languages: sanitizePtLanguages(details.find('b:contains(\'Idioma\')')[0].nextSibling.nodeValue)
}; };
}; }
}) })
resolve(torrent.filter((x) => x)); resolve(torrent.filter((x) => x));
}); });
} }
function parseName(name) {
return name
.replace(/S01|S02|S03|S04|S05|S06|S07|S08|S09/g, '')
}
function isDubled(name){
name = name.toLowerCase()
if(name.includes('dublado')){
return true
}
if(name.includes('dual')){
return true
}
if(name.includes('nacional')){
return true
}
if(name.includes('multi')){
return true
}
return false
}
function parseCategory(categorys) { function parseCategory(categorys) {
const $ = cheerio.load(categorys) const $ = cheerio.load(categorys)
const isAnime = $('a:contains(\'Animes\')').text() if ($('a:contains(\'Animes\')').text()) {
const isSerie = $('a:contains(\'Series\')').text()
if(isAnime) {
return Categories.ANIME return Categories.ANIME
} }
if(isSerie) { if ($('a:contains(\'Series\')').text()) {
return Categories.TV return Categories.TV
} }
return Categories.MOVIE return Categories.MOVIE

146
scraper/scrapers/lapumia/lapumia_scraper.js
View File

@@ -1,6 +1,6 @@
const moment = require("moment"); const moment = require("moment");
const Bottleneck = require("bottleneck"); const Bottleneck = require("bottleneck");
const leetx = require("./lapumia_api"); const lapumia = require("./lapumia_api");
const { Type } = require("../../lib/types"); const { Type } = require("../../lib/types");
const repository = require("../../lib/repository"); const repository = require("../../lib/repository");
const Promises = require("../../lib/promises"); const Promises = require("../../lib/promises");
@@ -20,114 +20,92 @@ async function scrape() {
console.log(`[${scrapeStart}] starting ${NAME} scrape...`); console.log(`[${scrapeStart}] starting ${NAME} scrape...`);
return scrapeLatestTorrents() return scrapeLatestTorrents()
.then(() => { .then(() => {
lastScrape.lastScraped = scrapeStart; lastScrape.lastScraped = scrapeStart;
return lastScrape.save(); return lastScrape.save();
}) })
.then(() => console.log(`[${moment()}] finished ${NAME} scrape`)); .then(() => console.log(`[${moment()}] finished ${NAME} scrape`));
} }
async function updateSeeders(torrent) { async function updateSeeders(torrent) {
return limiter.schedule(() => leetx.torrent(torrent.torrentId)); return limiter.schedule(() => lapumia.torrent(torrent.torrentId));
} }
async function scrapeLatestTorrents() { async function scrapeLatestTorrents() {
const allowedCategories = [ const allowedCategories = [
leetx.Categories.MOVIE, lapumia.Categories.MOVIE
leetx.Categories.TV,
leetx.Categories.ANIME,
]; ];
return Promises.sequence( return Promises.sequence(allowedCategories
allowedCategories.map( .map((category) => () => scrapeLatestTorrentsForCategory(category)))
(category) => () => scrapeLatestTorrentsForCategory(category) .then((entries) => entries.reduce((a, b) => a.concat(b), []));
)
).then((entries) => entries.reduce((a, b) => a.concat(b), []));
} }
async function scrapeLatestTorrentsForCategory(category, page = 1) { async function scrapeLatestTorrentsForCategory(category, page = 1) {
console.log({Scraper: `Scrapping ${NAME} ${category} category page ${page}`}); console.log(`Scrapping ${NAME} ${category} category page ${page}`);
return leetx return lapumia
.browse({ category, page }) .browse({ category, page })
.catch((error) => { .catch((error) => {
console.warn( console.warn(`Failed ${NAME} scrapping for [${page}] ${category} due: `, error);
`Failed ${NAME} scrapping for [${page}] ${category} due: `, return Promise.resolve([]);
error })
); .then((torrents) => Promise.all(torrents.map((torrent) => limiter.schedule(() => processEntry(torrent)))))
return Promise.resolve([]); .then((resolved) => resolved.length > 0 && page < untilPage(category)
}) ? scrapeLatestTorrentsForCategory(category, page + 1)
.then((torrents) => Promise.all(torrents.map((torrent) => limiter.schedule(() => processTorrentRecord(torrent))))) : Promise.resolve());
.then((resolved) => resolved.length > 0 && page < untilPage(category) ? scrapeLatestTorrentsForCategory(category, page + 1) : Promise.resolve());
} }
async function processTorrentRecord(record) { async function processEntry(entry) {
if (await checkAndUpdateTorrent({ provider: NAME, ...record })) { return lapumia.torrent(entry.torrentId)
return record; .then(records => Promises.sequence(records.map(record => () => processTorrentRecord(record))))
} .catch(() => undefined);
const torrentEntrys = await leetx }
.torrent(record.torrentId)
.catch(() => undefined);
if (torrentEntrys === undefined) {
return Promise.resolve([])
}
return await Promise.allSettled(
torrentEntrys.map(async (torrentFound) => {
if (!torrentFound || !TYPE_MAPPING[torrentFound.category]) {
return Promise.resolve("Invalid torrent record");
}
if (isNaN(torrentFound.uploadDate)) {
console.warn(
`Incorrect upload date for [${torrentFound.infoHash}] ${torrentFound.name}`
);
return;
}
if (await checkAndUpdateTorrent(torrentFound)) {
return torrentFound;
}
if (!torrentFound.size) {
await updateTorrentSize(torrentFound)
.catch((err) => Promise.resolve(err))
}
if (!torrentFound.seeders) {
await updateCurrentSeeders(torrentFound)
.then(response => response.seeders === 0 ? delete response.seeders : response)
}
if (!torrentFound.imdbId) {
torrentFound.imdbId = await getImdbId(torrentFound.original_name, torrentFound.year, TYPE_MAPPING[torrentFound.category])
}
const torrent = { async function processTorrentRecord(foundTorrent) {
infoHash: torrentFound.infoHash, if (await checkAndUpdateTorrent({ provider: NAME, ...foundTorrent })) {
provider: NAME, return foundTorrent;
torrentId: torrentFound.torrentId, }
name: torrentFound.original_name,
title: torrentFound.name.replace(/\t|\s+/g, " ").trim(), if (!foundTorrent.size) {
type: TYPE_MAPPING[torrentFound.category], await updateTorrentSize(foundTorrent);
year: torrentFound.year, }
imdbId: torrentFound.imdbId, if (!Number.isInteger(foundTorrent.seeders)) {
uploadDate: torrentFound.uploadDate, await updateCurrentSeeders(foundTorrent);
seeders: torrentFound.seeders, }
size: torrentFound.size, if (!foundTorrent.imdbId && TYPE_MAPPING[foundTorrent.category] !== Type.ANIME) {
files: torrentFound.files const info = { title: foundTorrent.originalName, year: foundTorrent.year };
}; foundTorrent.imdbId = await getImdbId(info, TYPE_MAPPING[foundTorrent.category]).catch(() => undefined);
return createTorrentEntry(torrent); }
})
); const torrent = {
infoHash: foundTorrent.infoHash,
provider: NAME,
torrentId: foundTorrent.torrentId,
title: foundTorrent.title,
type: TYPE_MAPPING[foundTorrent.category],
imdbId: foundTorrent.imdbId,
uploadDate: foundTorrent.uploadDate,
seeders: foundTorrent.seeders,
size: foundTorrent.size,
files: foundTorrent.files,
languages: foundTorrent.languages
};
return createTorrentEntry(torrent);
} }
function typeMapping() { function typeMapping() {
const mapping = {}; const mapping = {};
mapping[leetx.Categories.MOVIE] = Type.MOVIE; mapping[lapumia.Categories.MOVIE] = Type.MOVIE;
mapping[leetx.Categories.TV] = Type.SERIES; mapping[lapumia.Categories.TV] = Type.SERIES;
mapping[leetx.Categories.ANIME] = Type.ANIME; mapping[lapumia.Categories.ANIME] = Type.ANIME;
return mapping; return mapping;
} }
function untilPage(category) { function untilPage(category) {
if (leetx.Categories.TV === category) { if (lapumia.Categories.TV === category) {
return 5; return 5;
} }
if (leetx.Categories.ANIME === category) { if (lapumia.Categories.ANIME === category) {
return 2; return 2;
} }
return UNTIL_PAGE; return UNTIL_PAGE;