tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

check dubbed content when extracting magnets

Browse Source
This commit is contained in:
TheBeastLT
2021-09-15 08:56:49 +02:00
committed by TheBeastLT
parent c9d1d70b66
commit aaca34b2d6
4 changed files with 71 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
scraper/scrapers/comando/comando_api.js
View File

@@ -31,7 +31,10 @@ function torrent(torrentId, config = {}, retries = 2) {
.map((proxyUrl) => singleRequest(`${proxyUrl}/${slug}`, config))) .map((proxyUrl) => singleRequest(`${proxyUrl}/${slug}`, config)))
.then((body) => parseTorrentPage(body)) .then((body) => parseTorrentPage(body))
.then((torrent) => torrent.map(el => ({ torrentId: slug, ...el }))) .then((torrent) => torrent.map(el => ({ torrentId: slug, ...el })))
.catch((err) => torrent(slug, config, retries - 1)); .catch((err) => {
console.warn(`Failed ${slug} request: `, err);
return torrent(slug, config, retries - 1)
});
} }
function search(keyword, config = {}, retries = 2) { function search(keyword, config = {}, retries = 2) {
@@ -113,20 +116,19 @@ function parseTorrentPage(body) {
if (!$) { if (!$) {
reject(new Error('Failed loading body')); reject(new Error('Failed loading body'));
} }
let magnets = []; const magnets = $('h2 > strong')
$(`a[href^="magnet"]`).each((i, section) => { .filter((i, elem) => isPtDubbed($(elem).text())).parent()
let magnet = $(section).attr("href"); .map((i, elem) => $(elem).nextUntil('h2, hr'))
magnets.push(magnet); .map((i, elem) => $(elem).find('a[href^="magnet"]'))
}); .map((i, section) => $(section).attr("href")).get();
const details = $('b:contains(\'Original\')').parent() const details = $('b:contains(\'Original\'), strong:contains(\'Original\')').parent()
const isAnime = parseCategory($('div.entry-categories').html()) === Categories.ANIME
const imdbIdMatch = details.find('a[href*="imdb.com"]').attr('href') const imdbIdMatch = details.find('a[href*="imdb.com"]').attr('href')
const torrent = magnets.map(magnetLink => { const torrents = magnets.map(magnetLink => {
const name = escapeHTML(decode(magnetLink).name.replace(/\+/g, ' ')) const originalName = details.find('strong:contains(\'Original\')').next().text().trim() ||
if (isPtDubbed(name) || isAnime) { details.find('b:contains(\'Original\'), strong:contains(\'Original\')')[0].nextSibling.nodeValue;
return { return {
title: sanitizePtName(name), title: sanitizePtName(escapeHTML(decode(magnetLink).name.replace(/\+/g, ' '))),
originalName: details.find('b:contains(\'Original\')')[0].nextSibling.nodeValue.replace(/: ?/, ''), originalName: originalName.replace(/: ?/, '').trim(),
year: details.find('a[href*="comando.to/category/"]').text(), year: details.find('a[href*="comando.to/category/"]').text(),
infoHash: decode(magnetLink).infoHash, infoHash: decode(magnetLink).infoHash,
magnetLink: magnetLink, magnetLink: magnetLink,
@@ -135,10 +137,9 @@ function parseTorrentPage(body) {
imdbId: imdbIdMatch ? imdbIdMatch.split('/')[4] : null, imdbId: imdbIdMatch ? imdbIdMatch.split('/')[4] : null,
languages: sanitizePtLanguages(details.find( languages: sanitizePtLanguages(details.find(
'b:contains(\'Idioma\'), b:contains(\'Audio\'), b:contains(\'Áudio\')')[0].nextSibling.nodeValue) 'b:contains(\'Idioma\'), b:contains(\'Audio\'), b:contains(\'Áudio\')')[0].nextSibling.nodeValue)
};
} }
}) });
resolve(torrent.filter((x) => x)); resolve(torrents.filter((x) => x));
}); });
} }

15
scraper/scrapers/comoeubaixo/comoeubaixo_api.js
View File

@@ -113,21 +113,17 @@ function parseTorrentPage(body) {
if (!$) { if (!$) {
reject(new Error('Failed loading body')); reject(new Error('Failed loading body'));
} }
let magnets = []; const magnets = $(`a[href^="magnet"]`)
$(`a[href^="magnet"]`).each((i, section) => { .filter((i, elem) => isPtDubbed($(elem).attr('title')))
let magnet = $(section).attr("href"); .map((i, elem) => $(elem).attr("href")).get();
magnets.push(magnet);
});
const details = $('div#informacoes') const details = $('div#informacoes')
const category = details.find('strong:contains(\'Gêneros: \')').next().attr('href').split('/')[0] const category = details.find('strong:contains(\'Gêneros: \')').next().attr('href').split('/')[0]
const isAnime = category === Categories.ANIME const torrents = magnets.map(magnetLink => {
const torrent = magnets.map(magnetLink => {
const name = escapeHTML(decode(magnetLink).name.replace(/\+/g, ' ')) const name = escapeHTML(decode(magnetLink).name.replace(/\+/g, ' '))
const sanitizedTitle = sanitizePtName(name); const sanitizedTitle = sanitizePtName(name);
const originalTitle = details.find('strong:contains(\'Baixar\')')[0].nextSibling.nodeValue.split('-')[0]; const originalTitle = details.find('strong:contains(\'Baixar\')')[0].nextSibling.nodeValue.split('-')[0];
const year = details.find('strong:contains(\'Data de Lançamento: \')').next().text().trim(); const year = details.find('strong:contains(\'Data de Lançamento: \')').next().text().trim();
const fallBackTitle = `${originalTitle.trim()} ${year.trim()} ${sanitizedTitle.trim()}`; const fallBackTitle = `${originalTitle.trim()} ${year.trim()} ${sanitizedTitle.trim()}`;
if (isPtDubbed(name) || isAnime) {
return { return {
title: sanitizedTitle.length > 4 ? sanitizedTitle : fallBackTitle, title: sanitizedTitle.length > 4 ? sanitizedTitle : fallBackTitle,
infoHash: decode(magnetLink).infoHash, infoHash: decode(magnetLink).infoHash,
@@ -137,9 +133,8 @@ function parseTorrentPage(body) {
imdbId: details.find('a[href*="imdb.com"]').attr('href').split('/')[4], imdbId: details.find('a[href*="imdb.com"]').attr('href').split('/')[4],
languages: sanitizePtLanguages(details.find('strong:contains(\'Idioma\')')[0].nextSibling.nodeValue) languages: sanitizePtLanguages(details.find('strong:contains(\'Idioma\')')[0].nextSibling.nodeValue)
}; };
}
}) })
resolve(torrent.filter((x) => x)); resolve(torrents.filter((x) => x));
}); });
} }

27
scraper/scrapers/lapumia/lapumia_api.js
View File

@@ -112,21 +112,16 @@ function parseTorrentPage(body) {
if (!$) { if (!$) {
reject(new Error('Failed loading body')); reject(new Error('Failed loading body'));
} }
let magnets = []; const magnets = $('h2 > span')
$(`a[href^="magnet"]`).each((i, section) => { .filter((i, elem) => isPtDubbed($(elem).text())).parent()
let magnet = $(section).attr("href"); .map((i, elem) => $(elem).nextUntil('h2, hr'))
magnets.push(magnet); .map((i, elem) => $(elem).find('a[href^="magnet"]'))
}); .map((i, section) => $(section).attr("href")).get();
const category = parseCategory($('div.category').html()); const category = parseCategory($('div.category').html());
const details = $('div.content') const details = $('div.content')
const isAnime = category === Categories.ANIME const torrents = magnets.map(magnetLink => ({
const torrent = magnets.map(magnetLink => { title: sanitizePtName(escapeHTML(decode(magnetLink).name.replace(/\+/g, ' '))),
const name = escapeHTML(decode(magnetLink).name.replace(/\+/g, ' ')) originalName: sanitizePtOriginalName(details.find('b:contains(\'Titulo Original:\')')[0].nextSibling.nodeValue),
if (isPtDubbed(name) || isAnime) {
return {
title: sanitizePtName(name),
originalName: sanitizePtOriginalName(
details.find('b:contains(\'Titulo Original:\')')[0].nextSibling.nodeValue),
year: details.find('b:contains(\'Ano de Lançamento:\')')[0].nextSibling.nodeValue.trim(), year: details.find('b:contains(\'Ano de Lançamento:\')')[0].nextSibling.nodeValue.trim(),
infoHash: decode(magnetLink).infoHash, infoHash: decode(magnetLink).infoHash,
magnetLink: magnetLink, magnetLink: magnetLink,
@@ -134,10 +129,8 @@ function parseTorrentPage(body) {
uploadDate: new Date(moment($('div.infos').text().split('•')[0].trim(), 'LL', 'pt-br').format()), uploadDate: new Date(moment($('div.infos').text().split('•')[0].trim(), 'LL', 'pt-br').format()),
imdbId: $('.imdbRatingPlugin').attr('data-title') || null, imdbId: $('.imdbRatingPlugin').attr('data-title') || null,
languages: sanitizePtLanguages(details.find('b:contains(\'Idioma\')')[0].nextSibling.nodeValue) languages: sanitizePtLanguages(details.find('b:contains(\'Idioma\')')[0].nextSibling.nodeValue)
}; }))
} resolve(torrents.filter((x) => x));
})
resolve(torrent.filter((x) => x));
}); });
} }

22
scraper/scrapers/ondebaixa/ondebaixa_api.js
View File

@@ -112,19 +112,13 @@ function parseTorrentPage(body) {
if (!$) { if (!$) {
reject(new Error('Failed loading body')); reject(new Error('Failed loading body'));
} }
let magnets = []; const magnets = $(`a[href^="magnet"]`)
$(`a[href^="magnet"]`).each((i, section) => { .filter((i, elem) => isPtDubbed($(elem).attr('title')))
let magnet = $(section).attr("href"); .map((i, elem) => $(elem).attr("href")).get();
magnets.push(magnet);
});
const details = $('div#informacoes') const details = $('div#informacoes')
const category = details.find('span:contains(\'Gêneros: \')').next().html() const category = details.find('span:contains(\'Gêneros: \')').next().html()
const isAnime = parseCategory(category) === Categories.ANIME const torrents = magnets.map(magnetLink => ({
const torrent = magnets.map(magnetLink => { title: sanitizePtName(escapeHTML(decode(magnetLink).name.replace(/\+/g, ' '))),
const name = escapeHTML(decode(magnetLink).name.replace(/\+/g, ' '))
if (isPtDubbed(name) || isAnime) {
return {
title: sanitizePtName(name),
originalName: sanitizePtOriginalName(details.find('span:contains(\'Título Original: \')').next().text()), originalName: sanitizePtOriginalName(details.find('span:contains(\'Título Original: \')').next().text()),
year: details.find('span:contains(\'Ano de Lançamento: \')').next().text().trim(), year: details.find('span:contains(\'Ano de Lançamento: \')').next().text().trim(),
infoHash: decode(magnetLink).infoHash, infoHash: decode(magnetLink).infoHash,
@@ -132,10 +126,8 @@ function parseTorrentPage(body) {
category: parseCategory(category), category: parseCategory(category),
uploadDate: new Date($('time').attr('datetime')), uploadDate: new Date($('time').attr('datetime')),
languages: sanitizePtLanguages(details.find('span:contains(\'Idioma\')')[0].nextSibling.nodeValue) languages: sanitizePtLanguages(details.find('span:contains(\'Idioma\')')[0].nextSibling.nodeValue)
}; }));
} resolve(torrents.filter((x) => x));
})
resolve(torrent.filter((x) => x));
}); });
} }