tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

check dubbed content when extracting magnets

Browse Source
This commit is contained in:
TheBeastLT
2021-09-15 08:56:49 +02:00
committed by TheBeastLT
parent c9d1d70b66
commit aaca34b2d6
4 changed files with 71 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
scraper/scrapers/comando/comando_api.js
View File

@@ -31,7 +31,10 @@ function torrent(torrentId, config = {}, retries = 2) {
.map((proxyUrl) => singleRequest(`${proxyUrl}/${slug}`, config))) .map((proxyUrl) => singleRequest(`${proxyUrl}/${slug}`, config)))
.then((body) => parseTorrentPage(body)) .then((body) => parseTorrentPage(body))
.then((torrent) => torrent.map(el => ({ torrentId: slug, ...el }))) .then((torrent) => torrent.map(el => ({ torrentId: slug, ...el })))
.catch((err) => torrent(slug, config, retries - 1)); .catch((err) => {
console.warn(`Failed ${slug} request: `, err);
return torrent(slug, config, retries - 1)
});
} }
function search(keyword, config = {}, retries = 2) { function search(keyword, config = {}, retries = 2) {
@@ -113,32 +116,30 @@ function parseTorrentPage(body) {
if (!$) { if (!$) {
reject(new Error('Failed loading body')); reject(new Error('Failed loading body'));
} }
let magnets = []; const magnets = $('h2 > strong')
$(`a[href^="magnet"]`).each((i, section) => { .filter((i, elem) => isPtDubbed($(elem).text())).parent()
let magnet = $(section).attr("href"); .map((i, elem) => $(elem).nextUntil('h2, hr'))
magnets.push(magnet); .map((i, elem) => $(elem).find('a[href^="magnet"]'))
}); .map((i, section) => $(section).attr("href")).get();
const details = $('b:contains(\'Original\')').parent() const details = $('b:contains(\'Original\'), strong:contains(\'Original\')').parent()
const isAnime = parseCategory($('div.entry-categories').html()) === Categories.ANIME
const imdbIdMatch = details.find('a[href*="imdb.com"]').attr('href') const imdbIdMatch = details.find('a[href*="imdb.com"]').attr('href')
const torrent = magnets.map(magnetLink => { const torrents = magnets.map(magnetLink => {
const name = escapeHTML(decode(magnetLink).name.replace(/\+/g, ' ')) const originalName = details.find('strong:contains(\'Original\')').next().text().trim() ||
if (isPtDubbed(name) || isAnime) { details.find('b:contains(\'Original\'), strong:contains(\'Original\')')[0].nextSibling.nodeValue;
return { return {
title: sanitizePtName(name), title: sanitizePtName(escapeHTML(decode(magnetLink).name.replace(/\+/g, ' '))),
originalName: details.find('b:contains(\'Original\')')[0].nextSibling.nodeValue.replace(/: ?/, ''), originalName: originalName.replace(/: ?/, '').trim(),
year: details.find('a[href*="comando.to/category/"]').text(), year: details.find('a[href*="comando.to/category/"]').text(),
infoHash: decode(magnetLink).infoHash, infoHash: decode(magnetLink).infoHash,
magnetLink: magnetLink, magnetLink: magnetLink,
category: parseCategory($('div.entry-categories').html()), category: parseCategory($('div.entry-categories').html()),
uploadDate: new Date(moment($('a.updated').text(), 'LL', 'pt-br').format()), uploadDate: new Date(moment($('a.updated').text(), 'LL', 'pt-br').format()),
imdbId: imdbIdMatch ? imdbIdMatch.split('/')[4] : null, imdbId: imdbIdMatch ? imdbIdMatch.split('/')[4] : null,
languages: sanitizePtLanguages(details.find( languages: sanitizePtLanguages(details.find(
'b:contains(\'Idioma\'), b:contains(\'Audio\'), b:contains(\'Áudio\')')[0].nextSibling.nodeValue) 'b:contains(\'Idioma\'), b:contains(\'Audio\'), b:contains(\'Áudio\')')[0].nextSibling.nodeValue)
};
} }
}) });
resolve(torrent.filter((x) => x)); resolve(torrents.filter((x) => x));
}); });
} }

33
scraper/scrapers/comoeubaixo/comoeubaixo_api.js
View File

@@ -113,33 +113,28 @@ function parseTorrentPage(body) {
if (!$) { if (!$) {
reject(new Error('Failed loading body')); reject(new Error('Failed loading body'));
} }
let magnets = []; const magnets = $(`a[href^="magnet"]`)
$(`a[href^="magnet"]`).each((i, section) => { .filter((i, elem) => isPtDubbed($(elem).attr('title')))
let magnet = $(section).attr("href"); .map((i, elem) => $(elem).attr("href")).get();
magnets.push(magnet);
});
const details = $('div#informacoes') const details = $('div#informacoes')
const category = details.find('strong:contains(\'Gêneros: \')').next().attr('href').split('/')[0] const category = details.find('strong:contains(\'Gêneros: \')').next().attr('href').split('/')[0]
const isAnime = category === Categories.ANIME const torrents = magnets.map(magnetLink => {
const torrent = magnets.map(magnetLink => {
const name = escapeHTML(decode(magnetLink).name.replace(/\+/g, ' ')) const name = escapeHTML(decode(magnetLink).name.replace(/\+/g, ' '))
const sanitizedTitle = sanitizePtName(name); const sanitizedTitle = sanitizePtName(name);
const originalTitle = details.find('strong:contains(\'Baixar\')')[0].nextSibling.nodeValue.split('-')[0]; const originalTitle = details.find('strong:contains(\'Baixar\')')[0].nextSibling.nodeValue.split('-')[0];
const year = details.find('strong:contains(\'Data de Lançamento: \')').next().text().trim(); const year = details.find('strong:contains(\'Data de Lançamento: \')').next().text().trim();
const fallBackTitle = `${originalTitle.trim()} ${year.trim()} ${sanitizedTitle.trim()}`; const fallBackTitle = `${originalTitle.trim()} ${year.trim()} ${sanitizedTitle.trim()}`;
if (isPtDubbed(name) || isAnime) { return {
return { title: sanitizedTitle.length > 4 ? sanitizedTitle : fallBackTitle,
title: sanitizedTitle.length > 4 ? sanitizedTitle : fallBackTitle, infoHash: decode(magnetLink).infoHash,
infoHash: decode(magnetLink).infoHash, magnetLink: magnetLink,
magnetLink: magnetLink, category: category,
category: category, uploadDate: new Date($('time').attr('datetime')),
uploadDate: new Date($('time').attr('datetime')), imdbId: details.find('a[href*="imdb.com"]').attr('href').split('/')[4],
imdbId: details.find('a[href*="imdb.com"]').attr('href').split('/')[4], languages: sanitizePtLanguages(details.find('strong:contains(\'Idioma\')')[0].nextSibling.nodeValue)
languages: sanitizePtLanguages(details.find('strong:contains(\'Idioma\')')[0].nextSibling.nodeValue) };
};
}
}) })
resolve(torrent.filter((x) => x)); resolve(torrents.filter((x) => x));
}); });
} }

41
scraper/scrapers/lapumia/lapumia_api.js
View File

@@ -112,32 +112,25 @@ function parseTorrentPage(body) {
if (!$) { if (!$) {
reject(new Error('Failed loading body')); reject(new Error('Failed loading body'));
} }
let magnets = []; const magnets = $('h2 > span')
$(`a[href^="magnet"]`).each((i, section) => { .filter((i, elem) => isPtDubbed($(elem).text())).parent()
let magnet = $(section).attr("href"); .map((i, elem) => $(elem).nextUntil('h2, hr'))
magnets.push(magnet); .map((i, elem) => $(elem).find('a[href^="magnet"]'))
}); .map((i, section) => $(section).attr("href")).get();
const category = parseCategory($('div.category').html()); const category = parseCategory($('div.category').html());
const details = $('div.content') const details = $('div.content')
const isAnime = category === Categories.ANIME const torrents = magnets.map(magnetLink => ({
const torrent = magnets.map(magnetLink => { title: sanitizePtName(escapeHTML(decode(magnetLink).name.replace(/\+/g, ' '))),
const name = escapeHTML(decode(magnetLink).name.replace(/\+/g, ' ')) originalName: sanitizePtOriginalName(details.find('b:contains(\'Titulo Original:\')')[0].nextSibling.nodeValue),
if (isPtDubbed(name) || isAnime) { year: details.find('b:contains(\'Ano de Lançamento:\')')[0].nextSibling.nodeValue.trim(),
return { infoHash: decode(magnetLink).infoHash,
title: sanitizePtName(name), magnetLink: magnetLink,
originalName: sanitizePtOriginalName( category: category,
details.find('b:contains(\'Titulo Original:\')')[0].nextSibling.nodeValue), uploadDate: new Date(moment($('div.infos').text().split('•')[0].trim(), 'LL', 'pt-br').format()),
year: details.find('b:contains(\'Ano de Lançamento:\')')[0].nextSibling.nodeValue.trim(), imdbId: $('.imdbRatingPlugin').attr('data-title') || null,
infoHash: decode(magnetLink).infoHash, languages: sanitizePtLanguages(details.find('b:contains(\'Idioma\')')[0].nextSibling.nodeValue)
magnetLink: magnetLink, }))
category: category, resolve(torrents.filter((x) => x));
uploadDate: new Date(moment($('div.infos').text().split('•')[0].trim(), 'LL', 'pt-br').format()),
imdbId: $('.imdbRatingPlugin').attr('data-title') || null,
languages: sanitizePtLanguages(details.find('b:contains(\'Idioma\')')[0].nextSibling.nodeValue)
};
}
})
resolve(torrent.filter((x) => x));
}); });
} }

36
scraper/scrapers/ondebaixa/ondebaixa_api.js
View File

@@ -112,30 +112,22 @@ function parseTorrentPage(body) {
if (!$) { if (!$) {
reject(new Error('Failed loading body')); reject(new Error('Failed loading body'));
} }
let magnets = []; const magnets = $(`a[href^="magnet"]`)
$(`a[href^="magnet"]`).each((i, section) => { .filter((i, elem) => isPtDubbed($(elem).attr('title')))
let magnet = $(section).attr("href"); .map((i, elem) => $(elem).attr("href")).get();
magnets.push(magnet);
});
const details = $('div#informacoes') const details = $('div#informacoes')
const category = details.find('span:contains(\'Gêneros: \')').next().html() const category = details.find('span:contains(\'Gêneros: \')').next().html()
const isAnime = parseCategory(category) === Categories.ANIME const torrents = magnets.map(magnetLink => ({
const torrent = magnets.map(magnetLink => { title: sanitizePtName(escapeHTML(decode(magnetLink).name.replace(/\+/g, ' '))),
const name = escapeHTML(decode(magnetLink).name.replace(/\+/g, ' ')) originalName: sanitizePtOriginalName(details.find('span:contains(\'Título Original: \')').next().text()),
if (isPtDubbed(name) || isAnime) { year: details.find('span:contains(\'Ano de Lançamento: \')').next().text().trim(),
return { infoHash: decode(magnetLink).infoHash,
title: sanitizePtName(name), magnetLink: magnetLink,
originalName: sanitizePtOriginalName(details.find('span:contains(\'Título Original: \')').next().text()), category: parseCategory(category),
year: details.find('span:contains(\'Ano de Lançamento: \')').next().text().trim(), uploadDate: new Date($('time').attr('datetime')),
infoHash: decode(magnetLink).infoHash, languages: sanitizePtLanguages(details.find('span:contains(\'Idioma\')')[0].nextSibling.nodeValue)
magnetLink: magnetLink, }));
category: parseCategory(category), resolve(torrents.filter((x) => x));
uploadDate: new Date($('time').attr('datetime')),
languages: sanitizePtLanguages(details.find('span:contains(\'Idioma\')')[0].nextSibling.nodeValue)
};
}
})
resolve(torrent.filter((x) => x));
}); });
} }