tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Updated scrapers to latest available commit

Browse Source
This commit is contained in:
Gabisonfire
2024-01-17 16:43:58 -05:00
parent cab7f38c66
commit 909ade0d8e
39 changed files with 311 additions and 299 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
scraper/scrapers/comoeubaixo/comoeubaixo_api.js
View File

@@ -1,11 +1,11 @@
const needle = require("needle")
const axios = require('axios');
const cheerio = require("cheerio");
const decode = require('magnet-uri');
const { escapeHTML } = require('../../lib/metadata');
const { getRandomUserAgent } = require('../../lib/requestHelper');
const { isPtDubbed, sanitizePtName, sanitizePtLanguages } = require('../scraperHelper')
const defaultTimeout = 10000;
const defaultTimeout = 30000;
const maxSearchPage = 50
const baseUrl = 'https://comoeubaixo.com';
@@ -62,11 +62,11 @@ function browse(config = {}, retries = 2) {
function singleRequest(requestUrl, config = {}) {
const timeout = config.timeout || defaultTimeout;
const options = { userAgent: getRandomUserAgent(), open_timeout: timeout, follow: 2 };
const options = { headers: { 'User-Agent': getRandomUserAgent() }, timeout: timeout };
return needle('get', requestUrl, options)
return axios.get(requestUrl, options)
.then((response) => {
const body = response.body;
const body = response.data;
if (!body || (Buffer.isBuffer(body) && !body.size)) {
throw new Error(`No body: ${requestUrl}`);
} else if (body.includes('502: Bad gateway') ||
@@ -74,7 +74,8 @@ function singleRequest(requestUrl, config = {}) {
throw new Error(`Invalid body contents: ${requestUrl}`);
}
return body;
});
})
.catch(error => Promise.reject(error.message || error));
}
function parseTableBody(body) {
@@ -113,13 +114,12 @@ function parseTorrentPage(body) {
const category = details.find('strong:contains(\'Gêneros: \')').next().attr('href').split('/')[0]
const torrents = magnets.map(magnetLink => {
const decodedMagnet = decode(magnetLink);
const name = escapeHTML(decodedMagnet.name || '').replace(/\+/g, ' ');
const sanitizedTitle = sanitizePtName(name);
const name = sanitizePtName(escapeHTML(decodedMagnet.name || '').replace(/\+/g, ' '));
const originalTitle = details.find('strong:contains(\'Baixar\')')[0].nextSibling.nodeValue.split('-')[0];
const year = details.find('strong:contains(\'Data de Lançamento: \')').next().text().trim();
const fallBackTitle = `${originalTitle.trim()} ${year.trim()} ${sanitizedTitle.trim()}`;
const fallBackTitle = `${originalTitle.trim()} ${year.trim()} ${name.trim()}`;
return {
title: sanitizedTitle.length > 5 ? sanitizedTitle : fallBackTitle,
title: name.length > 5 ? name : fallBackTitle,
infoHash: decodedMagnet.infoHash,
magnetLink: magnetLink,
category: category,