tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

change proxy list config to base url

Browse Source
This commit is contained in:
TheBeastLT
2021-09-15 09:05:26 +02:00
committed by TheBeastLT
parent aaca34b2d6
commit e1a75e4d9a
6 changed files with 29 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
scraper/scrapers/animestorrent/animestorrent_api.js
View File

@@ -7,9 +7,7 @@ const { getRandomUserAgent } = require("../../lib/requestHelper");
const defaultTimeout = 10000; const defaultTimeout = 10000;
const maxSearchPage = 50; const maxSearchPage = 50;
const defaultProxies = [ const baseUrl = 'https://animestorrent.com';
'https://animestorrent.com'
];
const Categories = { const Categories = {
MOVIE: 'filme', MOVIE: 'filme',
@@ -21,9 +19,8 @@ function torrent(torrentId, config = {}, retries = 2) {
if (!torrentId || retries === 0) { if (!torrentId || retries === 0) {
return Promise.reject(new Error(`Failed ${torrentId} query`)); return Promise.reject(new Error(`Failed ${torrentId} query`));
} }
const proxyList = config.proxyList || defaultProxies;
const slug = torrentId.split("/")[3]; const slug = torrentId.split("/")[3];
return Promises.first(proxyList.map((proxyUrl) => singleRequest(`${proxyUrl}/${slug}`, config))) return singleRequest(`${baseUrl}/${slug}`, config)
.then((body) => parseTorrentPage(body)) .then((body) => parseTorrentPage(body))
.then((torrent) => torrent.map((el) => ({ torrentId: slug, ...el }))) .then((torrent) => torrent.map((el) => ({ torrentId: slug, ...el })))
.catch((err) => torrent(slug, config, retries - 1)); .catch((err) => torrent(slug, config, retries - 1));
@@ -33,12 +30,10 @@ function search(keyword, config = {}, retries = 2) {
if (!keyword || retries === 0) { if (!keyword || retries === 0) {
return Promise.reject(new Error(`Failed ${keyword} search`)); return Promise.reject(new Error(`Failed ${keyword} search`));
} }
const proxyList = config.proxyList || defaultProxies;
const page = config.page || 1; const page = config.page || 1;
const extendToPage = Math.min(maxSearchPage, config.extendToPage || 1); const extendToPage = Math.min(maxSearchPage, config.extendToPage || 1);
const requestUrl = (proxyUrl) => `${proxyUrl}/page/${page}/?s=${keyword}`;
return Promises.first(proxyList.map((proxyUrl) => singleRequest(requestUrl(proxyUrl), config))) return singleRequest(`${baseUrl}/page/${page}/?s=${keyword}`, config)
.then((body) => parseTableBody(body)) .then((body) => parseTableBody(body))
.then((torrents) => .then((torrents) =>
torrents.length === 40 && page < extendToPage torrents.length === 40 && page < extendToPage
@@ -53,15 +48,11 @@ function browse(config = {}, retries = 2) {
if (retries === 0) { if (retries === 0) {
return Promise.reject(new Error(`Failed browse request`)); return Promise.reject(new Error(`Failed browse request`));
} }
const proxyList = config.proxyList || defaultProxies;
const page = config.page || 1; const page = config.page || 1;
const category = config.category; const category = config.category;
const requestUrl = (proxyUrl) => const requestUrl = category ? `${baseUrl}/tipo/${category}/page/${page}/` : `${baseUrl}/page/${page}/`;
category
? `${proxyUrl}/tipo/${category}/page/${page}/`
: `${proxyUrl}/page/${page}/`;
return Promises.first(proxyList.map((proxyUrl) => singleRequest(requestUrl(proxyUrl), config))) return singleRequest(requestUrl, config)
.then((body) => parseTableBody(body)) .then((body) => parseTableBody(body))
.catch((err) => browse(config, retries - 1)); .catch((err) => browse(config, retries - 1));
} }

18
scraper/scrapers/comando/comando_api.js
View File

@@ -10,9 +10,7 @@ const { isPtDubbed, sanitizePtName, sanitizePtLanguages } = require('../scraperH
const defaultTimeout = 30000; const defaultTimeout = 30000;
const maxSearchPage = 50 const maxSearchPage = 50
const defaultProxies = [ const baseUrl = 'https://comando.to';
'https://comando.to'
];
const Categories = { const Categories = {
MOVIE: 'filmes', MOVIE: 'filmes',
@@ -25,10 +23,8 @@ function torrent(torrentId, config = {}, retries = 2) {
if (!torrentId || retries === 0) { if (!torrentId || retries === 0) {
return Promise.reject(new Error(`Failed ${torrentId} query`)); return Promise.reject(new Error(`Failed ${torrentId} query`));
} }
const proxyList = config.proxyList || defaultProxies;
const slug = torrentId.split("/")[3]; const slug = torrentId.split("/")[3];
return Promises.first(proxyList return singleRequest(`${baseUrl}/${slug}`, config)
.map((proxyUrl) => singleRequest(`${proxyUrl}/${slug}`, config)))
.then((body) => parseTorrentPage(body)) .then((body) => parseTorrentPage(body))
.then((torrent) => torrent.map(el => ({ torrentId: slug, ...el }))) .then((torrent) => torrent.map(el => ({ torrentId: slug, ...el })))
.catch((err) => { .catch((err) => {
@@ -41,13 +37,10 @@ function search(keyword, config = {}, retries = 2) {
if (!keyword || retries === 0) { if (!keyword || retries === 0) {
return Promise.reject(new Error(`Failed ${keyword} search`)); return Promise.reject(new Error(`Failed ${keyword} search`));
} }
const proxyList = config.proxyList || defaultProxies;
const page = config.page || 1; const page = config.page || 1;
const extendToPage = Math.min(maxSearchPage, (config.extendToPage || 1)) const extendToPage = Math.min(maxSearchPage, (config.extendToPage || 1))
const requestUrl = proxyUrl => `${proxyUrl}/page/${page}/?s=${keyword}`
return Promises.first(proxyList return singleRequest(`${baseUrl}/page/${page}/?s=${keyword}`, config)
.map(proxyUrl => singleRequest(requestUrl(proxyUrl), config)))
.then(body => parseTableBody(body)) .then(body => parseTableBody(body))
.then(torrents => torrents.length === 40 && page < extendToPage .then(torrents => torrents.length === 40 && page < extendToPage
? search(keyword, { ...config, page: page + 1 }).catch(() => []) ? search(keyword, { ...config, page: page + 1 }).catch(() => [])
@@ -60,13 +53,10 @@ function browse(config = {}, retries = 2) {
if (retries === 0) { if (retries === 0) {
return Promise.reject(new Error(`Failed browse request`)); return Promise.reject(new Error(`Failed browse request`));
} }
const proxyList = config.proxyList || defaultProxies;
const page = config.page || 1; const page = config.page || 1;
const category = config.category; const category = config.category;
const requestUrl = proxyUrl => `${proxyUrl}/category/${category}/page/${page}/`
return Promises.first(proxyList return singleRequest(`${baseUrl}/category/${category}/page/${page}/`, config)
.map((proxyUrl) => singleRequest(requestUrl(proxyUrl), config)))
.then((body) => parseTableBody(body)) .then((body) => parseTableBody(body))
.catch((err) => browse(config, retries - 1)); .catch((err) => browse(config, retries - 1));
} }

19
scraper/scrapers/comoeubaixo/comoeubaixo_api.js
View File

@@ -9,9 +9,7 @@ const { isPtDubbed, sanitizePtName, sanitizePtLanguages } = require('../scraperH
const defaultTimeout = 10000; const defaultTimeout = 10000;
const maxSearchPage = 50 const maxSearchPage = 50
const defaultProxies = [ const baseUrl = 'https://comoeubaixo.com';
'https://comoeubaixo.com'
];
const Categories = { const Categories = {
MOVIE: 'filmes', MOVIE: 'filmes',
@@ -24,10 +22,8 @@ function torrent(torrentId, config = {}, retries = 2) {
if (!torrentId || retries === 0) { if (!torrentId || retries === 0) {
return Promise.reject(new Error(`Failed ${torrentId} query`)); return Promise.reject(new Error(`Failed ${torrentId} query`));
} }
const proxyList = config.proxyList || defaultProxies;
const slug = torrentId.split("/")[3]; const slug = torrentId.split("/")[3];
return Promises.first(proxyList return singleRequest(`${baseUrl}/${slug}/`, config)
.map((proxyUrl) => singleRequest(`${proxyUrl}/${slug}/`, config)))
.then((body) => parseTorrentPage(body)) .then((body) => parseTorrentPage(body))
.then((torrent) => torrent.map(el => ({ torrentId: slug, ...el }))) .then((torrent) => torrent.map(el => ({ torrentId: slug, ...el })))
.catch((err) => torrent(slug, config, retries - 1)); .catch((err) => torrent(slug, config, retries - 1));
@@ -37,13 +33,10 @@ function search(keyword, config = {}, retries = 2) {
if (!keyword || retries === 0) { if (!keyword || retries === 0) {
return Promise.reject(new Error(`Failed ${keyword} search`)); return Promise.reject(new Error(`Failed ${keyword} search`));
} }
const proxyList = config.proxyList || defaultProxies;
const page = config.page || 1; const page = config.page || 1;
const extendToPage = Math.min(maxSearchPage, (config.extendToPage || 1)) const extendToPage = Math.min(maxSearchPage, (config.extendToPage || 1))
const requestUrl = proxyUrl => `${proxyUrl}/${keyword}/${page}/`
return Promises.first(proxyList return singleRequest(`${baseUrl}/${keyword}/${page}/`, config)
.map(proxyUrl => singleRequest(requestUrl(proxyUrl), config)))
.then(body => parseTableBody(body)) .then(body => parseTableBody(body))
.then(torrents => torrents.length === 40 && page < extendToPage .then(torrents => torrents.length === 40 && page < extendToPage
? search(keyword, { ...config, page: page + 1 }).catch(() => []) ? search(keyword, { ...config, page: page + 1 }).catch(() => [])
@@ -56,13 +49,11 @@ function browse(config = {}, retries = 2) {
if (retries === 0) { if (retries === 0) {
return Promise.reject(new Error(`Failed browse request`)); return Promise.reject(new Error(`Failed browse request`));
} }
const proxyList = config.proxyList || defaultProxies;
const page = config.page || 1; const page = config.page || 1;
const category = config.category; const category = config.category;
const requestUrl = proxyUrl => category ? `${proxyUrl}/${category}/${page}/` : `${proxyUrl}/${page}/`; const requestUrl = category ? `${baseUrl}/${category}/${page}/` : `${baseUrl}/${page}/`;
return Promises.first(proxyList return singleRequest(requestUrl, config)
.map((proxyUrl) => singleRequest(requestUrl(proxyUrl), config)))
.then((body) => parseTableBody(body)) .then((body) => parseTableBody(body))
.catch((err) => browse(config, retries - 1)); .catch((err) => browse(config, retries - 1));
} }

24
scraper/scrapers/darkmahou/darkmahou_api.js
View File

@@ -1,5 +1,4 @@
const needle = require("needle"); const needle = require("needle");
const Bottleneck = require("bottleneck");
const cheerio = require("cheerio"); const cheerio = require("cheerio");
const decode = require("magnet-uri"); const decode = require("magnet-uri");
const Promises = require("../../lib/promises"); const Promises = require("../../lib/promises");
@@ -8,7 +7,7 @@ const { getRandomUserAgent } = require("../../lib/requestHelper");
const defaultTimeout = 10000; const defaultTimeout = 10000;
const maxSearchPage = 50; const maxSearchPage = 50;
const defaultProxies = ['https://darkmahou.com']; const baseUrl = 'https://darkmahou.com';
const Categories = { const Categories = {
MOVIE: 'movie', MOVIE: 'movie',
@@ -20,11 +19,8 @@ function torrent(torrentId, config = {}, retries = 2) {
if (!torrentId || retries === 0) { if (!torrentId || retries === 0) {
return Promise.reject(new Error(`Failed ${torrentId} query`)); return Promise.reject(new Error(`Failed ${torrentId} query`));
} }
const proxyList = config.proxyList || defaultProxies;
const slug = torrentId.split("/")[3]; const slug = torrentId.split("/")[3];
return Promises.first( return singleRequest(`${baseUrl}/${slug}`, config)
proxyList.map((proxyUrl) => singleRequest(`${proxyUrl}/${slug}`, config))
)
.then((body) => parseTorrentPage(body)) .then((body) => parseTorrentPage(body))
.then((torrent) => torrent.map((el) => ({ torrentId: slug, ...el }))) .then((torrent) => torrent.map((el) => ({ torrentId: slug, ...el })))
.catch((err) => torrent(slug, config, retries - 1)); .catch((err) => torrent(slug, config, retries - 1));
@@ -34,14 +30,10 @@ function search(keyword, config = {}, retries = 2) {
if (!keyword || retries === 0) { if (!keyword || retries === 0) {
return Promise.reject(new Error(`Failed ${keyword} search`)); return Promise.reject(new Error(`Failed ${keyword} search`));
} }
const proxyList = config.proxyList || defaultProxies;
const page = config.page || 1; const page = config.page || 1;
const extendToPage = Math.min(maxSearchPage, config.extendToPage || 1); const extendToPage = Math.min(maxSearchPage, config.extendToPage || 1);
const requestUrl = (proxyUrl) => `${proxyUrl}/page/${page}/?s=${keyword}`;
return Promises.first( return singleRequest(`${baseUrl}/page/${page}/?s=${keyword}`, config)
proxyList.map((proxyUrl) => singleRequest(requestUrl(proxyUrl), config))
)
.then((body) => parseTableBody(body)) .then((body) => parseTableBody(body))
.then((torrents) => .then((torrents) =>
torrents.length === 40 && page < extendToPage torrents.length === 40 && page < extendToPage
@@ -57,17 +49,11 @@ function browse(config = {}, retries = 2) {
if (retries === 0) { if (retries === 0) {
return Promise.reject(new Error(`Failed browse request`)); return Promise.reject(new Error(`Failed browse request`));
} }
const proxyList = config.proxyList || defaultProxies;
const page = config.page || 1; const page = config.page || 1;
const category = config.category; const category = config.category;
const requestUrl = (proxyUrl) => const requestUrl = category ? `${baseUrl}/category/${category}/page/${page}/` : `${baseUrl}/page/${page}/`;
category
? `${proxyUrl}/category/${category}/page/${page}/`
: `${proxyUrl}/page/${page}/`;
return Promises.first( return singleRequest(requestUrl, config)
proxyList.map((proxyUrl) => singleRequest(requestUrl(proxyUrl), config))
)
.then((body) => parseTableBody(body)) .then((body) => parseTableBody(body))
.catch((err) => browse(config, retries - 1)); .catch((err) => browse(config, retries - 1));
} }

19
scraper/scrapers/lapumia/lapumia_api.js
View File

@@ -10,9 +10,7 @@ const { isPtDubbed, sanitizePtName, sanitizePtOriginalName, sanitizePtLanguages
const defaultTimeout = 10000; const defaultTimeout = 10000;
const maxSearchPage = 50 const maxSearchPage = 50
const defaultProxies = [ const baseUrl = 'https://lapumia.org';
'https://lapumia.org'
];
const Categories = { const Categories = {
MOVIE: null, MOVIE: null,
@@ -24,10 +22,8 @@ function torrent(torrentId, config = {}, retries = 2) {
if (!torrentId || retries === 0) { if (!torrentId || retries === 0) {
return Promise.reject(new Error(`Failed ${torrentId} query`)); return Promise.reject(new Error(`Failed ${torrentId} query`));
} }
const proxyList = config.proxyList || defaultProxies;
const slug = torrentId.split('?p=')[1]; const slug = torrentId.split('?p=')[1];
return Promises.first(proxyList return singleRequest(`${baseUrl}/?p=${slug}`, config)
.map((proxyUrl) => singleRequest(`${proxyUrl}/?p=${slug}`, config)))
.then((body) => parseTorrentPage(body)) .then((body) => parseTorrentPage(body))
.then((torrent) => torrent.map(el => ({ torrentId: slug, ...el }))) .then((torrent) => torrent.map(el => ({ torrentId: slug, ...el })))
.catch((err) => torrent(slug, config, retries - 1)); .catch((err) => torrent(slug, config, retries - 1));
@@ -37,13 +33,10 @@ function search(keyword, config = {}, retries = 2) {
if (!keyword || retries === 0) { if (!keyword || retries === 0) {
return Promise.reject(new Error(`Failed ${keyword} search`)); return Promise.reject(new Error(`Failed ${keyword} search`));
} }
const proxyList = config.proxyList || defaultProxies;
const page = config.page || 1; const page = config.page || 1;
const extendToPage = Math.min(maxSearchPage, (config.extendToPage || 1)) const extendToPage = Math.min(maxSearchPage, (config.extendToPage || 1))
const requestUrl = proxyUrl => `${proxyUrl}/page/${page}/?s=${keyword}`
return Promises.first(proxyList return singleRequest(`${baseUrl}/page/${page}/?s=${keyword}`, config)
.map(proxyUrl => singleRequest(requestUrl(proxyUrl), config)))
.then(body => parseTableBody(body)) .then(body => parseTableBody(body))
.then(torrents => torrents.length === 10 && page < extendToPage .then(torrents => torrents.length === 10 && page < extendToPage
? search(keyword, { ...config, page: page + 1 }).catch(() => []) ? search(keyword, { ...config, page: page + 1 }).catch(() => [])
@@ -56,13 +49,11 @@ function browse(config = {}, retries = 2) {
if (retries === 0) { if (retries === 0) {
return Promise.reject(new Error(`Failed browse request`)); return Promise.reject(new Error(`Failed browse request`));
} }
const proxyList = config.proxyList || defaultProxies;
const page = config.page || 1; const page = config.page || 1;
const category = config.category; const category = config.category;
const requestUrl = proxyUrl => category ? `${proxyUrl}/${category}/page/${page}/` : `${proxyUrl}/page/${page}/` const requestUrl = category ? `${baseUrl}/${category}/page/${page}/` : `${baseUrl}/page/${page}/`
return Promises.first(proxyList return singleRequest(requestUrl, config)
.map((proxyUrl) => singleRequest(requestUrl(proxyUrl), config)))
.then((body) => parseTableBody(body)) .then((body) => parseTableBody(body))
.catch((err) => browse(config, retries - 1)); .catch((err) => browse(config, retries - 1));
} }

19
scraper/scrapers/ondebaixa/ondebaixa_api.js
View File

@@ -9,9 +9,7 @@ const { isPtDubbed, sanitizePtName, sanitizePtOriginalName, sanitizePtLanguages
const defaultTimeout = 10000; const defaultTimeout = 10000;
const maxSearchPage = 50 const maxSearchPage = 50
const defaultProxies = [ const baseUrl = 'https://ondebaixa.com';
'https://ondebaixa.com'
];
const Categories = { const Categories = {
MOVIE: 'filmes', MOVIE: 'filmes',
@@ -24,10 +22,8 @@ function torrent(torrentId, config = {}, retries = 2) {
if (!torrentId || retries === 0) { if (!torrentId || retries === 0) {
return Promise.reject(new Error(`Failed ${torrentId} query`)); return Promise.reject(new Error(`Failed ${torrentId} query`));
} }
const proxyList = config.proxyList || defaultProxies;
const slug = torrentId.split("/")[3]; const slug = torrentId.split("/")[3];
return Promises.first(proxyList return singleRequest(`${baseUrl}/${slug}/`, config)
.map((proxyUrl) => singleRequest(`${proxyUrl}/${slug}/`, config)))
.then((body) => parseTorrentPage(body)) .then((body) => parseTorrentPage(body))
.then((torrent) => torrent.map(el => ({ torrentId: slug, ...el }))) .then((torrent) => torrent.map(el => ({ torrentId: slug, ...el })))
.catch((err) => torrent(slug, config, retries - 1)); .catch((err) => torrent(slug, config, retries - 1));
@@ -37,13 +33,10 @@ function search(keyword, config = {}, retries = 2) {
if (!keyword || retries === 0) { if (!keyword || retries === 0) {
return Promise.reject(new Error(`Failed ${keyword} search`)); return Promise.reject(new Error(`Failed ${keyword} search`));
} }
const proxyList = config.proxyList || defaultProxies;
const page = config.page || 1; const page = config.page || 1;
const extendToPage = Math.min(maxSearchPage, (config.extendToPage || 1)) const extendToPage = Math.min(maxSearchPage, (config.extendToPage || 1))
const requestUrl = proxyUrl => `${proxyUrl}/${keyword}/${page}/`
return Promises.first(proxyList return singleRequest(`${baseUrl}/${keyword}/${page}/`, config)
.map(proxyUrl => singleRequest(requestUrl(proxyUrl), config)))
.then(body => parseTableBody(body)) .then(body => parseTableBody(body))
.then(torrents => torrents.length === 40 && page < extendToPage .then(torrents => torrents.length === 40 && page < extendToPage
? search(keyword, { ...config, page: page + 1 }).catch(() => []) ? search(keyword, { ...config, page: page + 1 }).catch(() => [])
@@ -56,13 +49,11 @@ function browse(config = {}, retries = 2) {
if (retries === 0) { if (retries === 0) {
return Promise.reject(new Error(`Failed browse request`)); return Promise.reject(new Error(`Failed browse request`));
} }
const proxyList = config.proxyList || defaultProxies;
const page = config.page || 1; const page = config.page || 1;
const category = config.category; const category = config.category;
const requestUrl = proxyUrl => category ? `${proxyUrl}/${category}/${page}/` : `${proxyUrl}/${page}/`; const requestUrl = category ? `${baseUrl}/${category}/${page}/` : `${baseUrl}/${page}/`;
return Promises.first(proxyList return singleRequest(requestUrl, config)
.map((proxyUrl) => singleRequest(requestUrl(proxyUrl), config)))
.then((body) => parseTableBody(body)) .then((body) => parseTableBody(body))
.catch((err) => browse(config, retries - 1)); .catch((err) => browse(config, retries - 1));
} }