diff --git a/scraper/scrapers/animestorrent/animestorrent_api.js b/scraper/scrapers/animestorrent/animestorrent_api.js index 860d528..836f8f6 100644 --- a/scraper/scrapers/animestorrent/animestorrent_api.js +++ b/scraper/scrapers/animestorrent/animestorrent_api.js @@ -7,9 +7,7 @@ const { getRandomUserAgent } = require("../../lib/requestHelper"); const defaultTimeout = 10000; const maxSearchPage = 50; -const defaultProxies = [ - 'https://animestorrent.com' -]; +const baseUrl = 'https://animestorrent.com'; const Categories = { MOVIE: 'filme', @@ -21,9 +19,8 @@ function torrent(torrentId, config = {}, retries = 2) { if (!torrentId || retries === 0) { return Promise.reject(new Error(`Failed ${torrentId} query`)); } - const proxyList = config.proxyList || defaultProxies; const slug = torrentId.split("/")[3]; - return Promises.first(proxyList.map((proxyUrl) => singleRequest(`${proxyUrl}/${slug}`, config))) + return singleRequest(`${baseUrl}/${slug}`, config) .then((body) => parseTorrentPage(body)) .then((torrent) => torrent.map((el) => ({ torrentId: slug, ...el }))) .catch((err) => torrent(slug, config, retries - 1)); @@ -33,12 +30,10 @@ function search(keyword, config = {}, retries = 2) { if (!keyword || retries === 0) { return Promise.reject(new Error(`Failed ${keyword} search`)); } - const proxyList = config.proxyList || defaultProxies; const page = config.page || 1; const extendToPage = Math.min(maxSearchPage, config.extendToPage || 1); - const requestUrl = (proxyUrl) => `${proxyUrl}/page/${page}/?s=${keyword}`; - return Promises.first(proxyList.map((proxyUrl) => singleRequest(requestUrl(proxyUrl), config))) + return singleRequest(`${baseUrl}/page/${page}/?s=${keyword}`, config) .then((body) => parseTableBody(body)) .then((torrents) => torrents.length === 40 && page < extendToPage @@ -53,15 +48,11 @@ function browse(config = {}, retries = 2) { if (retries === 0) { return Promise.reject(new Error(`Failed browse request`)); } - const proxyList = config.proxyList || defaultProxies; const page = config.page || 1; const category = config.category; - const requestUrl = (proxyUrl) => - category - ? `${proxyUrl}/tipo/${category}/page/${page}/` - : `${proxyUrl}/page/${page}/`; + const requestUrl = category ? `${baseUrl}/tipo/${category}/page/${page}/` : `${baseUrl}/page/${page}/`; - return Promises.first(proxyList.map((proxyUrl) => singleRequest(requestUrl(proxyUrl), config))) + return singleRequest(requestUrl, config) .then((body) => parseTableBody(body)) .catch((err) => browse(config, retries - 1)); } diff --git a/scraper/scrapers/comando/comando_api.js b/scraper/scrapers/comando/comando_api.js index 485973c..21d00a4 100644 --- a/scraper/scrapers/comando/comando_api.js +++ b/scraper/scrapers/comando/comando_api.js @@ -10,9 +10,7 @@ const { isPtDubbed, sanitizePtName, sanitizePtLanguages } = require('../scraperH const defaultTimeout = 30000; const maxSearchPage = 50 -const defaultProxies = [ - 'https://comando.to' -]; +const baseUrl = 'https://comando.to'; const Categories = { MOVIE: 'filmes', @@ -25,10 +23,8 @@ function torrent(torrentId, config = {}, retries = 2) { if (!torrentId || retries === 0) { return Promise.reject(new Error(`Failed ${torrentId} query`)); } - const proxyList = config.proxyList || defaultProxies; const slug = torrentId.split("/")[3]; - return Promises.first(proxyList - .map((proxyUrl) => singleRequest(`${proxyUrl}/${slug}`, config))) + return singleRequest(`${baseUrl}/${slug}`, config) .then((body) => parseTorrentPage(body)) .then((torrent) => torrent.map(el => ({ torrentId: slug, ...el }))) .catch((err) => { @@ -41,13 +37,10 @@ function search(keyword, config = {}, retries = 2) { if (!keyword || retries === 0) { return Promise.reject(new Error(`Failed ${keyword} search`)); } - const proxyList = config.proxyList || defaultProxies; const page = config.page || 1; const extendToPage = Math.min(maxSearchPage, (config.extendToPage || 1)) - const requestUrl = proxyUrl => `${proxyUrl}/page/${page}/?s=${keyword}` - return Promises.first(proxyList - .map(proxyUrl => singleRequest(requestUrl(proxyUrl), config))) + return singleRequest(`${baseUrl}/page/${page}/?s=${keyword}`, config) .then(body => parseTableBody(body)) .then(torrents => torrents.length === 40 && page < extendToPage ? search(keyword, { ...config, page: page + 1 }).catch(() => []) @@ -60,13 +53,10 @@ function browse(config = {}, retries = 2) { if (retries === 0) { return Promise.reject(new Error(`Failed browse request`)); } - const proxyList = config.proxyList || defaultProxies; const page = config.page || 1; const category = config.category; - const requestUrl = proxyUrl => `${proxyUrl}/category/${category}/page/${page}/` - return Promises.first(proxyList - .map((proxyUrl) => singleRequest(requestUrl(proxyUrl), config))) + return singleRequest(`${baseUrl}/category/${category}/page/${page}/`, config) .then((body) => parseTableBody(body)) .catch((err) => browse(config, retries - 1)); } diff --git a/scraper/scrapers/comoeubaixo/comoeubaixo_api.js b/scraper/scrapers/comoeubaixo/comoeubaixo_api.js index b476f30..c108f15 100644 --- a/scraper/scrapers/comoeubaixo/comoeubaixo_api.js +++ b/scraper/scrapers/comoeubaixo/comoeubaixo_api.js @@ -9,9 +9,7 @@ const { isPtDubbed, sanitizePtName, sanitizePtLanguages } = require('../scraperH const defaultTimeout = 10000; const maxSearchPage = 50 -const defaultProxies = [ - 'https://comoeubaixo.com' -]; +const baseUrl = 'https://comoeubaixo.com'; const Categories = { MOVIE: 'filmes', @@ -24,10 +22,8 @@ function torrent(torrentId, config = {}, retries = 2) { if (!torrentId || retries === 0) { return Promise.reject(new Error(`Failed ${torrentId} query`)); } - const proxyList = config.proxyList || defaultProxies; const slug = torrentId.split("/")[3]; - return Promises.first(proxyList - .map((proxyUrl) => singleRequest(`${proxyUrl}/${slug}/`, config))) + return singleRequest(`${baseUrl}/${slug}/`, config) .then((body) => parseTorrentPage(body)) .then((torrent) => torrent.map(el => ({ torrentId: slug, ...el }))) .catch((err) => torrent(slug, config, retries - 1)); @@ -37,13 +33,10 @@ function search(keyword, config = {}, retries = 2) { if (!keyword || retries === 0) { return Promise.reject(new Error(`Failed ${keyword} search`)); } - const proxyList = config.proxyList || defaultProxies; const page = config.page || 1; const extendToPage = Math.min(maxSearchPage, (config.extendToPage || 1)) - const requestUrl = proxyUrl => `${proxyUrl}/${keyword}/${page}/` - return Promises.first(proxyList - .map(proxyUrl => singleRequest(requestUrl(proxyUrl), config))) + return singleRequest(`${baseUrl}/${keyword}/${page}/`, config) .then(body => parseTableBody(body)) .then(torrents => torrents.length === 40 && page < extendToPage ? search(keyword, { ...config, page: page + 1 }).catch(() => []) @@ -56,13 +49,11 @@ function browse(config = {}, retries = 2) { if (retries === 0) { return Promise.reject(new Error(`Failed browse request`)); } - const proxyList = config.proxyList || defaultProxies; const page = config.page || 1; const category = config.category; - const requestUrl = proxyUrl => category ? `${proxyUrl}/${category}/${page}/` : `${proxyUrl}/${page}/`; + const requestUrl = category ? `${baseUrl}/${category}/${page}/` : `${baseUrl}/${page}/`; - return Promises.first(proxyList - .map((proxyUrl) => singleRequest(requestUrl(proxyUrl), config))) + return singleRequest(requestUrl, config) .then((body) => parseTableBody(body)) .catch((err) => browse(config, retries - 1)); } diff --git a/scraper/scrapers/darkmahou/darkmahou_api.js b/scraper/scrapers/darkmahou/darkmahou_api.js index dc96576..e1d5a22 100644 --- a/scraper/scrapers/darkmahou/darkmahou_api.js +++ b/scraper/scrapers/darkmahou/darkmahou_api.js @@ -1,5 +1,4 @@ const needle = require("needle"); -const Bottleneck = require("bottleneck"); const cheerio = require("cheerio"); const decode = require("magnet-uri"); const Promises = require("../../lib/promises"); @@ -8,7 +7,7 @@ const { getRandomUserAgent } = require("../../lib/requestHelper"); const defaultTimeout = 10000; const maxSearchPage = 50; -const defaultProxies = ['https://darkmahou.com']; +const baseUrl = 'https://darkmahou.com'; const Categories = { MOVIE: 'movie', @@ -20,11 +19,8 @@ function torrent(torrentId, config = {}, retries = 2) { if (!torrentId || retries === 0) { return Promise.reject(new Error(`Failed ${torrentId} query`)); } - const proxyList = config.proxyList || defaultProxies; const slug = torrentId.split("/")[3]; - return Promises.first( - proxyList.map((proxyUrl) => singleRequest(`${proxyUrl}/${slug}`, config)) - ) + return singleRequest(`${baseUrl}/${slug}`, config) .then((body) => parseTorrentPage(body)) .then((torrent) => torrent.map((el) => ({ torrentId: slug, ...el }))) .catch((err) => torrent(slug, config, retries - 1)); @@ -34,14 +30,10 @@ function search(keyword, config = {}, retries = 2) { if (!keyword || retries === 0) { return Promise.reject(new Error(`Failed ${keyword} search`)); } - const proxyList = config.proxyList || defaultProxies; const page = config.page || 1; const extendToPage = Math.min(maxSearchPage, config.extendToPage || 1); - const requestUrl = (proxyUrl) => `${proxyUrl}/page/${page}/?s=${keyword}`; - return Promises.first( - proxyList.map((proxyUrl) => singleRequest(requestUrl(proxyUrl), config)) - ) + return singleRequest(`${baseUrl}/page/${page}/?s=${keyword}`, config) .then((body) => parseTableBody(body)) .then((torrents) => torrents.length === 40 && page < extendToPage @@ -57,17 +49,11 @@ function browse(config = {}, retries = 2) { if (retries === 0) { return Promise.reject(new Error(`Failed browse request`)); } - const proxyList = config.proxyList || defaultProxies; const page = config.page || 1; const category = config.category; - const requestUrl = (proxyUrl) => - category - ? `${proxyUrl}/category/${category}/page/${page}/` - : `${proxyUrl}/page/${page}/`; + const requestUrl = category ? `${baseUrl}/category/${category}/page/${page}/` : `${baseUrl}/page/${page}/`; - return Promises.first( - proxyList.map((proxyUrl) => singleRequest(requestUrl(proxyUrl), config)) - ) + return singleRequest(requestUrl, config) .then((body) => parseTableBody(body)) .catch((err) => browse(config, retries - 1)); } diff --git a/scraper/scrapers/lapumia/lapumia_api.js b/scraper/scrapers/lapumia/lapumia_api.js index 6c45dd5..080d2f6 100644 --- a/scraper/scrapers/lapumia/lapumia_api.js +++ b/scraper/scrapers/lapumia/lapumia_api.js @@ -10,9 +10,7 @@ const { isPtDubbed, sanitizePtName, sanitizePtOriginalName, sanitizePtLanguages const defaultTimeout = 10000; const maxSearchPage = 50 -const defaultProxies = [ - 'https://lapumia.org' -]; +const baseUrl = 'https://lapumia.org'; const Categories = { MOVIE: null, @@ -24,10 +22,8 @@ function torrent(torrentId, config = {}, retries = 2) { if (!torrentId || retries === 0) { return Promise.reject(new Error(`Failed ${torrentId} query`)); } - const proxyList = config.proxyList || defaultProxies; const slug = torrentId.split('?p=')[1]; - return Promises.first(proxyList - .map((proxyUrl) => singleRequest(`${proxyUrl}/?p=${slug}`, config))) + return singleRequest(`${baseUrl}/?p=${slug}`, config) .then((body) => parseTorrentPage(body)) .then((torrent) => torrent.map(el => ({ torrentId: slug, ...el }))) .catch((err) => torrent(slug, config, retries - 1)); @@ -37,13 +33,10 @@ function search(keyword, config = {}, retries = 2) { if (!keyword || retries === 0) { return Promise.reject(new Error(`Failed ${keyword} search`)); } - const proxyList = config.proxyList || defaultProxies; const page = config.page || 1; const extendToPage = Math.min(maxSearchPage, (config.extendToPage || 1)) - const requestUrl = proxyUrl => `${proxyUrl}/page/${page}/?s=${keyword}` - return Promises.first(proxyList - .map(proxyUrl => singleRequest(requestUrl(proxyUrl), config))) + return singleRequest(`${baseUrl}/page/${page}/?s=${keyword}`, config) .then(body => parseTableBody(body)) .then(torrents => torrents.length === 10 && page < extendToPage ? search(keyword, { ...config, page: page + 1 }).catch(() => []) @@ -56,13 +49,11 @@ function browse(config = {}, retries = 2) { if (retries === 0) { return Promise.reject(new Error(`Failed browse request`)); } - const proxyList = config.proxyList || defaultProxies; const page = config.page || 1; const category = config.category; - const requestUrl = proxyUrl => category ? `${proxyUrl}/${category}/page/${page}/` : `${proxyUrl}/page/${page}/` + const requestUrl = category ? `${baseUrl}/${category}/page/${page}/` : `${baseUrl}/page/${page}/` - return Promises.first(proxyList - .map((proxyUrl) => singleRequest(requestUrl(proxyUrl), config))) + return singleRequest(requestUrl, config) .then((body) => parseTableBody(body)) .catch((err) => browse(config, retries - 1)); } diff --git a/scraper/scrapers/ondebaixa/ondebaixa_api.js b/scraper/scrapers/ondebaixa/ondebaixa_api.js index a453095..2c4b4ef 100644 --- a/scraper/scrapers/ondebaixa/ondebaixa_api.js +++ b/scraper/scrapers/ondebaixa/ondebaixa_api.js @@ -9,9 +9,7 @@ const { isPtDubbed, sanitizePtName, sanitizePtOriginalName, sanitizePtLanguages const defaultTimeout = 10000; const maxSearchPage = 50 -const defaultProxies = [ - 'https://ondebaixa.com' -]; +const baseUrl = 'https://ondebaixa.com'; const Categories = { MOVIE: 'filmes', @@ -24,10 +22,8 @@ function torrent(torrentId, config = {}, retries = 2) { if (!torrentId || retries === 0) { return Promise.reject(new Error(`Failed ${torrentId} query`)); } - const proxyList = config.proxyList || defaultProxies; const slug = torrentId.split("/")[3]; - return Promises.first(proxyList - .map((proxyUrl) => singleRequest(`${proxyUrl}/${slug}/`, config))) + return singleRequest(`${baseUrl}/${slug}/`, config) .then((body) => parseTorrentPage(body)) .then((torrent) => torrent.map(el => ({ torrentId: slug, ...el }))) .catch((err) => torrent(slug, config, retries - 1)); @@ -37,13 +33,10 @@ function search(keyword, config = {}, retries = 2) { if (!keyword || retries === 0) { return Promise.reject(new Error(`Failed ${keyword} search`)); } - const proxyList = config.proxyList || defaultProxies; const page = config.page || 1; const extendToPage = Math.min(maxSearchPage, (config.extendToPage || 1)) - const requestUrl = proxyUrl => `${proxyUrl}/${keyword}/${page}/` - return Promises.first(proxyList - .map(proxyUrl => singleRequest(requestUrl(proxyUrl), config))) + return singleRequest(`${baseUrl}/${keyword}/${page}/`, config) .then(body => parseTableBody(body)) .then(torrents => torrents.length === 40 && page < extendToPage ? search(keyword, { ...config, page: page + 1 }).catch(() => []) @@ -56,13 +49,11 @@ function browse(config = {}, retries = 2) { if (retries === 0) { return Promise.reject(new Error(`Failed browse request`)); } - const proxyList = config.proxyList || defaultProxies; const page = config.page || 1; const category = config.category; - const requestUrl = proxyUrl => category ? `${proxyUrl}/${category}/${page}/` : `${proxyUrl}/${page}/`; + const requestUrl = category ? `${baseUrl}/${category}/${page}/` : `${baseUrl}/${page}/`; - return Promises.first(proxyList - .map((proxyUrl) => singleRequest(requestUrl(proxyUrl), config))) + return singleRequest(requestUrl, config) .then((body) => parseTableBody(body)) .catch((err) => browse(config, retries - 1)); }