diff --git a/scraper/lib/request_helper.js b/scraper/lib/request_helper.js index 9312103..2044dab 100644 --- a/scraper/lib/request_helper.js +++ b/scraper/lib/request_helper.js @@ -1,21 +1,10 @@ const UserAgent = require('user-agents'); - -const PROXY_HOSTS = process.env.PROXY_HOSTS && process.env.PROXY_HOSTS.split(','); -const PROXY_USERNAME = process.env.PROXY_USERNAME; -const PROXY_PASSWORD = process.env.PROXY_PASSWORD; const userAgent = new UserAgent(); function getRandomUserAgent() { return userAgent.random().toString(); } -function getRandomProxy() { - if (PROXY_HOSTS && PROXY_HOSTS.length && PROXY_USERNAME && PROXY_PASSWORD) { - return `http://${PROXY_USERNAME}:${PROXY_PASSWORD}@${PROXY_HOSTS[Math.floor(Math.random() * PROXY_HOSTS.length)]}`; - } - return undefined; -} - function defaultOptionsWithProxy() { if (process.env.PROXY_HOST && process.env.PROXY_TYPE) { return { @@ -29,4 +18,4 @@ function defaultOptionsWithProxy() { return { userAgent: getRandomUserAgent() }; } -module.exports = { getRandomUserAgent, getRandomProxy, defaultOptionsWithProxy }; \ No newline at end of file +module.exports = { getRandomUserAgent, defaultOptionsWithProxy }; \ No newline at end of file diff --git a/scraper/scrapers/kickass/kickass_api.js b/scraper/scrapers/kickass/kickass_api.js index 15d4ceb..d15d3e9 100644 --- a/scraper/scrapers/kickass/kickass_api.js +++ b/scraper/scrapers/kickass/kickass_api.js @@ -3,7 +3,6 @@ const needle = require('needle'); const moment = require('moment'); const decode = require('magnet-uri'); const Promises = require('../../lib/promises'); -const { defaultOptionsWithProxy } = require('./../../lib/request_helper'); const defaultProxies = [ 'https://katcr.co' @@ -65,7 +64,7 @@ function browse(config = {}, retries = 2) { function singleRequest(requestUrl, config = {}) { const timeout = config.timeout || defaultTimeout; - const options = { ...defaultOptionsWithProxy(), open_timeout: timeout, follow: 2 }; + const options = { open_timeout: timeout, follow: 2 }; return needle('get', requestUrl, options) .then((response) => {