diff --git a/scraper/lib/request_helper.js b/scraper/lib/request_helper.js index 61c4816..9312103 100644 --- a/scraper/lib/request_helper.js +++ b/scraper/lib/request_helper.js @@ -16,4 +16,17 @@ function getRandomProxy() { return undefined; } -module.exports = { getRandomUserAgent, getRandomProxy }; \ No newline at end of file +function defaultOptionsWithProxy() { + if (process.env.PROXY_HOST && process.env.PROXY_TYPE) { + return { + proxy: process.env.PROXY_HOST, + headers: { + 'user-agent': getRandomUserAgent(), + 'proxy-type': process.env.PROXY_TYPE + } + } + } + return { userAgent: getRandomUserAgent() }; +} + +module.exports = { getRandomUserAgent, getRandomProxy, defaultOptionsWithProxy }; \ No newline at end of file diff --git a/scraper/lib/video.js b/scraper/lib/video.js deleted file mode 100644 index 265c977..0000000 --- a/scraper/lib/video.js +++ /dev/null @@ -1,23 +0,0 @@ -const EXTENSIONS = [ - "3g2", - "3gp", - "avi", - "flv", - "mkv", - "mov", - "mp2", - "mp4", - "m4v", - "mpe", - "mpeg", - "mpg", - "mpv", - "webm", - "wmv", - "ogm" -]; - -module.exports = (filename) => { - const extensionMatch = filename.match(/\.(\w{2,4})$/); - return extensionMatch && EXTENSIONS.includes(extensionMatch[1].toLowerCase()); -}; \ No newline at end of file diff --git a/scraper/scrapers/1337x/1337x_api.js b/scraper/scrapers/1337x/1337x_api.js index 7e5fa45..f227ed9 100644 --- a/scraper/scrapers/1337x/1337x_api.js +++ b/scraper/scrapers/1337x/1337x_api.js @@ -4,6 +4,7 @@ const Sugar = require('sugar-date'); const decode = require('magnet-uri'); const Promises = require('../../lib/promises'); const { escapeHTML } = require('../../lib/metadata'); +const { defaultOptionsWithProxy } = require('../../lib/request_helper'); const defaultProxies = [ 'https://1337x.to' @@ -79,8 +80,9 @@ function browse(config = {}, retries = 2) { function singleRequest(requestUrl, config = {}) { const timeout = config.timeout || defaultTimeout; + const options = { ...defaultOptionsWithProxy(), open_timeout: timeout, follow: 2 }; - return needle('get', requestUrl, { open_timeout: timeout, follow: 2 }) + return needle('get', requestUrl, options) .then((response) => { const body = response.body; if (!body) { diff --git a/scraper/scrapers/kickass/kickass_api.js b/scraper/scrapers/kickass/kickass_api.js index e407cc2..15d4ceb 100644 --- a/scraper/scrapers/kickass/kickass_api.js +++ b/scraper/scrapers/kickass/kickass_api.js @@ -3,12 +3,12 @@ const needle = require('needle'); const moment = require('moment'); const decode = require('magnet-uri'); const Promises = require('../../lib/promises'); -const { getRandomUserAgent } = require('./../../lib/request_helper'); +const { defaultOptionsWithProxy } = require('./../../lib/request_helper'); const defaultProxies = [ 'https://katcr.co' ]; -const defaultTimeout = 30000; +const defaultTimeout = 10000; const Categories = { MOVIE: 'movies', @@ -65,7 +65,7 @@ function browse(config = {}, retries = 2) { function singleRequest(requestUrl, config = {}) { const timeout = config.timeout || defaultTimeout; - const options = { userAgent: getRandomUserAgent(), open_timeout: timeout, follow: 2 }; + const options = { ...defaultOptionsWithProxy(), open_timeout: timeout, follow: 2 }; return needle('get', requestUrl, options) .then((response) => { diff --git a/scraper/scrapers/rarbg/rargb_series_imdb_ids_2020-04-22.json b/scraper/scrapers/rarbg/rargb_series_imdb_ids_2020-04-22.json index 6d14c4a..a0cdb45 100644 --- a/scraper/scrapers/rarbg/rargb_series_imdb_ids_2020-04-22.json +++ b/scraper/scrapers/rarbg/rargb_series_imdb_ids_2020-04-22.json @@ -6300,5 +6300,6 @@ "tt9815454", "tt0879688", "tt5252734", - "tt9059820" + "tt9059820", + "tt5766086" ] \ No newline at end of file