[scraper] adds proxy for kat and 1337x

scraper/lib/request_helper.js

@@ -16,4 +16,17 @@ function getRandomProxy() {
   return undefined;
 }
 
-module.exports = { getRandomUserAgent, getRandomProxy };
+function defaultOptionsWithProxy() {
+  if (process.env.PROXY_HOST && process.env.PROXY_TYPE) {
+    return {
+      proxy: process.env.PROXY_HOST,
+      headers: {
+        'user-agent': getRandomUserAgent(),
+        'proxy-type': process.env.PROXY_TYPE
+      }
+    }
+  }
+  return { userAgent: getRandomUserAgent() };
+}
+
+module.exports = { getRandomUserAgent, getRandomProxy, defaultOptionsWithProxy };

scraper/lib/video.js

@@ -1,23 +0,0 @@
-const EXTENSIONS = [
-  "3g2",
-  "3gp",
-  "avi",
-  "flv",
-  "mkv",
-  "mov",
-  "mp2",
-  "mp4",
-  "m4v",
-  "mpe",
-  "mpeg",
-  "mpg",
-  "mpv",
-  "webm",
-  "wmv",
-  "ogm"
-];
-
-module.exports = (filename) => {
-  const extensionMatch = filename.match(/\.(\w{2,4})$/);
-  return extensionMatch && EXTENSIONS.includes(extensionMatch[1].toLowerCase());
-};

scraper/scrapers/1337x/1337x_api.js

@@ -4,6 +4,7 @@ const Sugar = require('sugar-date');
 const decode = require('magnet-uri');
 const Promises = require('../../lib/promises');
 const { escapeHTML } = require('../../lib/metadata');
+const { defaultOptionsWithProxy } = require('../../lib/request_helper');
 
 const defaultProxies = [
   'https://1337x.to'
@@ -79,8 +80,9 @@ function browse(config = {}, retries = 2) {
 
 function singleRequest(requestUrl, config = {}) {
   const timeout = config.timeout || defaultTimeout;
+  const options = { ...defaultOptionsWithProxy(), open_timeout: timeout, follow: 2 };
 
-  return needle('get', requestUrl, { open_timeout: timeout, follow: 2 })
+  return needle('get', requestUrl, options)
       .then((response) => {
         const body = response.body;
         if (!body) {

scraper/scrapers/kickass/kickass_api.js

@@ -3,12 +3,12 @@ const needle = require('needle');
 const moment = require('moment');
 const decode = require('magnet-uri');
 const Promises = require('../../lib/promises');
-const { getRandomUserAgent } = require('./../../lib/request_helper');
+const { defaultOptionsWithProxy } = require('./../../lib/request_helper');
 
 const defaultProxies = [
   'https://katcr.co'
 ];
-const defaultTimeout = 30000;
+const defaultTimeout = 10000;
 
 const Categories = {
   MOVIE: 'movies',
@@ -65,7 +65,7 @@ function browse(config = {}, retries = 2) {
 
 function singleRequest(requestUrl, config = {}) {
   const timeout = config.timeout || defaultTimeout;
-  const options = { userAgent: getRandomUserAgent(), open_timeout: timeout, follow: 2 };
+  const options = { ...defaultOptionsWithProxy(), open_timeout: timeout, follow: 2 };
 
   return needle('get', requestUrl, options)
       .then((response) => {

scraper/scrapers/rarbg/rargb_series_imdb_ids_2020-04-22.json

@@ -6300,5 +6300,6 @@
   "tt9815454",
   "tt0879688",
   "tt5252734",
-  "tt9059820"
+  "tt9059820",
+  "tt5766086"
 ]