[scraper] adds helper class for proxy and user agent

scraper/lib/metadata.js

@@ -5,6 +5,7 @@ const bing = require('nodejs-bing');
 const he = require('he');
 const { cacheWrapImdbId, cacheWrapKitsuId, cacheWrapMetadata } = require('./cache');
 const { Type } = require('./types');
+const { getRandomUserAgent } = require('./request_helper');
 
 const CINEMETA_URL = 'https://v3-cinemeta.strem.io';
 const KITSU_URL = 'https://anime-kitsu.strem.fun';
@@ -107,7 +108,7 @@ async function getImdbId(info, type) {
             reject(err || new Error('failed imdbId search'));
           }
         });
-      }).catch(() => googleIt({ query, disableConsole: true })
+      }).catch(() => googleIt({ query, userAgent: getRandomUserAgent(), disableConsole: true })
           .catch(() => bing.web(query))
           .then(results => results
               .map(result => result.link)

scraper/lib/request_helper.js

@@ -0,0 +1,26 @@
+const UserAgent = require('user-agents');
+
+const PROXY_HOSTS = process.env.PROXY_HOST && process.env.PROXY_HOST.split(',');
+const PROXY_USERNAME = process.env.PROXY_USERNAME;
+const PROXY_PASSWORD = process.env.PROXY_PASSWORD;
+const userAgent = new UserAgent();
+
+function getRandomUserAgent() {
+  return userAgent.random().toString();
+}
+
+function getRandomProxy() {
+  if (PROXY_HOSTS && PROXY_HOSTS.length && PROXY_USERNAME && PROXY_PASSWORD) {
+    return `http://${PROXY_USERNAME}:${PROXY_PASSWORD}@${PROXY_HOSTS[Math.floor(Math.random() * PROXY_HOSTS.length)]}`;
+  }
+  return undefined;
+}
+
+function getProxy() {
+  if (PROXY_HOSTS && PROXY_HOSTS.length && PROXY_USERNAME && PROXY_PASSWORD) {
+    return `http://${PROXY_USERNAME}:${PROXY_PASSWORD}@${PROXY_HOSTS[0]}`;
+  }
+  return undefined;
+}
+
+module.exports = { getRandomUserAgent, getRandomProxy, getProxy };