From 33d9921179458585b137d07d053968b587883b4c Mon Sep 17 00:00:00 2001
From: TheBeastLT <pauliox@beyond.lt>
Date: Fri, 3 Apr 2020 13:27:26 +0200
Subject: [PATCH] [scraper] cache rd options per ip

---
 scraper/index.js              |  2 +-
 scraper/lib/cache.js          | 15 ++++++++++++++-
 scraper/lib/request_helper.js |  2 +-
 scraper/moch/realdebrid.js    | 17 +++++++++--------
 4 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/scraper/index.js b/scraper/index.js
index bd3359d..94a3aa1 100644
--- a/scraper/index.js
+++ b/scraper/index.js
@@ -50,7 +50,7 @@ server.get('/', function (req, res) {
 
 server.get('/realdebrid/:apiKey/:infoHash/:cachedFileIds/:fileIndex?', (req, res) => {
   const { apiKey, infoHash, cachedFileIds, fileIndex } = req.params;
-  realDebrid.resolve(apiKey, infoHash, cachedFileIds, isNaN(fileIndex) ? undefined : parseInt(fileIndex))
+  realDebrid.resolve(req.ip, apiKey, infoHash, cachedFileIds, isNaN(fileIndex) ? undefined : parseInt(fileIndex))
       .then(url => {
         res.writeHead(301, { Location: url });
         res.end();
diff --git a/scraper/lib/cache.js b/scraper/lib/cache.js
index 399bd80..d2978e7 100644
--- a/scraper/lib/cache.js
+++ b/scraper/lib/cache.js
@@ -6,10 +6,12 @@ const IMDB_ID_PREFIX = `${GLOBAL_KEY_PREFIX}|imdb_id`;
 const KITSU_ID_PREFIX = `${GLOBAL_KEY_PREFIX}|kitsu_id`;
 const METADATA_PREFIX = `${GLOBAL_KEY_PREFIX}|metadata`;
 const RESOLVED_URL_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|moch`;
+const PROXY_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|proxy`;
 const TORRENT_FILES_KEY_PREFIX = `stremio-tpb|files`;
 
 const GLOBAL_TTL = process.env.METADATA_TTL || 7 * 24 * 60 * 60; // 7 days
 const MEMORY_TTL = process.env.METADATA_TTL || 2 * 60 * 60; // 2 hours
+const PROXY_TTL = 8 * 60 * 60; // 8 hours
 
 const MONGO_URI = process.env.MONGODB_URI;
 
@@ -89,5 +91,16 @@ function cacheWrapResolvedUrl(id, method) {
   return cacheWrap(memoryCache, `${RESOLVED_URL_KEY_PREFIX}:${id}`, method, { ttl: { MEMORY_TTL } });
 }
 
-module.exports = { cacheWrapImdbId, cacheWrapKitsuId, cacheWrapMetadata, retrieveTorrentFiles, cacheWrapResolvedUrl };
+function cacheWrapProxy(id, method) {
+  return cacheWrap(memoryCache, `${PROXY_KEY_PREFIX}:${id}`, method, { ttl: { PROXY_TTL } });
+}
+
+module.exports = {
+  cacheWrapImdbId,
+  cacheWrapKitsuId,
+  cacheWrapMetadata,
+  retrieveTorrentFiles,
+  cacheWrapResolvedUrl,
+  cacheWrapProxy
+};
 
diff --git a/scraper/lib/request_helper.js b/scraper/lib/request_helper.js
index c9860fa..61c4816 100644
--- a/scraper/lib/request_helper.js
+++ b/scraper/lib/request_helper.js
@@ -1,6 +1,6 @@
 const UserAgent = require('user-agents');
 
-const PROXY_HOSTS = process.env.PROXY_HOST && process.env.PROXY_HOST.split(',');
+const PROXY_HOSTS = process.env.PROXY_HOSTS && process.env.PROXY_HOSTS.split(',');
 const PROXY_USERNAME = process.env.PROXY_USERNAME;
 const PROXY_PASSWORD = process.env.PROXY_PASSWORD;
 const userAgent = new UserAgent();
diff --git a/scraper/moch/realdebrid.js b/scraper/moch/realdebrid.js
index c7e32ed..c09a426 100644
--- a/scraper/moch/realdebrid.js
+++ b/scraper/moch/realdebrid.js
@@ -1,28 +1,28 @@
 const { encode } = require('magnet-uri');
 const RealDebridClient = require('real-debrid-api');
 const namedQueue = require('named-queue');
-const { cacheWrapResolvedUrl } = require('../lib/cache');
+const { cacheWrapResolvedUrl, cacheWrapProxy } = require('../lib/cache');
 const { getRandomProxy, getRandomUserAgent } = require('../lib/request_helper');
 
 const unrestrictQueue = new namedQueue((task, callback) => task.method()
     .then(result => callback(false, result))
     .catch((error => callback(error))));
 
-async function resolve(apiKey, infoHash, cachedFileIds, fileIndex) {
+async function resolve(ip, apiKey, infoHash, cachedFileIds, fileIndex) {
   if (!apiKey || !infoHash || !cachedFileIds || !cachedFileIds.length) {
     return Promise.reject("No valid parameters passed");
   }
   const id = `${apiKey}_${infoHash}_${fileIndex}`;
-  const method = () => cacheWrapResolvedUrl(id, () => _unrestrict(apiKey, infoHash, cachedFileIds, fileIndex));
+  const method = () => cacheWrapResolvedUrl(id, () => _unrestrict(ip, apiKey, infoHash, cachedFileIds, fileIndex));
 
   return new Promise(((resolve, reject) => {
     unrestrictQueue.push({ id, method }, (error, result) => result ? resolve(result) : reject(error));
   }));
 }
 
-async function _unrestrict(apiKey, infoHash, cachedFileIds, fileIndex) {
+async function _unrestrict(ip, apiKey, infoHash, cachedFileIds, fileIndex) {
   console.log(`Unrestricting ${infoHash} [${fileIndex}]`);
-  const RD = new RealDebridClient(apiKey, getDefaultOptions());
+  const RD = await getDefaultOptions(ip).then(options => new RealDebridClient(apiKey, options));
   const torrentId = await _createOrFindTorrentId(RD, infoHash, cachedFileIds);
   if (torrentId) {
     const info = await RD.torrents.info(torrentId);
@@ -67,13 +67,14 @@ async function _unrestrictLink(RD, link) {
   // });
 }
 
-function getDefaultOptions() {
-  return {
+function getDefaultOptions(ip) {
+  const generateOptions = () => ({
     proxy: getRandomProxy(),
     headers: {
       'User-Agent': getRandomUserAgent()
     }
-  };
+  });
+  return cacheWrapProxy(ip, generateOptions).catch(() => generateOptions());
 }
 
 module.exports = { resolve };
\ No newline at end of file