[scraper] use same cached proxy for rd requests

scraper/lib/cache.js

@@ -7,33 +7,18 @@ const KITSU_ID_PREFIX = `${GLOBAL_KEY_PREFIX}|kitsu_id`;
 const METADATA_PREFIX = `${GLOBAL_KEY_PREFIX}|metadata`;
 const RESOLVED_URL_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|moch`;
 const PROXY_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|proxy`;
-const TORRENT_FILES_KEY_PREFIX = `stremio-tpb|files`;
+const USER_AGENT_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|agent`;
 
 const GLOBAL_TTL = process.env.METADATA_TTL || 7 * 24 * 60 * 60; // 7 days
 const MEMORY_TTL = process.env.METADATA_TTL || 2 * 60 * 60; // 2 hours
 const RESOLVED_URL_TTL = 2 * 60; // 2 minutes
-const PROXY_TTL = 8 * 60 * 60; // 8 hours
+const PROXY_TTL = 60 * 60; // 60 minutes
+const USER_AGENT_TTL = 2 * 24 * 60 * 60; // 2 days
 
 const MONGO_URI = process.env.MONGODB_URI;
 
 const memoryCache = initiateMemoryCache();
 const remoteCache = initiateRemoteCache();
-const torrentFilesCache = initiateTorrentFilesCache();
-
-function initiateTorrentFilesCache() {
-  if (MONGO_URI) {
-    return cacheManager.caching({
-      store: mangodbStore,
-      uri: MONGO_URI,
-      options: {
-        collection: 'cacheManager',
-        useUnifiedTopology: true,
-      },
-      ttl: GLOBAL_TTL,
-      ignoreCacheErrors: true
-    });
-  }
-}
 
 function initiateRemoteCache() {
   if (MONGO_URI) {
@@ -62,16 +47,6 @@ function initiateMemoryCache() {
   });
 }
 
-function retrieveTorrentFiles(infoHash) {
-  return torrentFilesCache.get(`${TORRENT_FILES_KEY_PREFIX}:${infoHash}`)
-      .then((results) => {
-        if (!results) {
-          throw new Error('No cached files found');
-        }
-        return results;
-      });
-}
-
 function cacheWrap(cache, key, method, options) {
   return cache.wrap(key, method, options);
 }
@@ -92,16 +67,20 @@ function cacheWrapResolvedUrl(id, method) {
   return cacheWrap(memoryCache, `${RESOLVED_URL_KEY_PREFIX}:${id}`, method, { ttl: { RESOLVED_URL_TTL } });
 }
 
-function cacheWrapOptions(id, method) {
+function cacheWrapProxy(id, method) {
   return cacheWrap(memoryCache, `${PROXY_KEY_PREFIX}:${id}`, method, { ttl: { PROXY_TTL } });
 }
 
+function cacheUserAgent(id, method) {
+  return cacheWrap(memoryCache, `${USER_AGENT_KEY_PREFIX}:${id}`, method, { ttl: { USER_AGENT_TTL } });
+}
+
 module.exports = {
   cacheWrapImdbId,
   cacheWrapKitsuId,
   cacheWrapMetadata,
-  retrieveTorrentFiles,
   cacheWrapResolvedUrl,
-  cacheWrapOptions
+  cacheWrapProxy,
+  cacheUserAgent
 };
 

scraper/lib/repository.js

@@ -28,7 +28,8 @@ const Torrent = database.define('torrent',
       seeders: { type: Sequelize.SMALLINT },
       trackers: { type: Sequelize.STRING(4096) },
       languages: { type: Sequelize.STRING(256) },
-      resolution: { type: Sequelize.STRING(16) }
+      resolution: { type: Sequelize.STRING(16) },
+      reviewed: { type: Sequelize.BOOLEAN, allowNull: false, defaultValue: false }
     }
 );
 
@@ -118,7 +119,7 @@ function getTorrentsBasedOnTitle(titleQuery, type) {
 function getTorrentsWithoutSize() {
   return Torrent.findAll({
     where: literal(
-        'exists (select 1 from files where files."infoHash" = torrent."infoHash" and files.size = 300000000)'),
+        'exists (select 1 from files where files."infoHash" = torrent."infoHash" and files.size = 300000000) and random() < 0.01'),
     order: [
       ['seeders', 'DESC']
     ],

scraper/lib/torrent.js

@@ -38,7 +38,7 @@ module.exports.updateTorrentSize = function (torrent) {
       .then(result => ({ ...torrent, size: result.size, files: result.files }));
 };
 
-module.exports.sizeAndFiles = torrent => filesAndSizeFromTorrentStream(torrent, 20000);
+module.exports.sizeAndFiles = torrent => filesAndSizeFromTorrentStream(torrent, 30000);
 
 module.exports.torrentFiles = function (torrent) {
   return getFilesFromObject(torrent)

scraper/moch/realdebrid.js

@@ -1,7 +1,7 @@
 const { encode } = require('magnet-uri');
 const RealDebridClient = require('real-debrid-api');
 const namedQueue = require('named-queue');
-const { cacheWrapResolvedUrl, cacheWrapOptions } = require('../lib/cache');
+const { cacheWrapResolvedUrl, cacheWrapProxy, cacheUserAgent } = require('../lib/cache');
 const { getRandomProxy, getRandomUserAgent } = require('../lib/request_helper');
 
 const unrestrictQueue = new namedQueue((task, callback) => task.method()
@@ -68,14 +68,16 @@ async function _unrestrictLink(RD, link) {
   // });
 }
 
-function getDefaultOptions(ip) {
-  const generateOptions = () => ({
-    proxy: getRandomProxy(),
+async function getDefaultOptions(ip) {
+  const userAgent = await cacheUserAgent(ip, () => getRandomUserAgent()).catch(() => getRandomUserAgent());
+  const proxy = await cacheWrapProxy('realdebrid', () => getRandomProxy()).catch(() => getRandomProxy());
+
+  return {
+    proxy: proxy,
     headers: {
-      'User-Agent': getRandomUserAgent()
+      'User-Agent': userAgent
     }
-  });
-  return cacheWrapOptions(ip, generateOptions).catch(() => generateOptions());
+  };
 }
 
 module.exports = { resolve };