[scraper] adds update seeders cron

scraper/lib/cache.js

@@ -8,12 +8,14 @@ const METADATA_PREFIX = `${GLOBAL_KEY_PREFIX}|metadata`;
 const RESOLVED_URL_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|moch`;
 const PROXY_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|proxy`;
 const USER_AGENT_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|agent`;
+const TRACKERS_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|trackers`;
 
 const GLOBAL_TTL = process.env.METADATA_TTL || 7 * 24 * 60 * 60; // 7 days
 const MEMORY_TTL = process.env.METADATA_TTL || 2 * 60 * 60; // 2 hours
 const RESOLVED_URL_TTL = 2 * 60; // 2 minutes
 const PROXY_TTL = 60 * 60; // 60 minutes
 const USER_AGENT_TTL = 2 * 24 * 60 * 60; // 2 days
+const TRACKERS_TTL = 2 * 24 * 60 * 60; // 2 days
 
 const MONGO_URI = process.env.MONGODB_URI;
 
@@ -75,12 +77,17 @@ function cacheUserAgent(id, method) {
   return cacheWrap(memoryCache, `${USER_AGENT_KEY_PREFIX}:${id}`, method, { ttl: { USER_AGENT_TTL } });
 }
 
+function cacheTrackers(method) {
+  return cacheWrap(memoryCache, `${TRACKERS_KEY_PREFIX}`, method, { ttl: { TRACKERS_TTL } });
+}
+
 module.exports = {
   cacheWrapImdbId,
   cacheWrapKitsuId,
   cacheWrapMetadata,
   cacheWrapResolvedUrl,
   cacheWrapProxy,
-  cacheUserAgent
+  cacheUserAgent,
+  cacheTrackers
 };
 

scraper/lib/repository.js

@@ -1,3 +1,4 @@
+const moment = require('moment');
 const { Sequelize, fn, col, literal } = require('sequelize');
 const Op = Sequelize.Op;
 
@@ -119,11 +120,18 @@ function getTorrentsBasedOnTitle(titleQuery, type) {
 function getTorrentsWithoutSize() {
   return Torrent.findAll({
     where: literal(
-        'exists (select 1 from files where files."infoHash" = torrent."infoHash" and files.size = 300000000) and random() < 0.01'),
+        'exists (select 1 from files where files."infoHash" = torrent."infoHash" and files.size = 300000000)'),
     order: [
       ['seeders', 'DESC']
-    ],
-    limit: 1000
+    ]
+  });
+}
+
+function getUpdateSeedersTorrents() {
+  const until = moment().subtract(7, 'days').format('YYYY-MM-DD');
+  return Torrent.findAll({
+    where: literal(`torrent."updatedAt" < \'${until}\' and random() < 0.001`),
+    limit: 100
   });
 }
 
@@ -186,6 +194,7 @@ module.exports = {
   createTorrent,
   getTorrent,
   getTorrentsBasedOnTitle,
+  getUpdateSeedersTorrents,
   createFile,
   getFiles,
   getFilesBasedOnTitle,

scraper/lib/torrent.js

@@ -4,20 +4,22 @@ const parseTorrent = require('parse-torrent');
 const async = require('async');
 const decode = require('magnet-uri');
 const isVideo = require('./video');
+const { cacheTrackers } = require('./cache');
 
+const TRACKERS_URL = 'https://ngosang.github.io/trackerslist/trackers_best.txt';
 const MAX_PEER_CONNECTIONS = process.env.MAX_PEER_CONNECTIONS || 20;
 const SEEDS_CHECK_TIMEOUT = process.env.SEEDS_CHECK_TIMEOUT || 10 * 1000; // 10 secs
 
 module.exports.updateCurrentSeeders = function (torrent) {
-  return new Promise((resolve) => {
-    if (!torrent.magnetLink) {
+  return new Promise(async (resolve) => {
+    if (!torrent.magnetLink && !torrent.infoHash) {
       return resolve(0);
     }
     const Tracker = require("peer-search/tracker");
 
     const seeders = {};
-    const decodedMagnetLink = decode(torrent.magnetLink);
-    const trackers = decodedMagnetLink && decodedMagnetLink.tr;
+    const decodedMagnetLink = torrent.magnetLink && decode(torrent.magnetLink);
+    const trackers = decodedMagnetLink && decodedMagnetLink.tr || torrent.trackers || await getDefaultTrackers();
     const callback = () => resolve(Math.max(...Object.values(seeders).map(values => values[0]).concat(0)));
     setTimeout(callback, SEEDS_CHECK_TIMEOUT);
 
@@ -30,12 +32,19 @@ module.exports.updateCurrentSeeders = function (torrent) {
         ready();
       });
     }, callback);
-  }).then(seeders => ({ ...torrent, seeders: torrent.seeders || seeders }));
+  }).then(seeders => {
+    torrent.seeders = seeders;
+    return torrent;
+  });
 };
 
 module.exports.updateTorrentSize = function (torrent) {
   return filesAndSizeFromTorrentStream(torrent, SEEDS_CHECK_TIMEOUT)
-      .then(result => ({ ...torrent, size: result.size, files: result.files }));
+      .then(result => {
+        torrent.size = result.size;
+        torrent.files = result.files;
+        return torrent;
+      });
 };
 
 module.exports.sizeAndFiles = torrent => filesAndSizeFromTorrentStream(torrent, 30000);
@@ -124,19 +133,9 @@ function filterExtras(files) {
   return files.filter(file => !isExtra(file));
 }
 
-function dynamicTimeout(torrent) {
-  if (torrent.seeders < 5) {
-    return 5000;
-  } else if (torrent.seeders < 10) {
-    return 7000;
-  } else if (torrent.seeders < 20) {
-    return 10000;
-  } else if (torrent.seeders < 30) {
-    return 15000;
-  } else if (torrent.seeders < 50) {
-    return 20000;
-  } else {
-    return 30000;
-  }
+async function getDefaultTrackers() {
+  return cacheTrackers(() => needle('get', TRACKERS_URL, { open_timeout: SEEDS_CHECK_TIMEOUT })
+      .then(response => response.body && response.body.trim())
+      .then(body => body && body.split('\n\n') || []));
 }
 

scraper/lib/torrentEntries.js

@@ -63,6 +63,8 @@ async function updateTorrentSeeders(torrent) {
       .catch(() => undefined)
       .then(stored => {
         if (stored) {
+          console.log(
+              `Updated seeder for ${torrent.provider} [${torrent.infoHash}] ${torrent.title} from ${stored.seeders} to ${torrent.seeders}`);
           stored.seeders = torrent.seeders;
           stored.changed('updatedAt', true);
           return stored.save()