[scraper] reduces number of calls to 1337x

scraper/lib/repository.js

@@ -163,13 +163,10 @@ function getProvider(provider) {
 }
 
 function getTorrent(torrent) {
-  return Torrent.findByPk(torrent.infoHash)
-      .then((result) => {
-        if (!result) {
-          throw new Error(`torrent not found: ${torrent.infoHash}`);
-        }
-        return result;
-      })
+  const where = torrent.infoHash
+      ? { infoHash: torrent.infoHash }
+      : { provider: torrent.provider, torrentId: torrent.torrentId }
+  return Torrent.findOne({ where: where });
 }
 
 function getTorrentsBasedOnTitle(titleQuery, type) {
@@ -212,10 +209,13 @@ function createTorrent(torrent) {
       .then(() => createSubtitles(torrent.infoHash, torrent.subtitles));
 }
 
-function setTorrentSeeders(infoHash, seeders) {
+function setTorrentSeeders(torrent, seeders) {
+  const where = torrent.infoHash
+      ? { infoHash: torrent.infoHash }
+      : { provider: torrent.provider, torrentId: torrent.torrentId }
   return Torrent.update(
       { seeders: seeders },
-      { where: { infoHash: infoHash } }
+      { where: where }
   );
 }
 

scraper/lib/torrentEntries.js

@@ -151,11 +151,11 @@ async function createTorrentContents(torrent) {
 }
 
 async function updateTorrentSeeders(torrent) {
-  if (!torrent.infoHash || !Number.isInteger(torrent.seeders)) {
+  if (!(torrent.infoHash || (torrent.provider && torrent.torrentId)) || !Number.isInteger(torrent.seeders)) {
     return;
   }
 
-  return repository.setTorrentSeeders(torrent.infoHash, torrent.seeders)
+  return repository.setTorrentSeeders(torrent, torrent.seeders)
       .catch(error => {
         console.warn('Failed updating seeders:', error);
         return undefined;

scraper/scrapers/1337x/1337x_api.js

@@ -4,7 +4,7 @@ const Sugar = require('sugar-date');
 const decode = require('magnet-uri');
 const Promises = require('../../lib/promises');
 const { escapeHTML } = require('../../lib/metadata');
-const { defaultOptionsWithProxy } = require('../../lib/request_helper');
+const { getRandomUserAgent, defaultOptionsWithProxy } = require('../../lib/request_helper');
 
 const defaultProxies = [
   'https://1337x.to'
@@ -80,7 +80,7 @@ function browse(config = {}, retries = 2) {
 
 function singleRequest(requestUrl, config = {}) {
   const timeout = config.timeout || defaultTimeout;
-  const options = { ...defaultOptionsWithProxy(), open_timeout: timeout, follow: 2 };
+  const options = { userAgent: getRandomUserAgent(), open_timeout: timeout, follow: 2 };
 
   return needle('get', requestUrl, options)
       .then((response) => {

scraper/scrapers/1337x/1337x_scraper.js

@@ -56,6 +56,10 @@ async function scrapeLatestTorrentsForCategory(category, page = 1) {
 }
 
 async function processTorrentRecord(record) {
+  if (await checkAndUpdateTorrent({ provider: NAME, ...record })) {
+    return record;
+  }
+
   const torrentFound = await leetx.torrent(record.torrentId).catch(() => undefined);
 
   if (!torrentFound || !TYPE_MAPPING[torrentFound.category]) {
@@ -65,9 +69,6 @@ async function processTorrentRecord(record) {
     console.warn(`Incorrect upload date for [${torrentFound.infoHash}] ${torrentFound.name}`);
     return;
   }
-  if (await checkAndUpdateTorrent(torrentFound)) {
-    return torrentFound;
-  }
 
   const torrent = {
     infoHash: torrentFound.infoHash,

scraper/scrapers/horriblesubs/horriblesubs_mapping.json

@@ -639,7 +639,7 @@
   },
   "Grisaia no Meikyuu": {
     "showId": "159",
-    "kitsu_id": "11236"
+    "kitsu_id": "10154"
   },
   "Grisaia no Rakuen": {
     "showId": "160",