From 5426707dd90e2d335c62311c8dca2566aedaae02 Mon Sep 17 00:00:00 2001
From: TheBeastLT <pauliox@beyond.lt>
Date: Fri, 2 Oct 2020 12:49:45 +0200
Subject: [PATCH] [scraper] reduces number of calls to 1337x

---
 scraper/lib/repository.js                      | 18 +++++++++---------
 scraper/lib/torrentEntries.js                  |  4 ++--
 scraper/scrapers/1337x/1337x_api.js            |  4 ++--
 scraper/scrapers/1337x/1337x_scraper.js        |  7 ++++---
 .../horriblesubs/horriblesubs_mapping.json     |  2 +-
 5 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/scraper/lib/repository.js b/scraper/lib/repository.js
index 0c44a48..44159e9 100644
--- a/scraper/lib/repository.js
+++ b/scraper/lib/repository.js
@@ -163,13 +163,10 @@ function getProvider(provider) {
 }
 
 function getTorrent(torrent) {
-  return Torrent.findByPk(torrent.infoHash)
-      .then((result) => {
-        if (!result) {
-          throw new Error(`torrent not found: ${torrent.infoHash}`);
-        }
-        return result;
-      })
+  const where = torrent.infoHash
+      ? { infoHash: torrent.infoHash }
+      : { provider: torrent.provider, torrentId: torrent.torrentId }
+  return Torrent.findOne({ where: where });
 }
 
 function getTorrentsBasedOnTitle(titleQuery, type) {
@@ -212,10 +209,13 @@ function createTorrent(torrent) {
       .then(() => createSubtitles(torrent.infoHash, torrent.subtitles));
 }
 
-function setTorrentSeeders(infoHash, seeders) {
+function setTorrentSeeders(torrent, seeders) {
+  const where = torrent.infoHash
+      ? { infoHash: torrent.infoHash }
+      : { provider: torrent.provider, torrentId: torrent.torrentId }
   return Torrent.update(
       { seeders: seeders },
-      { where: { infoHash: infoHash } }
+      { where: where }
   );
 }
 
diff --git a/scraper/lib/torrentEntries.js b/scraper/lib/torrentEntries.js
index d2b47f8..925c8e7 100644
--- a/scraper/lib/torrentEntries.js
+++ b/scraper/lib/torrentEntries.js
@@ -151,11 +151,11 @@ async function createTorrentContents(torrent) {
 }
 
 async function updateTorrentSeeders(torrent) {
-  if (!torrent.infoHash || !Number.isInteger(torrent.seeders)) {
+  if (!(torrent.infoHash || (torrent.provider && torrent.torrentId)) || !Number.isInteger(torrent.seeders)) {
     return;
   }
 
-  return repository.setTorrentSeeders(torrent.infoHash, torrent.seeders)
+  return repository.setTorrentSeeders(torrent, torrent.seeders)
       .catch(error => {
         console.warn('Failed updating seeders:', error);
         return undefined;
diff --git a/scraper/scrapers/1337x/1337x_api.js b/scraper/scrapers/1337x/1337x_api.js
index f227ed9..f6d0626 100644
--- a/scraper/scrapers/1337x/1337x_api.js
+++ b/scraper/scrapers/1337x/1337x_api.js
@@ -4,7 +4,7 @@ const Sugar = require('sugar-date');
 const decode = require('magnet-uri');
 const Promises = require('../../lib/promises');
 const { escapeHTML } = require('../../lib/metadata');
-const { defaultOptionsWithProxy } = require('../../lib/request_helper');
+const { getRandomUserAgent, defaultOptionsWithProxy } = require('../../lib/request_helper');
 
 const defaultProxies = [
   'https://1337x.to'
@@ -80,7 +80,7 @@ function browse(config = {}, retries = 2) {
 
 function singleRequest(requestUrl, config = {}) {
   const timeout = config.timeout || defaultTimeout;
-  const options = { ...defaultOptionsWithProxy(), open_timeout: timeout, follow: 2 };
+  const options = { userAgent: getRandomUserAgent(), open_timeout: timeout, follow: 2 };
 
   return needle('get', requestUrl, options)
       .then((response) => {
diff --git a/scraper/scrapers/1337x/1337x_scraper.js b/scraper/scrapers/1337x/1337x_scraper.js
index e84638d..2c9d8b9 100644
--- a/scraper/scrapers/1337x/1337x_scraper.js
+++ b/scraper/scrapers/1337x/1337x_scraper.js
@@ -56,6 +56,10 @@ async function scrapeLatestTorrentsForCategory(category, page = 1) {
 }
 
 async function processTorrentRecord(record) {
+  if (await checkAndUpdateTorrent({ provider: NAME, ...record })) {
+    return record;
+  }
+
   const torrentFound = await leetx.torrent(record.torrentId).catch(() => undefined);
 
   if (!torrentFound || !TYPE_MAPPING[torrentFound.category]) {
@@ -65,9 +69,6 @@ async function processTorrentRecord(record) {
     console.warn(`Incorrect upload date for [${torrentFound.infoHash}] ${torrentFound.name}`);
     return;
   }
-  if (await checkAndUpdateTorrent(torrentFound)) {
-    return torrentFound;
-  }
 
   const torrent = {
     infoHash: torrentFound.infoHash,
diff --git a/scraper/scrapers/horriblesubs/horriblesubs_mapping.json b/scraper/scrapers/horriblesubs/horriblesubs_mapping.json
index 9898011..1b94e30 100644
--- a/scraper/scrapers/horriblesubs/horriblesubs_mapping.json
+++ b/scraper/scrapers/horriblesubs/horriblesubs_mapping.json
@@ -639,7 +639,7 @@
   },
   "Grisaia no Meikyuu": {
     "showId": "159",
-    "kitsu_id": "11236"
+    "kitsu_id": "10154"
   },
   "Grisaia no Rakuen": {
     "showId": "160",