[scraper] updates hs scraper to update seeders for existing latest entries

scraper/lib/torrentEntries.js

@@ -4,7 +4,7 @@ const repository = require('./repository');
 const { getImdbId, getKitsuId } = require('./metadata');
 const { parseTorrentFiles } = require('./torrentFiles');
 
-async function createTorrentEntry(torrent) {
+async function createTorrentEntry(torrent, overwrite = false) {
   const titleInfo = parse(torrent.title);
 
   if (titleInfo.seasons && torrent.type === Type.MOVIE) {
@@ -33,7 +33,8 @@ async function createTorrentEntry(torrent) {
     return;
   }
 
-  const files = await parseTorrentFiles(torrent);
+  const files = await parseTorrentFiles(torrent)
+      .then(files => overwrite ? overwriteExistingFiles(torrent, files) : files);
   if (!files || !files.length) {
     console.log(`no video files found for [${torrent.infoHash}] ${torrent.title}`);
     return;
@@ -44,6 +45,34 @@ async function createTorrentEntry(torrent) {
       .then(() => console.log(`Created entry for [${torrent.infoHash}] ${torrent.title}`));
 }
 
+async function overwriteExistingFiles(torrent, files) {
+  if (files && files.length) {
+    const existingFiles = await repository.getFiles({ infoHash: files[0].infoHash })
+        .then((existing) => existing
+            .reduce((map, next) => {
+              const fileIndex = next.fileIndex !== undefined ? next.fileIndex : null;
+              map[fileIndex] = (map[fileIndex] || []).concat(next);
+              return map;
+            }, {}))
+        .catch(() => undefined);
+    if (existingFiles && Object.keys(existingFiles).length) {
+      return files
+          .map(file => {
+            const mapping = files.length === 1 && Object.keys(existingFiles).length === 1
+                ? Object.values(existingFiles)[0]
+                : existingFiles[file.fileIndex !== undefined ? file.fileIndex : null];
+            if (mapping) {
+              const originalFile = mapping.shift();
+              return { ...file, id: originalFile.id, size: originalFile.size || file.size };
+            }
+            return file;
+          })
+    }
+    return files;
+  }
+  return Promise.reject(`No video files found for: ${torrent.title}`);
+}
+
 async function createSkipTorrentEntry(torrent) {
   return repository.createSkipTorrent(torrent);
 }

scraper/scrapers/horriblesubs/horriblesubs_scraper.js

@@ -6,8 +6,7 @@ const horriblesubs = require('./horriblesubs_api.js');
 const repository = require('../../lib/repository');
 const { Type } = require('../../lib/types');
 const { updateCurrentSeeders, updateTorrentSize } = require('../../lib/torrent');
-const { parseTorrentFiles } = require('../../lib/torrentFiles');
+const { createTorrentEntry, updateTorrentSeeders } = require('../../lib/torrentEntries');
-const { updateTorrentSeeders } = require('../../lib/torrentEntries');
 const { getMetadata, getKitsuId } = require('../../lib/metadata');
 const showMappings = require('./horriblesubs_mapping.json');
 
@@ -151,9 +150,9 @@ async function _parseShowData(showData) {
   };
 
   return Promise.all([].concat(showData.singleEpisodes || []).concat(showData.packEpisodes || [])
-      .map((episodeInfo) => episodeInfo.mirrors
+      .map(episodeInfo => episodeInfo.mirrors
-          .filter((mirror) => mirror.magnetLink && mirror.magnetLink.length)
+          .filter(mirror => mirror.magnetLink && mirror.magnetLink.length)
-          .map((mirror) => ({
+          .map(mirror => ({
             provider: NAME,
             ...mirror,
             infoHash: decode(mirror.magnetLink).infoHash,
@@ -164,55 +163,22 @@ async function _parseShowData(showData) {
             uploadDate: episodeInfo.uploadDate,
           })))
       .reduce((a, b) => a.concat(b), [])
-      .filter((incompleteTorrent) => incompleteTorrent.kitsuId)
+      .filter(torrent => torrent.kitsuId)
-      .map((incompleteTorrent) => entryLimiter.schedule(() => checkIfExists(incompleteTorrent)
+      .map(torrent => entryLimiter.schedule(() => processTorrentRecord(torrent))))
-          .then((torrent) => torrent && updateTorrentSize(torrent))
-          .then((torrent) => torrent && updateCurrentSeeders(torrent))
-          .then((torrent) => torrent && parseTorrentFiles(torrent)
-              .then((files) => verifyFiles(torrent, files))
-              .then((files) => repository.createTorrent(torrent)
-                  .then(() => files.forEach(file => repository.createFile(file)))
-                  .then(() => console.log(`Created entry for [${torrent.infoHash}] ${torrent.title}`))))
-          .catch(error => console.warn(`Failed creating entry for ${incompleteTorrent.title}:`, error)))))
       .then(() => console.log(`${NAME}: finished scrapping ${showData.title} data`));
 }
 
-async function verifyFiles(torrent, files) {
+async function processTorrentRecord(torrent) {
-  if (files && files.length) {
-    const existingFiles = await repository.getFiles({ infoHash: files[0].infoHash })
-        .then((existing) => existing
-            .reduce((map, next) => {
-              const fileIndex = next.fileIndex !== undefined ? next.fileIndex : null;
-              map[fileIndex] = (map[fileIndex] || []).concat(next);
-              return map;
-            }, {}))
-        .catch(() => undefined);
-    if (existingFiles && Object.keys(existingFiles).length) {
-      return files
-          .map(file => {
-            const mapping = files.length === 1 && Object.keys(existingFiles).length === 1
-                ? Object.values(existingFiles)[0]
-                : existingFiles[file.fileIndex !== undefined ? file.fileIndex : null];
-            if (mapping) {
-              const originalFile = mapping.shift();
-              return { ...file, id: originalFile.id, size: originalFile.size || file.size };
-            }
-            return file;
-          })
-    }
-    return files;
-  }
-  return Promise.reject(`No video files found for: ${torrent.title}`);
-}
-
-async function checkIfExists(torrent) {
   const existingTorrent = await repository.getTorrent(torrent).catch(() => undefined);
-  if (!existingTorrent) {
+
-    return torrent; // no torrent exists yet
+  if (existingTorrent && existingTorrent.provider === NAME) {
-  } else if (existingTorrent.provider === NAME) {
+    return updateCurrentSeeders(torrent).then(updatedSeeders => updateTorrentSeeders(updatedSeeders))
-    return undefined; // torrent by this provider already exists
   }
-  return { ...torrent, size: existingTorrent.size, seeders: existingTorrent.seeders };
+
+  return updateTorrentSize(torrent)
+      .then(updated => updateCurrentSeeders(updated))
+      .then(updated => createTorrentEntry(updated, true))
+      .catch(error => console.warn(`Failed creating entry for ${torrent.title}:`, error));
 }
 
 module.exports = { scrape, updateSeeders, NAME };