adds movie packs handling and movies in series packs

lib/cache.js

@@ -1,4 +1,3 @@
-
 const cacheManager = require('cache-manager');
 const mangodbStore = require('cache-manager-mongodb');
 

lib/repository.js

@@ -82,6 +82,10 @@ function createFile(file) {
   return File.upsert(file);
 }
 
+function getFiles(torrent) {
+  return File.findAll({ where: { infoHash: torrent.infoHash } });
+}
+
 function getSkipTorrent(torrent) {
   return SkipTorrent.findByPk(torrent.infoHash)
       .then((result) =>{
@@ -110,4 +114,4 @@ function createFailedImdbTorrent(torrent) {
   return FailedImdbTorrent.upsert(torrent);
 }
 
-module.exports = { connect, getProvider, updateProvider, getTorrent, createTorrent, createFile, getSkipTorrent, createSkipTorrent, createFailedImdbTorrent };
+module.exports = { connect, getProvider, updateProvider, getTorrent, createTorrent, createFile, getFiles, getSkipTorrent, createSkipTorrent, createFailedImdbTorrent };

lib/torrentFiles.js

@@ -1,12 +1,32 @@
 const { torrentFiles } = require('../lib/torrent');
-const { getMetadata } = require('../lib/metadata');
+const { escapeTitle, getMetadata, getImdbId } = require('../lib/metadata');
 const { parse } = require('parse-torrent-title');
 const { Type } = require('./types');
 
 const MIN_SIZE = 20 * 1024 * 1024; // 20 MB
 
 async function parseTorrentFiles(torrent, imdbId) {
+  const parsedTorrentName = parse(torrent.title);
+
   if (torrent.type === Type.MOVIE) {
+    if (parsedTorrentName.complete) {
+      return torrentFiles(torrent)
+          .then(files => files.filter(file => file.size > MIN_SIZE))
+          .then(files => Promise.all(files
+              .map((file) => findMovieImdbId(file.name)
+                  .then((newImdbId) => ({
+                    infoHash: torrent.infoHash,
+                    fileIndex: file.fileIndex,
+                    title: file.name,
+                    size: file.size,
+                    imdbId: newImdbId,
+                  })))))
+          .catch(error => {
+            console.log(`Failed getting files for ${torrent.title}`, error.message);
+            return [];
+          });
+    }
+
     return [{
       infoHash: torrent.infoHash,
       title: torrent.title,
@@ -14,7 +34,7 @@ async function parseTorrentFiles(torrent, imdbId) {
       imdbId: imdbId,
     }];
   }
-  const parsedTorrentName = parse(torrent.title);
+
   if (parsedTorrentName.season && parsedTorrentName.episode) {
     return [{
       infoHash: torrent.infoHash,
@@ -26,29 +46,47 @@ async function parseTorrentFiles(torrent, imdbId) {
     }];
   }
 
+  parsedTorrentName.hasMovies = parsedTorrentName.complete || !!torrent.title.match(/movies?(?:\W|$)/);
   return torrentFiles(torrent)
-      .then(files => files
+      .then((files) => files
-          .filter(file => file.size > MIN_SIZE)
+          .filter((file) => file.size > MIN_SIZE)
-          .map(file => parseFile(file, parsedTorrentName)))
+          .map((file) => parseSeriesFile(file, parsedTorrentName)))
-      .then(files => decomposeAbsoluteEpisodes(files, torrent, imdbId))
+      .then((files) => decomposeAbsoluteEpisodes(files, torrent, imdbId))
-      .then(files => files
+      .then((files) => Promise.all(files.map(file => file.isMovie
-          .filter(file => file.season && file.episodes && file.episodes.length)
+          ? mapSeriesMovie(file, torrent.infoHash)
-          .map(file => file.episodes.map(episode => ({
+          : mapSeriesEpisode(file, torrent.infoHash, imdbId))))
-              infoHash: torrent.infoHash,
+      .then((files) => files.reduce((a, b) => a.concat(b), []))
-              fileIndex: file.fileIndex,
+      .catch((error) => {
-              title: file.name,
-              size: file.size,
-              imdbId: imdbId,
-              imdbSeason: file.season,
-              imdbEpisode: episode})))
-          .reduce((a, b) => a.concat(b), []))
-      .catch(error => {
         console.log(`Failed getting files for ${torrent.title}`, error.message);
         return [];
       });
 }
 
-function parseFile(file, parsedTorrentName) {
+async function mapSeriesEpisode(file, infoHash, imdbId) {
+  if (!file.episodes) {
+    return Promise.resolve([]);
+  }
+  return Promise.resolve(file.episodes.map(episode => ({
+    infoHash: infoHash,
+    fileIndex: file.fileIndex,
+    title: file.name,
+    size: file.size,
+    imdbId: imdbId,
+    imdbSeason: file.season,
+    imdbEpisode: episode})))
+}
+
+async function mapSeriesMovie(file, infoHash) {
+  return findMovieImdbId(file).then((imdbId) => [{
+    infoHash: infoHash,
+    fileIndex: file.fileIndex,
+    title: file.name,
+    size: file.size,
+    imdbId: imdbId
+  }])
+}
+
+function parseSeriesFile(file, parsedTorrentName) {
   const fileInfo = parse(file.name);
   // the episode may be in a folder containing season number
   if (!fileInfo.season && parsedTorrentName.season) {
@@ -58,10 +96,21 @@ function parseFile(file, parsedTorrentName) {
     const pathInfo = parse(folders[folders.length - 2]);
     fileInfo.season = pathInfo.season;
   }
+  fileInfo.isMovie = parsedTorrentName.hasMovies && !fileInfo.season && !fileInfo.episodes || !!fileInfo.year;
 
   return { ...file, ...fileInfo };
 }
 
+function findMovieImdbId(title) {
+  const parsedTitle = typeof title === 'string' ? parse(title) : title;
+  const searchQuery = {
+    name: escapeTitle(parsedTitle.title).toLowerCase(),
+    year: parsedTitle.year,
+    type: Type.MOVIE
+  };
+  return getImdbId(searchQuery).catch((error) => undefined);
+}
+
 async function decomposeAbsoluteEpisodes(files, torrent, imdbId) {
   if (files.every((file) => !file.episodes || file.episodes.every((ep) => ep < 100))) {
     return files; // nothing to decompose

package-lock.json

@@ -1714,7 +1714,7 @@
       }
     },
     "parse-torrent-title": {
-      "version": "git://github.com/TheBeastLT/parse-torrent-title.git#986c51f919c09e8f3a7937d22c8aea1fb9759090",
+      "version": "git://github.com/TheBeastLT/parse-torrent-title.git#b85e60d555fac758b4827e0e130e723d66502afc",
       "from": "git://github.com/TheBeastLT/parse-torrent-title.git#master"
     },
     "parseurl": {

scrapers/manual/manual.js

@@ -0,0 +1,39 @@
+require('dotenv').config();
+const repository = require('../../lib/repository');
+const { parseTorrentFiles } = require('../../lib/torrentFiles');
+const { Type } = require('../../lib/types');
+
+async function addMissingEpisodes() {
+  const torrent = { infoHash: '0ec780c2c7f8d5b38e61827f0b53c77c3d22f955' };
+  const torrentFiles = await require('../../lib/torrent').torrentFiles(torrent);
+  const storedFiles = await repository.getFiles(torrent)
+      .then((files) => files.reduce((map, next) => (map[next.fileIndex] = next, map), {}));
+  const imdbId = Object.values(storedFiles)[0].imdbId;
+
+  torrentFiles
+      .filter((file) => !storedFiles[file.fileIndex])
+      .map((file) => ({
+        infoHash: torrent.infoHash,
+        fileIndex: file.fileIndex,
+        title: file.name,
+        size: file.size,
+        imdbId: imdbId,
+        imdbSeason: parseInt(file.name.match(/(\d+)[ .]?-[ .]?\d+/)[1], 10),
+        imdbEpisode: parseInt(file.name.match(/\d+[ .]?-[ .]?(\d+)/)[1], 10),
+      }))
+      .forEach((file) => repository.createFile(file));
+}
+
+async function findAllFiles() {
+  const torrent = {
+    infoHash: '6b95e5cfde9aaa71970a14f6bb6b9de19e2cbfa1',
+    title: '[OMDA] Bleach + Filmes + Ovas (480p-720p x264 AAC-MP3) [rich_jc]',
+    type: Type.SERIES
+  };
+  const imdbId = 'tt0434665';
+
+  return parseTorrentFiles(torrent, imdbId).then((files) => console.log(files));
+}
+
+//addMissingEpisodes().then(() => console.log('Finished'));
+findAllFiles().then(() => console.log('Finished'));

scrapers/piratebay_dump.js

@@ -20,7 +20,7 @@ const limiter = new Bottleneck({maxConcurrent: 40});
 async function scrape() {
   const lastScraped = await repository.getProvider({ name: NAME });
   const lastDump = { updatedAt: 2147000000 };
-  const checkPoint = moment('2019-03-30 00:00:00', 'YYYY-MMM-DD HH:mm:ss').toDate();
+  const checkPoint = moment('2016-06-17 00:00:00', 'YYYY-MMM-DD HH:mm:ss').toDate();
   //const lastDump = await pirata.dumps().then((dumps) => dumps.sort((a, b) => b.updatedAt - a.updatedAt)[0]);
 
   if (!lastScraped.lastScraped || lastScraped.lastScraped < lastDump.updatedAt) {