[scraper] adds nyaa pantsu scraper

scraper/lib/metadata.js

@@ -5,7 +5,7 @@ const bing = require('nodejs-bing');
 const he = require('he');
 const { cacheWrapImdbId, cacheWrapKitsuId, cacheWrapMetadata } = require('./cache');
 const { Type } = require('./types');
-const { getRandomUserAgent } = require('./request_helper');
+const { getRandomUserAgent } = require('./requestHelper');
 
 const CINEMETA_URL = 'https://v3-cinemeta.strem.io';
 const KITSU_URL = 'https://anime-kitsu.strem.fun';
@@ -18,24 +18,23 @@ function getMetadata(id, type = Type.SERIES) {
 
   const key = Number.isInteger(id) || id.match(/^\d+$/) ? `kitsu:${id}` : id;
   const metaType = type === Type.MOVIE ? Type.MOVIE : Type.SERIES;
-  return cacheWrapMetadata(key,
-      () => _requestMetadata(`${KITSU_URL}/meta/${metaType}/${key}.json`)
-          .catch(() => _requestMetadata(`${CINEMETA_URL}/meta/${metaType}/${key}.json`))
-          .catch(() => {
-            // try different type in case there was a mismatch
-            const otherType = metaType === Type.MOVIE ? Type.SERIES : Type.MOVIE;
-            return _requestMetadata(`${CINEMETA_URL}/meta/${otherType}/${key}.json`)
-          })
-          .catch((error) => {
-            throw new Error(`failed metadata query ${key} due: ${error.message}`);
-          }));
+  return cacheWrapMetadata(key, () => _requestMetadata(`${KITSU_URL}/meta/${metaType}/${key}.json`)
+      .catch(() => _requestMetadata(`${CINEMETA_URL}/meta/${metaType}/${key}.json`))
+      .catch(() => {
+        // try different type in case there was a mismatch
+        const otherType = metaType === Type.MOVIE ? Type.SERIES : Type.MOVIE;
+        return _requestMetadata(`${CINEMETA_URL}/meta/${otherType}/${key}.json`)
+      })
+      .catch((error) => {
+        throw new Error(`failed metadata query ${key} due: ${error.message}`);
+      }));
 }
 
 function _requestMetadata(url) {
   return needle('get', url, { open_timeout: TIMEOUT })
       .then((response) => {
         const body = response.body;
-        if (body && body.meta && body.meta.id) {
+        if (body && body.meta && (body.meta.imdb_id || body.meta.kitsu_id)) {
           return {
             kitsuId: body.meta.kitsu_id,
             imdbId: body.meta.imdb_id,

scraper/lib/parseHelper.js

@@ -0,0 +1,65 @@
+const { parse } = require('parse-torrent-title');
+const { Type } = require('./types');
+
+function parseSeriesVideos(torrent, videos) {
+  const parsedTorrentName = parse(torrent.title);
+  const hasMovies = parsedTorrentName.complete || !!torrent.title.match(/movies?(?:\W|$)/i);
+  const parsedVideos = videos.map(video => parseSeriesVideo(video, parsedTorrentName));
+  return parsedVideos.map(video => ({ ...video, isMovie: isMovieVideo(video, parsedVideos, torrent.type, hasMovies) }));
+}
+
+function parseSeriesVideo(video, parsedTorrentName) {
+  const videoInfo = parse(video.name);
+  // the episode may be in a folder containing season number
+  if (!videoInfo.season && video.path.includes('/')) {
+    const folders = video.path.split('/');
+    const pathInfo = parse(folders[folders.length - 2]);
+    videoInfo.season = pathInfo.season;
+  }
+  if (!videoInfo.season && parsedTorrentName.season) {
+    videoInfo.season = parsedTorrentName.season;
+  }
+  if (!videoInfo.season && videoInfo.seasons && videoInfo.seasons.length > 1) {
+    // in case single file was interpreted as having multiple seasons
+    videoInfo.season = videoInfo.seasons[0];
+  }
+  // sometimes video file does not have correct date format as in torrent title
+  if (!videoInfo.episodes && !videoInfo.date && parsedTorrentName.date) {
+    videoInfo.date = parsedTorrentName.date;
+  }
+  // force episode to any found number if it was not parsed
+  if (!videoInfo.episodes && !videoInfo.date) {
+    const epMatcher = videoInfo.title.match(
+        /(?<!season\W*|disk\W*|movie\W*|film\W*)(?:^|\W)(\d{1,4})(?:a|b|c|v\d)?(?:\W|$)(?!disk|movie|film)/i);
+    videoInfo.episodes = epMatcher && [parseInt(epMatcher[1], 10)];
+    videoInfo.episode = videoInfo.episodes && videoInfo.episodes[0];
+  }
+
+  return { ...video, ...videoInfo };
+}
+
+function isMovieVideo(video, otherVideos, type, hasMovies) {
+  if (Number.isInteger(video.season)) {
+    // not movie if video has season
+    return false;
+  }
+  if (video.name.match(/\b(?:\d+[ .]movie|movie[ .]\d+)\b/i)) {
+    // movie if video explicitly has numbered movie keyword in the name, ie. 1 Movie or Movie 1
+    return true;
+  }
+  if (!hasMovies && type !== Type.ANIME) {
+    // not movie if torrent name does not contain movies keyword or is not a pack torrent and is not anime
+    return false;
+  }
+  if (!video.episodes) {
+    // movie if there's no episode info it could be a movie
+    return true;
+  }
+  // movie if contains year info and there aren't more than 3 video with same title and year
+  // as some series titles might contain year in it.
+  return !!video.year
+      && otherVideos.length > 3
+      && otherVideos.filter(other => other.title === video.title && other.year === video.year) < 3;
+}
+
+module.exports = { parseSeriesVideos }

scraper/lib/torrentFiles.js

@@ -4,6 +4,7 @@ const { parse } = require('parse-torrent-title');
 const Promises = require('../lib/promises');
 const { torrentFiles } = require('../lib/torrent');
 const { getMetadata, getImdbId, getKitsuId } = require('../lib/metadata');
+const { parseSeriesVideos } = require('../lib/parseHelper');
 const { Type } = require('./types');
 const { isDisk } = require('./extension');
 
@@ -12,8 +13,8 @@ const MULTIPLE_FILES_SIZE = 4 * 1024 * 1024 * 1024; // 4 GB
 
 async function parseTorrentFiles(torrent) {
   const parsedTorrentName = parse(torrent.title);
-  parsedTorrentName.hasMovies = parsedTorrentName.complete || !!torrent.title.match(/movies?(?:\W|$)/i);
   const metadata = await getMetadata(torrent.kitsuId || torrent.imdbId, torrent.type || Type.MOVIE)
+      .then(meta => Object.assign({}, meta))
       .catch(() => undefined);
 
   // if (metadata && metadata.type !== torrent.type && torrent.type !== Type.ANIME) {
@@ -66,9 +67,8 @@ async function parseMovieFiles(torrent, parsedName, metadata) {
 async function parseSeriesFiles(torrent, parsedName, metadata) {
   const { contents, videos, subtitles } = await getSeriesTorrentContent(torrent, parsedName);
   const parsedVideos = await Promise.resolve(videos)
-      .then(videos => videos
-          .filter(video => video.size > MIN_SIZE)
-          .map(video => parseSeriesFile(video, parsedName, torrent.type)))
+      .then(videos => videos.filter(video => videos.length === 1 || video.size > MIN_SIZE))
+      .then(videos => parseSeriesVideos(torrent, videos))
       .then(videos => decomposeEpisodes(torrent, videos, metadata))
       .then(videos => assignKitsuOrImdbEpisodes(torrent, videos, metadata))
       .then(videos => Promise.all(videos.map(video => video.isMovie
@@ -96,7 +96,7 @@ async function getMoviesTorrentContent(torrent, parsedName) {
 }
 
 async function getSeriesTorrentContent(torrent, parsedName) {
-  const hasMultipleEpisodes = parsedName.complete || parsedName.hasMovies || torrent.size > MULTIPLE_FILES_SIZE ||
+  const hasMultipleEpisodes = parsedName.complete || torrent.size > MULTIPLE_FILES_SIZE ||
       (parsedName.seasons && parsedName.seasons.length > 1);
   const hasSingleEpisode = Number.isInteger(parsedName.episode) || (!parsedName.episodes && parsedName.date);
   return torrentFiles(torrent)
@@ -138,7 +138,7 @@ async function mapSeriesEpisode(file, torrent, files) {
 async function mapSeriesMovie(file, torrent) {
   const kitsuId = torrent.type === Type.ANIME ? await findMovieKitsuId(file) : undefined;
   const imdbId = !kitsuId ? await findMovieImdbId(file) : undefined;
-  const metadata = getMetadata(imdbId, Type.MOVIE).catch(() => undefined);
+  const metadata = await getMetadata(kitsuId || imdbId, Type.MOVIE).catch(() => undefined);
   return [{
     infoHash: torrent.infoHash,
     fileIndex: file.fileIndex,
@@ -149,39 +149,6 @@ async function mapSeriesMovie(file, torrent) {
   }];
 }
 
-function parseSeriesFile(file, parsedTorrentName, type) {
-  const fileInfo = parse(file.name);
-  // the episode may be in a folder containing season number
-  if (!fileInfo.season && file.path.includes('/')) {
-    const folders = file.path.split('/');
-    const pathInfo = parse(folders[folders.length - 2]);
-    fileInfo.season = pathInfo.season;
-  }
-  if (!fileInfo.season && parsedTorrentName.season) {
-    fileInfo.season = parsedTorrentName.season;
-  }
-  if (!fileInfo.season && fileInfo.seasons && fileInfo.seasons.length > 1) {
-    // in case single file was interpreted as having multiple seasons
-    fileInfo.season = fileInfo.seasons[0];
-  }
-  // sometimes video file does not have correct date format as in torrent title
-  if (!fileInfo.episodes && !fileInfo.date && parsedTorrentName.date) {
-    fileInfo.date = parsedTorrentName.date;
-  }
-  // force episode to any found number if it was not parsed
-  if (!fileInfo.episodes && !fileInfo.date) {
-    const epMatcher = fileInfo.title.match(
-        /(?<!season\W*|disk\W*|movie\W*|film\W*)(?:^|\W)(\d{1,4})(?:a|b|c|v\d)?(?:\W|$)(?!disk|movie|film)/i);
-    fileInfo.episodes = epMatcher && [parseInt(epMatcher[1], 10)];
-    fileInfo.episode = fileInfo.episodes && fileInfo.episodes[0];
-  }
-  fileInfo.isMovie = ((parsedTorrentName.hasMovies || type === Type.ANIME)
-      && !fileInfo.season && (!fileInfo.episodes || !!fileInfo.year))
-      || (!fileInfo.season && !!file.name.match(/\b(?:\d+[ .]movie|movie[ .]\d+)\b/i));
-
-  return { ...file, ...fileInfo };
-}
-
 async function decomposeEpisodes(torrent, files, metadata = { episodeCount: [] }) {
   if (files.every(file => !file.episodes && !file.date)) {
     return files;
@@ -413,7 +380,8 @@ async function updateToCinemetaMetadata(metadata) {
         metadata.episodeCount = newMetadata.episodeCount;
         metadata.totalCount = newMetadata.totalCount;
         return metadata;
-      });
+      })
+      .catch(error => console.warn(`Failed ${metadata.imdbId} metadata cinemeta update due: ${error.message}`));
 }
 
 function findMovieImdbId(title) {