diff --git a/lib/cache.js b/lib/cache.js index 557721d..5509694 100644 --- a/lib/cache.js +++ b/lib/cache.js @@ -1,4 +1,3 @@ - const cacheManager = require('cache-manager'); const mangodbStore = require('cache-manager-mongodb'); diff --git a/lib/repository.js b/lib/repository.js index 34cb0f3..2280a1e 100644 --- a/lib/repository.js +++ b/lib/repository.js @@ -82,6 +82,10 @@ function createFile(file) { return File.upsert(file); } +function getFiles(torrent) { + return File.findAll({ where: { infoHash: torrent.infoHash } }); +} + function getSkipTorrent(torrent) { return SkipTorrent.findByPk(torrent.infoHash) .then((result) =>{ @@ -110,4 +114,4 @@ function createFailedImdbTorrent(torrent) { return FailedImdbTorrent.upsert(torrent); } -module.exports = { connect, getProvider, updateProvider, getTorrent, createTorrent, createFile, getSkipTorrent, createSkipTorrent, createFailedImdbTorrent }; \ No newline at end of file +module.exports = { connect, getProvider, updateProvider, getTorrent, createTorrent, createFile, getFiles, getSkipTorrent, createSkipTorrent, createFailedImdbTorrent }; \ No newline at end of file diff --git a/lib/torrentFiles.js b/lib/torrentFiles.js index c4aba6b..389772a 100644 --- a/lib/torrentFiles.js +++ b/lib/torrentFiles.js @@ -1,12 +1,32 @@ const { torrentFiles } = require('../lib/torrent'); -const { getMetadata } = require('../lib/metadata'); +const { escapeTitle, getMetadata, getImdbId } = require('../lib/metadata'); const { parse } = require('parse-torrent-title'); const { Type } = require('./types'); const MIN_SIZE = 20 * 1024 * 1024; // 20 MB async function parseTorrentFiles(torrent, imdbId) { + const parsedTorrentName = parse(torrent.title); + if (torrent.type === Type.MOVIE) { + if (parsedTorrentName.complete) { + return torrentFiles(torrent) + .then(files => files.filter(file => file.size > MIN_SIZE)) + .then(files => Promise.all(files + .map((file) => findMovieImdbId(file.name) + .then((newImdbId) => ({ + infoHash: torrent.infoHash, + fileIndex: file.fileIndex, + title: file.name, + size: file.size, + imdbId: newImdbId, + }))))) + .catch(error => { + console.log(`Failed getting files for ${torrent.title}`, error.message); + return []; + }); + } + return [{ infoHash: torrent.infoHash, title: torrent.title, @@ -14,7 +34,7 @@ async function parseTorrentFiles(torrent, imdbId) { imdbId: imdbId, }]; } - const parsedTorrentName = parse(torrent.title); + if (parsedTorrentName.season && parsedTorrentName.episode) { return [{ infoHash: torrent.infoHash, @@ -26,29 +46,47 @@ async function parseTorrentFiles(torrent, imdbId) { }]; } + parsedTorrentName.hasMovies = parsedTorrentName.complete || !!torrent.title.match(/movies?(?:\W|$)/); return torrentFiles(torrent) - .then(files => files - .filter(file => file.size > MIN_SIZE) - .map(file => parseFile(file, parsedTorrentName))) - .then(files => decomposeAbsoluteEpisodes(files, torrent, imdbId)) - .then(files => files - .filter(file => file.season && file.episodes && file.episodes.length) - .map(file => file.episodes.map(episode => ({ - infoHash: torrent.infoHash, - fileIndex: file.fileIndex, - title: file.name, - size: file.size, - imdbId: imdbId, - imdbSeason: file.season, - imdbEpisode: episode}))) - .reduce((a, b) => a.concat(b), [])) - .catch(error => { + .then((files) => files + .filter((file) => file.size > MIN_SIZE) + .map((file) => parseSeriesFile(file, parsedTorrentName))) + .then((files) => decomposeAbsoluteEpisodes(files, torrent, imdbId)) + .then((files) => Promise.all(files.map(file => file.isMovie + ? mapSeriesMovie(file, torrent.infoHash) + : mapSeriesEpisode(file, torrent.infoHash, imdbId)))) + .then((files) => files.reduce((a, b) => a.concat(b), [])) + .catch((error) => { console.log(`Failed getting files for ${torrent.title}`, error.message); return []; }); } -function parseFile(file, parsedTorrentName) { +async function mapSeriesEpisode(file, infoHash, imdbId) { + if (!file.episodes) { + return Promise.resolve([]); + } + return Promise.resolve(file.episodes.map(episode => ({ + infoHash: infoHash, + fileIndex: file.fileIndex, + title: file.name, + size: file.size, + imdbId: imdbId, + imdbSeason: file.season, + imdbEpisode: episode}))) +} + +async function mapSeriesMovie(file, infoHash) { + return findMovieImdbId(file).then((imdbId) => [{ + infoHash: infoHash, + fileIndex: file.fileIndex, + title: file.name, + size: file.size, + imdbId: imdbId + }]) +} + +function parseSeriesFile(file, parsedTorrentName) { const fileInfo = parse(file.name); // the episode may be in a folder containing season number if (!fileInfo.season && parsedTorrentName.season) { @@ -58,10 +96,21 @@ function parseFile(file, parsedTorrentName) { const pathInfo = parse(folders[folders.length - 2]); fileInfo.season = pathInfo.season; } + fileInfo.isMovie = parsedTorrentName.hasMovies && !fileInfo.season && !fileInfo.episodes || !!fileInfo.year; return { ...file, ...fileInfo }; } +function findMovieImdbId(title) { + const parsedTitle = typeof title === 'string' ? parse(title) : title; + const searchQuery = { + name: escapeTitle(parsedTitle.title).toLowerCase(), + year: parsedTitle.year, + type: Type.MOVIE + }; + return getImdbId(searchQuery).catch((error) => undefined); +} + async function decomposeAbsoluteEpisodes(files, torrent, imdbId) { if (files.every((file) => !file.episodes || file.episodes.every((ep) => ep < 100))) { return files; // nothing to decompose diff --git a/package-lock.json b/package-lock.json index fdd6a1b..0d0ed57 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1714,7 +1714,7 @@ } }, "parse-torrent-title": { - "version": "git://github.com/TheBeastLT/parse-torrent-title.git#986c51f919c09e8f3a7937d22c8aea1fb9759090", + "version": "git://github.com/TheBeastLT/parse-torrent-title.git#b85e60d555fac758b4827e0e130e723d66502afc", "from": "git://github.com/TheBeastLT/parse-torrent-title.git#master" }, "parseurl": { diff --git a/scrapers/manual/manual.js b/scrapers/manual/manual.js new file mode 100644 index 0000000..6e502ed --- /dev/null +++ b/scrapers/manual/manual.js @@ -0,0 +1,39 @@ +require('dotenv').config(); +const repository = require('../../lib/repository'); +const { parseTorrentFiles } = require('../../lib/torrentFiles'); +const { Type } = require('../../lib/types'); + +async function addMissingEpisodes() { + const torrent = { infoHash: '0ec780c2c7f8d5b38e61827f0b53c77c3d22f955' }; + const torrentFiles = await require('../../lib/torrent').torrentFiles(torrent); + const storedFiles = await repository.getFiles(torrent) + .then((files) => files.reduce((map, next) => (map[next.fileIndex] = next, map), {})); + const imdbId = Object.values(storedFiles)[0].imdbId; + + torrentFiles + .filter((file) => !storedFiles[file.fileIndex]) + .map((file) => ({ + infoHash: torrent.infoHash, + fileIndex: file.fileIndex, + title: file.name, + size: file.size, + imdbId: imdbId, + imdbSeason: parseInt(file.name.match(/(\d+)[ .]?-[ .]?\d+/)[1], 10), + imdbEpisode: parseInt(file.name.match(/\d+[ .]?-[ .]?(\d+)/)[1], 10), + })) + .forEach((file) => repository.createFile(file)); +} + +async function findAllFiles() { + const torrent = { + infoHash: '6b95e5cfde9aaa71970a14f6bb6b9de19e2cbfa1', + title: '[OMDA] Bleach + Filmes + Ovas (480p-720p x264 AAC-MP3) [rich_jc]', + type: Type.SERIES + }; + const imdbId = 'tt0434665'; + + return parseTorrentFiles(torrent, imdbId).then((files) => console.log(files)); +} + +//addMissingEpisodes().then(() => console.log('Finished')); +findAllFiles().then(() => console.log('Finished')); \ No newline at end of file diff --git a/scrapers/piratebay_dump.js b/scrapers/piratebay_dump.js index 37ceeb3..b242aab 100644 --- a/scrapers/piratebay_dump.js +++ b/scrapers/piratebay_dump.js @@ -20,7 +20,7 @@ const limiter = new Bottleneck({maxConcurrent: 40}); async function scrape() { const lastScraped = await repository.getProvider({ name: NAME }); const lastDump = { updatedAt: 2147000000 }; - const checkPoint = moment('2019-03-30 00:00:00', 'YYYY-MMM-DD HH:mm:ss').toDate(); + const checkPoint = moment('2016-06-17 00:00:00', 'YYYY-MMM-DD HH:mm:ss').toDate(); //const lastDump = await pirata.dumps().then((dumps) => dumps.sort((a, b) => b.updatedAt - a.updatedAt)[0]); if (!lastScraped.lastScraped || lastScraped.lastScraped < lastDump.updatedAt) {