diff --git a/src/node/consumer/src/lib/interfaces/parsable_torrent_file.ts b/src/node/consumer/src/lib/interfaces/parsable_torrent_file.ts deleted file mode 100644 index 394570f..0000000 --- a/src/node/consumer/src/lib/interfaces/parsable_torrent_file.ts +++ /dev/null @@ -1,12 +0,0 @@ -import {ParseTorrentTitleResult} from "./parse_torrent_title_result"; - -export interface ParsableTorrentFile extends ParseTorrentTitleResult { - name?: string; - path?: string; - size?: number; - length?: number; - fileIndex?: number; - isMovie?: boolean; - imdbId?: string | number; - kitsuId?: number | string; -} \ No newline at end of file diff --git a/src/node/consumer/src/lib/interfaces/parsed_torrent.ts b/src/node/consumer/src/lib/interfaces/parsed_torrent.ts new file mode 100644 index 0000000..fad6616 --- /dev/null +++ b/src/node/consumer/src/lib/interfaces/parsed_torrent.ts @@ -0,0 +1,18 @@ +import {ParseTorrentTitleResult} from "./parse_torrent_title_result"; +import {TorrentType} from "../enums/torrent_types"; +import {TorrentFileCollection} from "./torrent_file_collection"; + +export interface ParsedTorrent extends ParseTorrentTitleResult { + size?: number; + isPack?: boolean; + imdbId?: string | number; + kitsuId?: string | number; + trackers?: string; + provider?: string | null; + infoHash: string | null; + type: string | TorrentType; + uploadDate?: Date; + seeders?: number; + torrentId?: string; + fileCollection?: TorrentFileCollection; +} \ No newline at end of file diff --git a/src/node/consumer/src/lib/interfaces/torrent_file_collection.ts b/src/node/consumer/src/lib/interfaces/torrent_file_collection.ts index 500c5b0..1e00090 100644 --- a/src/node/consumer/src/lib/interfaces/torrent_file_collection.ts +++ b/src/node/consumer/src/lib/interfaces/torrent_file_collection.ts @@ -1,7 +1,9 @@ -import {ParsableTorrentFile} from "./parsable_torrent_file"; +import {ContentAttributes} from "../../repository/interfaces/content_attributes"; +import {FileAttributes} from "../../repository/interfaces/file_attributes"; +import {SubtitleAttributes} from "../../repository/interfaces/subtitle_attributes"; export interface TorrentFileCollection { - contents?: ParsableTorrentFile[]; - videos?: ParsableTorrentFile[]; - subtitles?: ParsableTorrentFile[]; + contents?: ContentAttributes[]; + videos?: FileAttributes[]; + subtitles?: SubtitleAttributes[]; } \ No newline at end of file diff --git a/src/node/consumer/src/lib/interfaces/torrent_info.ts b/src/node/consumer/src/lib/interfaces/torrent_info.ts deleted file mode 100644 index 93f7082..0000000 --- a/src/node/consumer/src/lib/interfaces/torrent_info.ts +++ /dev/null @@ -1,16 +0,0 @@ -import {TorrentType} from "../enums/torrent_types"; - -export interface TorrentInfo { - title: string | null; - torrentId: string; - infoHash: string | null; - seeders: number; - uploadDate: Date; - imdbId?: string; - kitsuId?: string; - type: TorrentType; - provider?: string | null; - trackers: string; - size?: number; - pack?: boolean; -} \ No newline at end of file diff --git a/src/node/consumer/src/lib/services/parsing_service.ts b/src/node/consumer/src/lib/services/parsing_service.ts deleted file mode 100644 index bb8da27..0000000 --- a/src/node/consumer/src/lib/services/parsing_service.ts +++ /dev/null @@ -1,106 +0,0 @@ -import { parse } from 'parse-torrent-title'; -import { TorrentType } from '../enums/torrent_types'; -import {ParseTorrentTitleResult} from "../interfaces/parse_torrent_title_result"; -import {ParsableTorrentFile} from "../interfaces/parsable_torrent_file"; -import {TorrentInfo} from "../interfaces/torrent_info"; - -class ParsingService { - private readonly MULTIPLE_FILES_SIZE = 4 * 1024 * 1024 * 1024; // 4 GB - - public parseSeriesVideos(torrent: TorrentInfo, videos: ParsableTorrentFile[]): ParsableTorrentFile[] { - const parsedTorrentName = parse(torrent.title); - const hasMovies = parsedTorrentName.complete || !!torrent.title.match(/movies?(?:\W|$)/i); - const parsedVideos = videos.map(video => this.parseSeriesVideo(video, parsedTorrentName)); - return parsedVideos.map(video => ({ ...video, isMovie: this.isMovieVideo(video, parsedVideos, torrent.type, hasMovies) })); - } - - public isPackTorrent(torrent: TorrentInfo): boolean { - if (torrent.pack) { - return true; - } - const parsedInfo = parse(torrent.title); - if (torrent.type === TorrentType.Movie) { - return parsedInfo.complete || typeof parsedInfo.year === 'string' || /movies/i.test(torrent.title); - } - const hasMultipleEpisodes = parsedInfo.complete || - torrent.size > this.MULTIPLE_FILES_SIZE || - (parsedInfo.seasons && parsedInfo.seasons.length > 1) || - (parsedInfo.episodes && parsedInfo.episodes.length > 1) || - (parsedInfo.seasons && !parsedInfo.episodes); - const hasSingleEpisode = Number.isInteger(parsedInfo.episode) || (!parsedInfo.episodes && parsedInfo.date); - return hasMultipleEpisodes && !hasSingleEpisode; - } - - private parseSeriesVideo(video: ParsableTorrentFile, parsedTorrentName: ParseTorrentTitleResult): ParseTorrentTitleResult { - const videoInfo = parse(video.name); - // the episode may be in a folder containing season number - if (!Number.isInteger(videoInfo.season) && video.path.includes('/')) { - const folders = video.path.split('/'); - const pathInfo = parse(folders[folders.length - 2]); - videoInfo.season = pathInfo.season; - } - if (!Number.isInteger(videoInfo.season) && parsedTorrentName.season) { - videoInfo.season = parsedTorrentName.season; - } - if (!Number.isInteger(videoInfo.season) && videoInfo.seasons && videoInfo.seasons.length > 1) { - // in case single file was interpreted as having multiple seasons - videoInfo.season = videoInfo.seasons[0]; - } - if (!Number.isInteger(videoInfo.season) && video.path.includes('/') && parsedTorrentName.seasons - && parsedTorrentName.seasons.length > 1) { - // russian season are usually named with 'series name-2` i.e. Улицы разбитых фонарей-6/22. Одиночный выстрел.mkv - const folderPathSeasonMatch = video.path.match(/[\u0400-\u04ff]-(\d{1,2})(?=.*\/)/); - videoInfo.season = folderPathSeasonMatch && parseInt(folderPathSeasonMatch[1], 10) || undefined; - } - // sometimes video file does not have correct date format as in torrent title - if (!videoInfo.episodes && !videoInfo.date && parsedTorrentName.date) { - videoInfo.date = parsedTorrentName.date; - } - // limit number of episodes in case of incorrect parsing - if (videoInfo.episodes && videoInfo.episodes.length > 20) { - videoInfo.episodes = [videoInfo.episodes[0]]; - videoInfo.episode = videoInfo.episodes[0]; - } - // force episode to any found number if it was not parsed - if (!videoInfo.episodes && !videoInfo.date) { - const epMatcher = videoInfo.title.match( - /(? 3 - && otherVideos.filter(other => other.title === video.title && other.year === video.year).length < 3; - } -} - -export const parsingService = new ParsingService(); - diff --git a/src/node/consumer/src/lib/services/torrent_download_service.ts b/src/node/consumer/src/lib/services/torrent_download_service.ts index 58a7241..dae27e1 100644 --- a/src/node/consumer/src/lib/services/torrent_download_service.ts +++ b/src/node/consumer/src/lib/services/torrent_download_service.ts @@ -1,10 +1,20 @@ -import { encode } from 'magnet-uri'; +import {encode} from 'magnet-uri'; import torrentStream from 'torrent-stream'; -import { configurationService } from './configuration_service'; +import {configurationService} from './configuration_service'; import {extensionService} from './extension_service'; -import {TorrentInfo} from "../interfaces/torrent_info"; import {TorrentFileCollection} from "../interfaces/torrent_file_collection"; -import {ParsableTorrentFile} from "../interfaces/parsable_torrent_file"; +import {ParsedTorrent} from "../interfaces/parsed_torrent"; +import {FileAttributes} from "../../repository/interfaces/file_attributes"; +import {SubtitleAttributes} from "../../repository/interfaces/subtitle_attributes"; +import {ContentAttributes} from "../../repository/interfaces/content_attributes"; +import {parse} from "parse-torrent-title"; + +interface TorrentFile { + name: string; + path: string; + length: number; + fileIndex: number; +} class TorrentDownloadService { private engineOptions: TorrentStream.TorrentEngineOptions = { @@ -14,73 +24,124 @@ class TorrentDownloadService { dht: false, tracker: true, }; - - public async getTorrentFiles(torrent: TorrentInfo, timeout: number = 30000): Promise { - return this.filesFromTorrentStream(torrent, timeout) - .then((files: Array) => ({ - contents: files, - videos: this.filterVideos(files), - subtitles: this.filterSubtitles(files) - })); + + public async getTorrentFiles(torrent: ParsedTorrent, timeout: number = 30000): Promise { + const torrentFiles: TorrentFile[] = await this.filesFromTorrentStream(torrent, timeout); + + const videos = this.filterVideos(torrent, torrentFiles); + const subtitles = this.filterSubtitles(torrent, torrentFiles); + const contents = this.createContent(torrent, torrentFiles); + + return { + contents: contents, + videos: videos, + subtitles: subtitles, + }; } - private async filesFromTorrentStream(torrent: TorrentInfo, timeout: number): Promise> { + private async filesFromTorrentStream(torrent: ParsedTorrent, timeout: number): Promise { if (!torrent.infoHash) { return Promise.reject(new Error("No infoHash...")); } - const magnet = encode({ infoHash: torrent.infoHash, announce: torrent.trackers.split(',') }); + const magnet = encode({infoHash: torrent.infoHash, announce: torrent.trackers.split(',')}); return new Promise((resolve, reject) => { let engine: TorrentStream.TorrentEngine; const timeoutId = setTimeout(() => { - engine.destroy(() => {}); + engine.destroy(() => { + }); reject(new Error('No available connections for torrent!')); }, timeout); engine = torrentStream(magnet, this.engineOptions); engine.on("ready", () => { - const files: ParsableTorrentFile[] = engine.files.map((file, fileId) => ({ + const files: TorrentFile[] = engine.files.map((file, fileId) => ({ ...file, fileIndex: fileId, size: file.length, - title: file.name})); - + title: file.name + })); + resolve(files); - engine.destroy(() => {}); + engine.destroy(() => { + }); clearTimeout(timeoutId); }); }); } - private filterVideos(files: Array): Array { - if (files.length === 1 && !Number.isInteger(files[0].fileIndex)) { - return files; + private filterVideos(torrent: ParsedTorrent, torrentFiles: TorrentFile[]): FileAttributes[] { + if (torrentFiles.length === 1 && !Number.isInteger(torrentFiles[0].fileIndex)) { + return [this.mapTorrentFileToFileAttributes(torrent, torrentFiles[0])]; } - const videos = files.filter(file => extensionService.isVideo(file.path || '')); - const maxSize = Math.max(...videos.map((video: ParsableTorrentFile) => video.length)); + const videos = torrentFiles.filter(file => extensionService.isVideo(file.path || '')); + const maxSize = Math.max(...videos.map((video: TorrentFile) => video.length)); const minSampleRatio = videos.length <= 3 ? 3 : 10; const minAnimeExtraRatio = 5; const minRedundantRatio = videos.length <= 3 ? 30 : Number.MAX_VALUE; - const isSample = (video: ParsableTorrentFile) => video.path?.match(/sample|bonus|promo/i) && maxSize / parseInt(video.length.toString()) > minSampleRatio; - const isRedundant = (video: ParsableTorrentFile) => maxSize / parseInt(video.length.toString()) > minRedundantRatio; - const isExtra = (video: ParsableTorrentFile) => video.path?.match(/extras?\//i); - const isAnimeExtra = (video: ParsableTorrentFile) => video.path?.match(/(?:\b|_)(?:NC)?(?:ED|OP|PV)(?:v?\d\d?)?(?:\b|_)/i) + + const isSample = (video: TorrentFile) => video.path?.match(/sample|bonus|promo/i) && maxSize / parseInt(video.path.toString()) > minSampleRatio; + const isRedundant = (video: TorrentFile) => maxSize / parseInt(video.path.toString()) > minRedundantRatio; + const isExtra = (video: TorrentFile) => video.path?.match(/extras?\//i); + const isAnimeExtra = (video: TorrentFile) => video.path?.match(/(?:\b|_)(?:NC)?(?:ED|OP|PV)(?:v?\d\d?)?(?:\b|_)/i) && maxSize / parseInt(video.length.toString()) > minAnimeExtraRatio; - const isWatermark = (video: ParsableTorrentFile) => video.path?.match(/^[A-Z-]+(?:\.[A-Z]+)?\.\w{3,4}$/) + const isWatermark = (video: TorrentFile) => video.path?.match(/^[A-Z-]+(?:\.[A-Z]+)?\.\w{3,4}$/) && maxSize / parseInt(video.length.toString()) > minAnimeExtraRatio + return videos .filter(video => !isSample(video)) .filter(video => !isExtra(video)) .filter(video => !isAnimeExtra(video)) .filter(video => !isRedundant(video)) - .filter(video => !isWatermark(video)); + .filter(video => !isWatermark(video)) + .map(video => this.mapTorrentFileToFileAttributes(torrent, video)); } - private filterSubtitles(files: Array): Array { - return files.filter(file => extensionService.isSubtitle(file.path || '')); + private filterSubtitles(torrent: ParsedTorrent, torrentFiles: TorrentFile[]): SubtitleAttributes[] { + return torrentFiles.filter(file => extensionService.isSubtitle(file.name || '')) + .map(file => this.mapTorrentFileToSubtitleAttributes(torrent, file)); + } + + private createContent(torrent: ParsedTorrent, torrentFiles: TorrentFile[]): ContentAttributes[] { + return torrentFiles.map(file => this.mapTorrentFileToContentAttributes(torrent, file)); + } + + private mapTorrentFileToFileAttributes(torrent: ParsedTorrent, file: TorrentFile): FileAttributes { + const videoFile: FileAttributes = { + title: file.name, + size: file.length, + fileIndex: file.fileIndex || 0, + infoHash: torrent.infoHash, + imdbId: torrent.imdbId.toString(), + imdbSeason: torrent.season || 0, + imdbEpisode: torrent.episode || 0, + kitsuId: parseInt(torrent.kitsuId.toString()) || 0, + kitsuEpisode: torrent.episode || 0 + }; + + return {...videoFile, ...parse(file.name)}; + } + + private mapTorrentFileToSubtitleAttributes(torrent: ParsedTorrent, file: TorrentFile): SubtitleAttributes { + return { + title: file.name, + infoHash: torrent.infoHash, + fileIndex: file.fileIndex, + fileId: file.fileIndex, + path: file.path, + }; + } + + private mapTorrentFileToContentAttributes(torrent: ParsedTorrent, file: TorrentFile): ContentAttributes { + return { + infoHash: torrent.infoHash, + fileIndex: file.fileIndex, + path: file.path, + size: file.length, + }; } } diff --git a/src/node/consumer/src/lib/services/torrent_entries_service.ts b/src/node/consumer/src/lib/services/torrent_entries_service.ts new file mode 100644 index 0000000..c4b9fcc --- /dev/null +++ b/src/node/consumer/src/lib/services/torrent_entries_service.ts @@ -0,0 +1,233 @@ +import {parse} from 'parse-torrent-title'; +import {ParsedTorrent} from "../interfaces/parsed_torrent"; +import {repository} from '../../repository/database_repository'; +import {TorrentType} from '../enums/torrent_types'; +import {TorrentFileCollection} from "../interfaces/torrent_file_collection"; +import {Torrent} from "../../repository/models/torrent"; +import {PromiseHelpers} from '../helpers/promises_helpers'; +import {logger} from './logging_service'; +import {metadataService} from './metadata_service'; +import {torrentFileService} from './torrent_file_service'; +import {torrentSubtitleService} from './torrent_subtitle_service'; +import {TorrentAttributes} from "../../repository/interfaces/torrent_attributes"; +import {File} from "../../repository/models/file"; +import {Subtitle} from "../../repository/models/subtitle"; + +class TorrentEntriesService { + public async createTorrentEntry(torrent: ParsedTorrent, overwrite = false): Promise { + const titleInfo = parse(torrent.title); + + if (!torrent.imdbId && torrent.type !== TorrentType.Anime) { + const imdbQuery = { + title: titleInfo.title, + year: titleInfo.year, + type: torrent.type + }; + torrent.imdbId = await metadataService.getImdbId(imdbQuery) + .catch(() => undefined); + } + if (torrent.imdbId && torrent.imdbId.toString().length < 9) { + // pad zeros to imdbId if missing + torrent.imdbId = 'tt' + torrent.imdbId.toString().replace('tt', '').padStart(7, '0'); + } + if (torrent.imdbId && torrent.imdbId.toString().length > 9 && torrent.imdbId.toString().startsWith('tt0')) { + // sanitize imdbId from redundant zeros + torrent.imdbId = torrent.imdbId.toString().replace(/tt0+([0-9]{7,})$/, 'tt$1'); + } + if (!torrent.kitsuId && torrent.type === TorrentType.Anime) { + const kitsuQuery = { + title: titleInfo.title, + year: titleInfo.year, + season: titleInfo.season, + }; + torrent.kitsuId = await metadataService.getKitsuId(kitsuQuery) + .catch(() => undefined); + } + + if (!torrent.imdbId && !torrent.kitsuId && !torrentFileService.isPackTorrent(torrent)) { + logger.warn(`imdbId or kitsuId not found: ${torrent.provider} ${torrent.title}`); + return; + } + + const fileCollection: TorrentFileCollection = await torrentFileService.parseTorrentFiles(torrent) + .then((torrentContents: TorrentFileCollection) => overwrite ? this.overwriteExistingFiles(torrent, torrentContents) : torrentContents) + .then((torrentContents: TorrentFileCollection) => torrentSubtitleService.assignSubtitles(torrentContents)) + .catch(error => { + logger.warn(`Failed getting files for ${torrent.title}`, error.message); + return {}; + }); + + if (!fileCollection.videos || !fileCollection.videos.length) { + logger.warn(`no video files found for ${torrent.provider} [${torrent.infoHash}] ${torrent.title}`); + return; + } + + const newTorrent: Torrent = Torrent.build({ + ...torrent, + contents: fileCollection.contents, + subtitles: fileCollection.subtitles + }); + + return repository.createTorrent(newTorrent) + .then(() => PromiseHelpers.sequence(fileCollection.videos.map(video => () => { + const newVideo = File.build(video); + return repository.createFile(newVideo) + }))) + .then(() => logger.info(`Created ${torrent.provider} entry for [${torrent.infoHash}] ${torrent.title}`)); + } + + public async createSkipTorrentEntry(torrent: Torrent) { + return repository.createSkipTorrent(torrent); + } + + public async getStoredTorrentEntry(torrent: Torrent) { + return repository.getSkipTorrent(torrent.infoHash) + .catch(() => repository.getTorrent(torrent)) + .catch(() => undefined); + } + + public async checkAndUpdateTorrent(torrent: ParsedTorrent): Promise { + const query: TorrentAttributes = { + infoHash: torrent.infoHash, + provider: torrent.provider, + } + + const existingTorrent = await repository.getTorrent(query).catch(() => undefined); + + if (!existingTorrent) { + return false; + } + + if (existingTorrent.provider === 'RARBG') { + return true; + } + if (existingTorrent.provider === 'KickassTorrents' && torrent.provider) { + existingTorrent.provider = torrent.provider; + existingTorrent.torrentId = torrent.torrentId; + } + + if (!existingTorrent.languages && torrent.languages && existingTorrent.provider !== 'RARBG') { + existingTorrent.languages = torrent.languages; + await existingTorrent.save(); + logger.debug(`Updated [${existingTorrent.infoHash}] ${existingTorrent.title} language to ${torrent.languages}`); + } + return this.createTorrentContents(existingTorrent) + .then(() => this.updateTorrentSeeders(existingTorrent)); + } + + public async createTorrentContents(torrent: Torrent) { + if (torrent.opened) { + return; + } + + const storedVideos: File[] = await repository.getFiles(torrent.infoHash).catch(() => []); + if (!storedVideos || !storedVideos.length) { + return; + } + const notOpenedVideo = storedVideos.length === 1 && !Number.isInteger(storedVideos[0].fileIndex); + const imdbId: string | undefined = PromiseHelpers.mostCommonValue(storedVideos.map(stored => stored.imdbId)); + const kitsuId: number | undefined = PromiseHelpers.mostCommonValue(storedVideos.map(stored => stored.kitsuId)); + + const fileCollection: TorrentFileCollection = await torrentFileService.parseTorrentFiles(torrent) + .then(torrentContents => notOpenedVideo ? torrentContents : {...torrentContents, videos: storedVideos}) + .then(torrentContents => torrentSubtitleService.assignSubtitles(torrentContents)) + .then(torrentContents => this.assignMetaIds(torrentContents, imdbId, kitsuId)) + .catch(error => { + logger.warn(`Failed getting contents for [${torrent.infoHash}] ${torrent.title}`, error.message); + return {}; + }); + + this.assignMetaIds(fileCollection, imdbId, kitsuId); + + if (!fileCollection.contents || !fileCollection.contents.length) { + return; + } + + if (notOpenedVideo && fileCollection.videos.length === 1) { + // if both have a single video and stored one was not opened, update stored one to true metadata and use that + storedVideos[0].fileIndex = fileCollection.videos[0].fileIndex; + storedVideos[0].title = fileCollection.videos[0].title; + storedVideos[0].size = fileCollection.videos[0].size; + storedVideos[0].subtitles = fileCollection.videos[0].subtitles.map(subtitle => Subtitle.build(subtitle)); + fileCollection.videos[0] = storedVideos[0]; + } + // no videos available or more than one new videos were in the torrent + const shouldDeleteOld = notOpenedVideo && fileCollection.videos.every(video => !video.id); + + const newTorrent: Torrent = Torrent.build({ + ...torrent, + contents: fileCollection.contents, + subtitles: fileCollection.subtitles + }); + + return repository.createTorrent(newTorrent) + .then(() => { + if (shouldDeleteOld) { + logger.debug(`Deleting old video for [${torrent.infoHash}] ${torrent.title}`) + return storedVideos[0].destroy(); + } + return Promise.resolve(); + }) + .then(() => PromiseHelpers.sequence(fileCollection.videos.map(video => () => { + const newVideo = File.build(video); + return repository.createFile(newVideo) + }))) + .then(() => logger.info(`Created contents for ${torrent.provider} [${torrent.infoHash}] ${torrent.title}`)) + .catch(error => logger.error(`Failed saving contents for [${torrent.infoHash}] ${torrent.title}`, error)); + } + + public async updateTorrentSeeders(torrent: TorrentAttributes) { + if (!(torrent.infoHash || (torrent.provider && torrent.torrentId)) || !Number.isInteger(torrent.seeders)) { + return torrent; + } + + return repository.setTorrentSeeders(torrent, torrent.seeders) + .catch(error => { + logger.warn('Failed updating seeders:', error); + return undefined; + }); + } + + private assignMetaIds(fileCollection: TorrentFileCollection, imdbId: string, kitsuId: number): TorrentFileCollection { + if (fileCollection.videos && fileCollection.videos.length) { + fileCollection.videos.forEach(video => { + video.imdbId = imdbId; + video.kitsuId = kitsuId; + }); + } + + return fileCollection; + } + + private async overwriteExistingFiles(torrent: ParsedTorrent, torrentContents: TorrentFileCollection) { + const videos = torrentContents && torrentContents.videos; + if (videos && videos.length) { + const existingFiles = await repository.getFiles(torrent.infoHash) + .then((existing) => existing + .reduce((map, next) => { + const fileIndex = next.fileIndex !== undefined ? next.fileIndex : null; + map[fileIndex] = (map[fileIndex] || []).concat(next); + return map; + }, {})) + .catch(() => undefined); + if (existingFiles && Object.keys(existingFiles).length) { + const overwrittenVideos = videos + .map(file => { + const mapping = videos.length === 1 && Object.keys(existingFiles).length === 1 + ? Object.values(existingFiles)[0] + : existingFiles[file.fileIndex !== undefined ? file.fileIndex : null]; + if (mapping) { + const originalFile = mapping.shift(); + return {id: originalFile.id, ...file}; + } + return file; + }); + return {...torrentContents, videos: overwrittenVideos}; + } + return torrentContents; + } + return Promise.reject(`No video files found for: ${torrent.title}`); + } +} + +export const torrentEntriesService = new TorrentEntriesService(); \ No newline at end of file diff --git a/src/node/consumer/src/lib/services/torrent_file_service.ts b/src/node/consumer/src/lib/services/torrent_file_service.ts index 29aa558..4d00026 100644 --- a/src/node/consumer/src/lib/services/torrent_file_service.ts +++ b/src/node/consumer/src/lib/services/torrent_file_service.ts @@ -1,30 +1,32 @@ import Bottleneck from 'bottleneck'; import moment from 'moment'; -import { parse } from 'parse-torrent-title'; +import {parse} from 'parse-torrent-title'; import {PromiseHelpers} from '../helpers/promises_helpers'; -import { TorrentType } from '../enums/torrent_types'; -import {TorrentInfo} from "../interfaces/torrent_info"; -import { configurationService } from './configuration_service'; -import { extensionService } from './extension_service'; -import { metadataService } from './metadata_service'; -import { parsingService } from './parsing_service'; -import { torrentDownloadService } from "./torrent_download_service"; +import {TorrentType} from '../enums/torrent_types'; +import {configurationService} from './configuration_service'; +import {extensionService} from './extension_service'; +import {metadataService} from './metadata_service'; +import {torrentDownloadService} from "./torrent_download_service"; import {logger} from "./logging_service"; import {MetadataResponse} from "../interfaces/metadata_response"; -import {ParseTorrentTitleResult} from "../interfaces/parse_torrent_title_result"; import {MetaDataQuery} from "../interfaces/metadata_query"; -import {ParsableTorrentFile} from "../interfaces/parsable_torrent_file"; import {CommonVideoMetadata} from "../interfaces/common_video_metadata"; import {TorrentFileCollection} from "../interfaces/torrent_file_collection"; +import {ParsedTorrent} from "../interfaces/parsed_torrent"; +import {FileAttributes} from "../../repository/interfaces/file_attributes"; +import {ContentAttributes} from "../../repository/interfaces/content_attributes"; class TorrentFileService { private readonly MIN_SIZE: number = 5 * 1024 * 1024; // 5 MB + + private readonly MULTIPLE_FILES_SIZE = 4 * 1024 * 1024 * 1024; // 4 GB + private readonly imdb_limiter: Bottleneck = new Bottleneck({ maxConcurrent: configurationService.metadataConfig.IMDB_CONCURRENT, minTime: configurationService.metadataConfig.IMDB_INTERVAL_MS }); - public async parseTorrentFiles(torrent: TorrentInfo) { + public async parseTorrentFiles(torrent: ParsedTorrent): Promise { const parsedTorrentName = parse(torrent.title); const query: MetaDataQuery = { id: torrent.kitsuId || torrent.imdbId, @@ -47,7 +49,32 @@ class TorrentFileService { return this.parseSeriesFiles(torrent, metadata) } - private async parseMovieFiles(torrent: TorrentInfo, metadata: MetadataResponse): Promise { + public isPackTorrent(torrent: ParsedTorrent): boolean { + if (torrent.isPack) { + return true; + } + const parsedInfo = parse(torrent.title); + if (torrent.type === TorrentType.Movie) { + return parsedInfo.complete || typeof parsedInfo.year === 'string' || /movies/i.test(torrent.title); + } + const hasMultipleEpisodes = parsedInfo.complete || + torrent.size > this.MULTIPLE_FILES_SIZE || + (parsedInfo.seasons && parsedInfo.seasons.length > 1) || + (parsedInfo.episodes && parsedInfo.episodes.length > 1) || + (parsedInfo.seasons && !parsedInfo.episodes); + const hasSingleEpisode = Number.isInteger(parsedInfo.episode) || (!parsedInfo.episodes && parsedInfo.date); + return hasMultipleEpisodes && !hasSingleEpisode; + } + + private parseSeriesVideos(torrent: ParsedTorrent): FileAttributes[] { + const parsedTorrentName = parse(torrent.title); + const hasMovies = parsedTorrentName.complete || !!torrent.title.match(/movies?(?:\W|$)/i); + const parsedVideos = torrent.fileCollection.videos.map(video => this.parseSeriesVideo(video)); + + return parsedVideos.map(video => ({ ...video, isMovie: this.isMovieVideo(torrent, video, parsedVideos, hasMovies) })); + } + + private async parseMovieFiles(torrent: ParsedTorrent, metadata: MetadataResponse): Promise { const {contents, videos, subtitles} = await this.getMoviesTorrentContent(torrent); const filteredVideos = videos .filter(video => video.size > this.MIN_SIZE) @@ -66,7 +93,7 @@ class TorrentFileService { const parsedVideos = await PromiseHelpers.sequence(filteredVideos.map(video => () => this.isFeaturette(video) ? Promise.resolve(video) - : this.findMovieImdbId(video.name).then(imdbId => ({...video, imdbId})))) + : this.findMovieImdbId(video.title).then(imdbId => ({...video, imdbId})))) .then(videos => videos.map(video => ({ infoHash: torrent.infoHash, fileIndex: video.fileIndex, @@ -77,53 +104,58 @@ class TorrentFileService { return {contents, videos: parsedVideos, subtitles}; } - private async parseSeriesFiles(torrent: TorrentInfo, metadata: MetadataResponse) { - const {contents, videos, subtitles} = await this.getSeriesTorrentContent(torrent); - const parsedVideos = await Promise.resolve(videos) + private async parseSeriesFiles(torrent: ParsedTorrent, metadata: MetadataResponse): TorrentFileCollection { + const fileCollection: TorrentFileCollection = await this.getSeriesTorrentContent(torrent); + const parsedVideos: FileAttributes[] = await Promise.resolve(fileCollection.videos) .then(videos => videos.filter(video => videos.length === 1 || video.size > this.MIN_SIZE)) - .then(videos => parsingService.parseSeriesVideos(torrent, videos)) - .then(videos => this.decomposeEpisodes(torrent, videos, metadata)) - .then(videos => this.assignKitsuOrImdbEpisodes(torrent, videos, metadata)) + .then(videos => this.parseSeriesVideos(torrent)) + .then(videos => this.decomposeEpisodes(torrent, metadata)) + .then(videos => this.assignKitsuOrImdbEpisodes(torrent, fileCollection.videos, metadata)) .then(videos => Promise.all(videos.map(video => video.isMovie ? this.mapSeriesMovie(video, torrent) : this.mapSeriesEpisode(video, torrent, videos)))) .then(videos => videos - .map((video: ParsableTorrentFile) => this.isFeaturette(video) ? this.clearInfoFields(video) : video)) - return {contents, videos: parsedVideos, subtitles}; + .map((video: ParsedTorrent) => this.isFeaturette(video) ? this.clearInfoFields(video) : video)) + return {...torrent.fileCollection, videos: parsedVideos}; } - private async getMoviesTorrentContent(torrent: TorrentInfo) { + private async getMoviesTorrentContent(torrent: ParsedTorrent) { const files = await torrentDownloadService.getTorrentFiles(torrent) .catch(error => { - if (!parsingService.isPackTorrent(torrent)) { - return { videos: [{name: torrent.title, path: torrent.title, size: torrent.size, fileIndex: null}], contents:[], subtitles: []} + if (!this.isPackTorrent(torrent)) { + const entries = [{name: torrent.title, path: torrent.title, size: torrent.size, fileIndex: null}]; + return { videos: entries, contents:[], subtitles: [], files: entries} } return Promise.reject(error); }); if (files.contents && files.contents.length && !files.videos.length && this.isDiskTorrent(files.contents)) { - files.videos = [{name: torrent.title, path: torrent.title, size: torrent.size}]; + files.videos = [{name: torrent.title, path: torrent.title, size: torrent.size, fileIndex: null}]; } return files; } + + private getDefaultFileEntries(torrent: ParsedTorrent): FileAttributes[] { + return [{title: torrent.title, path: torrent.title, size: torrent.size, fileIndex: null}]; + } - private async getSeriesTorrentContent(torrent: TorrentInfo) { + private async getSeriesTorrentContent(torrent: ParsedTorrent): Promise { return torrentDownloadService.getTorrentFiles(torrent) .catch(error => { - if (!parsingService.isPackTorrent(torrent)) { - return { videos: [{ name: torrent.title, path: torrent.title, size: torrent.size }], subtitles: [], contents: [] } + if (!this.isPackTorrent(torrent)) { + return { videos: this.getDefaultFileEntries(torrent), subtitles: [], contents: [] } } return Promise.reject(error); }); } - private async mapSeriesEpisode(file: ParsableTorrentFile, torrent: TorrentInfo, files: ParsableTorrentFile[]) : Promise { + private async mapSeriesEpisode(file: FileAttributes, torrent: ParsedTorrent, files: ParsedTorrent[]) : Promise { if (!file.episodes && !file.episodes) { if (files.length === 1 || files.some(f => f.episodes || f.episodes) || parse(torrent.title).seasons) { return Promise.resolve({ infoHash: torrent.infoHash, fileIndex: file.fileIndex, - title: file.path || file.name, + title: file.path || file.title, size: file.size, imdbId: torrent.imdbId || file.imdbId, }); @@ -134,17 +166,20 @@ class TorrentFileService { return Promise.resolve(episodeIndexes.map((index) => ({ infoHash: torrent.infoHash, fileIndex: file.fileIndex, - title: file.path || file.name, + title: file.path || file.title, size: file.size, - imdbId: file.imdbId || torrent.imdbId, + imdbId: file.imdbId.toString() || torrent.imdbId.toString(), + imdbSeason: file.season, season: file.season, + imdbEpisode: file.episodes && file.episodes[index], episode: file.episodes && file.episodes[index], + kitsuEpisode: file.episodes && file.episodes[index], episodes: file.episodes, - kitsuId: file.kitsuId || torrent.kitsuId, + kitsuId: parseInt(file.kitsuId.toString() || torrent.kitsuId.toString()), }))) } - private async mapSeriesMovie(file: ParsableTorrentFile, torrent: TorrentInfo): Promise { + private async mapSeriesMovie(torrent: ParsedTorrent, file: FileAttributes): Promise { const kitsuId= torrent.type === TorrentType.Anime ? await this.findMovieKitsuId(file) .then(result => { if (result instanceof Error) { @@ -168,10 +203,10 @@ class TorrentFileService { return [{ infoHash: torrent.infoHash, fileIndex: file.fileIndex, - title: file.path || file.name, + title: file.path || file.title, size: file.size, imdbId: imdbId, - kitsuId: kitsuId, + kitsuId: parseInt(kitsuId), episodes: undefined, imdbSeason: undefined, imdbEpisode: undefined, @@ -185,16 +220,17 @@ class TorrentFileService { return [{ infoHash: torrent.infoHash, fileIndex: file.fileIndex, - title: file.path || file.name, + title: file.path || file.title, size: file.size, - imdbId: metadata.imdbId || imdbId, - kitsuId: metadata.kitsuId || kitsuId, - season: episodeVideo && metadata.imdbId ? episodeVideo.season : undefined, - episode: episodeVideo && metadata.imdbId | metadata.kitsuId ? episodeVideo.episode || episodeVideo.episode : undefined, + imdbId: metadata.imdbId.toString() || imdbId, + kitsuId: parseInt(metadata.kitsuId.toString() || kitsuId), + imdbSeason: episodeVideo && metadata.imdbId ? episodeVideo.season : undefined, + imdbEpisode: episodeVideo && metadata.imdbId | metadata.kitsuId ? episodeVideo.episode || episodeVideo.episode : undefined, + kitsuEpisode: episodeVideo && metadata.imdbId | metadata.kitsuId ? episodeVideo.episode || episodeVideo.episode : undefined, }]; } - private async decomposeEpisodes(torrent: TorrentInfo, files: ParsableTorrentFile[], metadata: MetadataResponse = { episodeCount: [] }) { + private async decomposeEpisodes(torrent: ParsedTorrent, files: FileAttributes[], metadata: MetadataResponse = { episodeCount: [] }) { if (files.every(file => !file.episodes && !file.date)) { return files; } @@ -239,7 +275,7 @@ class TorrentFileService { return files; } - private preprocessEpisodes(files: ParsableTorrentFile[]) { + private preprocessEpisodes(files: FileAttributes[]) { // reverse special episode naming when they named with 0 episode, ie. S02E00 files .filter(file => Number.isInteger(file.season) && file.episode === 0) @@ -250,7 +286,7 @@ class TorrentFileService { }) } - private isConcatSeasonAndEpisodeFiles(files: ParsableTorrentFile[], sortedEpisodes: number[], metadata: MetadataResponse) { + private isConcatSeasonAndEpisodeFiles(files: FileAttributes[], sortedEpisodes: number[], metadata: MetadataResponse) { if (metadata.kitsuId !== undefined) { // anime does not use this naming scheme in 99% of cases; return false; @@ -277,11 +313,11 @@ class TorrentFileService { || concatAboveTotalEpisodeCount.length >= thresholdAbove; } - private isDateEpisodeFiles(files: ParsableTorrentFile[], metadata: MetadataResponse) { + private isDateEpisodeFiles(files: FileAttributes[], metadata: MetadataResponse) { return files.every(file => (!file.season || !metadata.episodeCount[file.season - 1]) && file.date); } - private isAbsoluteEpisodeFiles(torrent: TorrentInfo, files: ParsableTorrentFile[], metadata: MetadataResponse) { + private isAbsoluteEpisodeFiles(torrent: ParsedTorrent, files: FileAttributes[], metadata: MetadataResponse) { const threshold = Math.ceil(files.length / 5); const isAnime = torrent.type === TorrentType.Anime && torrent.kitsuId; const nonMovieEpisodes = files @@ -294,18 +330,18 @@ class TorrentFileService { || absoluteEpisodes.length >= threshold; } - private isNewEpisodeNotInMetadata(torrent: TorrentInfo, file: ParsableTorrentFile, metadata: MetadataResponse) { + private isNewEpisodeNotInMetadata(torrent: ParsedTorrent, video: FileAttributes, metadata: MetadataResponse) { // new episode might not yet been indexed by cinemeta. // detect this if episode number is larger than the last episode or season is larger than the last one // only for non anime metas const isAnime = torrent.type === TorrentType.Anime && torrent.kitsuId; - return !isAnime && !file.isMovie && file.episodes && file.season !== 1 + return !isAnime && !video.isMovie && video.episodes && video.season !== 1 && /continuing|current/i.test(metadata.status) - && file.season >= metadata.episodeCount.length - && file.episodes.every(ep => ep > (metadata.episodeCount[file.season - 1] || 0)); + && video.season >= metadata.episodeCount.length + && video.episodes.every(ep => ep > (metadata.episodeCount[video.season - 1] || 0)); } - private decomposeConcatSeasonAndEpisodeFiles(files: ParsableTorrentFile[], metadata: MetadataResponse) { + private decomposeConcatSeasonAndEpisodeFiles(files: FileAttributes[], metadata: MetadataResponse) { files .filter(file => file.episodes && file.season !== 0 && file.episodes.every(ep => ep > 100)) .filter(file => metadata.episodeCount[(file.season || this.div100(file.episodes[0])) - 1] < 100) @@ -317,16 +353,16 @@ class TorrentFileService { } - private decomposeAbsoluteEpisodeFiles(torrent: TorrentInfo, files: ParsableTorrentFile[], metadata: MetadataResponse) { + private decomposeAbsoluteEpisodeFiles(torrent: ParsedTorrent, videos: FileAttributes[], metadata: MetadataResponse) { if (metadata.episodeCount.length === 0) { - files + videos .filter(file => !Number.isInteger(file.season) && file.episodes && !file.isMovie) .forEach(file => { file.season = 1; }); return; } - files + videos .filter(file => file.episodes && !file.isMovie && file.season !== 0) .filter(file => !this.isNewEpisodeNotInMetadata(torrent, file, metadata)) .filter(file => !file.season || (metadata.episodeCount[file.season - 1] || 0) < file.episodes[0]) @@ -341,7 +377,7 @@ class TorrentFileService { }); } - private decomposeDateEpisodeFiles(files: ParsableTorrentFile[], metadata: MetadataResponse) { + private decomposeDateEpisodeFiles(files: FileAttributes[], metadata: MetadataResponse) { if (!metadata || !metadata.videos || !metadata.videos.length) { return; } @@ -375,7 +411,7 @@ class TorrentFileService { } } - private assignKitsuOrImdbEpisodes(torrent: TorrentInfo, files: ParsableTorrentFile[], metadata: MetadataResponse) { + private assignKitsuOrImdbEpisodes(torrent: ParsedTorrent, files: FileAttributes[], metadata: MetadataResponse) { if (!metadata || !metadata.videos || !metadata.videos.length) { if (torrent.type === TorrentType.Anime) { // assign episodes as kitsu episodes for anime when no metadata available for imdb mapping @@ -387,7 +423,7 @@ class TorrentFileService { }) if (metadata.type === TorrentType.Movie && files.every(file => !file.imdbId)) { // sometimes a movie has episode naming, thus not recognized as a movie and imdbId not assigned - files.forEach(file => file.imdbId = metadata.imdbId); + files.forEach(file => file.imdbId = metadata.imdbId.toString()); } } return files; @@ -402,14 +438,14 @@ class TorrentFileService { }, {}); if (metadata.videos.some(video => Number.isInteger(video.season)) || !metadata.imdbId) { - files.filter((file => Number.isInteger(file.season) && file.episodes)) + files.filter((file => Number.isInteger(torrent.season) && torrent.episodes)) .map(file => { const seasonMapping = seriesMapping[file.season]; const episodeMapping = seasonMapping && seasonMapping[file.episodes[0]]; - if (episodeMapping && Number.isInteger(episodeMapping.imdbSeason)) { - file.imdbId = metadata.imdbId; - file.season = episodeMapping.imdbSeason; - file.episodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].imdbEpisode); + if (episodeMapping && Number.isInteger(episodeMapping.season)) { + file.imdbId = metadata.imdbId.toString(); + file.season = episodeMapping.season; + file.episodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].episode); } else { // no imdb mapping available for episode file.season = undefined; @@ -423,7 +459,7 @@ class TorrentFileService { .forEach(file => { if (seriesMapping[file.season]) { const seasonMapping = seriesMapping[file.season]; - file.imdbId = metadata.imdbId; + file.imdbId = metadata.imdbId.toString(); file.kitsuId = seasonMapping[file.episodes[0]] && seasonMapping[file.episodes[0]].kitsuId; file.episodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].kitsuEpisode); } else if (seriesMapping[file.season - 1]) { @@ -434,12 +470,12 @@ class TorrentFileService { const differentTitlesCount = new Set(episodes.map(ep => ep.id)).size const skippedCount = episodes.filter(ep => ep.id === firstKitsuId).length; const seasonEpisodes = files - .filter(otherFile => otherFile.season === file.season) + .filter((otherFile: FileAttributes) => otherFile.season === file.season) .reduce((a, b) => a.episodes.concat(b.episodes), []); const isAbsoluteOrder = seasonEpisodes.episodes.every(ep => ep > skippedCount && ep <= episodes.length) const isNormalOrder = seasonEpisodes.episodes.every(ep => ep + skippedCount <= episodes.length) if (differentTitlesCount >= 1 && (isAbsoluteOrder || isNormalOrder)) { - file.imdbId = metadata.imdbId; + file.imdbId = metadata.imdbId.toString(); file.season = file.season - 1; file.episodes = file.episodes.map(ep => isAbsoluteOrder ? ep : ep + skippedCount); file.kitsuId = seasonMapping[file.episodes[0]].kitsuId; @@ -448,7 +484,7 @@ class TorrentFileService { } else if (Object.values(seriesMapping).length === 1 && seriesMapping[1]) { // sometimes series might be named with sequel season but it's not a season on imdb and a new title const seasonMapping = seriesMapping[1]; - file.imdbId = metadata.imdbId; + file.imdbId = metadata.imdbId.toString(); file.season = 1; file.kitsuId = seasonMapping[file.episodes[0]].kitsuId; file.episodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].kitsuEpisode); @@ -458,7 +494,7 @@ class TorrentFileService { return files; } - private needsCinemetaMetadataForAnime(files: ParsableTorrentFile[], metadata: MetadataResponse) { + private needsCinemetaMetadataForAnime(files: FileAttributes[], metadata: MetadataResponse) { if (!metadata || !metadata.imdbId || !metadata.videos || !metadata.videos.length) { return false; } @@ -500,7 +536,7 @@ class TorrentFileService { }) } - private findMovieImdbId(title: ParseTorrentTitleResult | string) { + private findMovieImdbId(title: FileAttributes | string) { const parsedTitle = typeof title === 'string' ? parse(title) : title; logger.debug(`Finding movie imdbId for ${title}`); return this.imdb_limiter.schedule(async () => { @@ -517,7 +553,7 @@ class TorrentFileService { }); } - private async findMovieKitsuId(title: ParseTorrentTitleResult | string) { + private async findMovieKitsuId(title: FileAttributes | string) { const parsedTitle = typeof title === 'string' ? parse(title) : title; const kitsuQuery = { title: parsedTitle.title, @@ -532,26 +568,97 @@ class TorrentFileService { } } - private isDiskTorrent(contents: ParsableTorrentFile[]) { + private isDiskTorrent(contents: ContentAttributes[]) { return contents.some(content => extensionService.isDisk(content.path)); } - private isSingleMovie(videos: ParsableTorrentFile[]) { + private isSingleMovie(videos: FileAttributes[]) { return videos.length === 1 || (videos.length === 2 && videos.find(v => /\b(?:part|disc|cd)[ ._-]?0?1\b|^0?1\.\w{2,4}$/i.test(v.path)) && videos.find(v => /\b(?:part|disc|cd)[ ._-]?0?2\b|^0?2\.\w{2,4}$/i.test(v.path))); } - private isFeaturette(video: ParsableTorrentFile) { + private isFeaturette(video: FileAttributes) { return /featurettes?\/|extras-grym/i.test(video.path); } - private clearInfoFields(video: ParsableTorrentFile) { + private parseSeriesVideo(video: FileAttributes): FileAttributes { + const videoInfo = parse(video.title); + // the episode may be in a folder containing season number + if (!Number.isInteger(videoInfo.season) && video.path.includes('/')) { + const folders = video.path.split('/'); + const pathInfo = parse(folders[folders.length - 2]); + videoInfo.season = pathInfo.season; + } + if (!Number.isInteger(videoInfo.season) && video.season) { + videoInfo.season = video.season; + } + if (!Number.isInteger(videoInfo.season) && videoInfo.seasons && videoInfo.seasons.length > 1) { + // in case single file was interpreted as having multiple seasons + videoInfo.season = videoInfo.seasons[0]; + } + if (!Number.isInteger(videoInfo.season) && video.path.includes('/') && video.seasons + && video.seasons.length > 1) { + // russian season are usually named with 'series name-2` i.e. Улицы разбитых фонарей-6/22. Одиночный выстрел.mkv + const folderPathSeasonMatch = video.path.match(/[\u0400-\u04ff]-(\d{1,2})(?=.*\/)/); + videoInfo.season = folderPathSeasonMatch && parseInt(folderPathSeasonMatch[1], 10) || undefined; + } + // sometimes video file does not have correct date format as in torrent title + if (!videoInfo.episodes && !videoInfo.date && video.date) { + videoInfo.date = video.date; + } + // limit number of episodes in case of incorrect parsing + if (videoInfo.episodes && videoInfo.episodes.length > 20) { + videoInfo.episodes = [videoInfo.episodes[0]]; + videoInfo.episode = videoInfo.episodes[0]; + } + // force episode to any found number if it was not parsed + if (!videoInfo.episodes && !videoInfo.date) { + const epMatcher = videoInfo.title.match( + /(? 3 + && otherVideos.filter(other => other.title === video.title && other.year === video.year).length < 3; + } + + private clearInfoFields(video: FileAttributes) { video.imdbId = undefined; - video.season = undefined; - video.episode = undefined; + video.imdbSeason = undefined; + video.imdbEpisode = undefined; video.kitsuId = undefined; + video.kitsuEpisode = undefined; return video; } diff --git a/src/node/consumer/src/lib/services/torrent_processing_service.ts b/src/node/consumer/src/lib/services/torrent_processing_service.ts index c74a951..9c59c39 100644 --- a/src/node/consumer/src/lib/services/torrent_processing_service.ts +++ b/src/node/consumer/src/lib/services/torrent_processing_service.ts @@ -1,23 +1,23 @@ -import {TorrentInfo} from "../interfaces/torrent_info"; import {TorrentType} from "../enums/torrent_types"; import {logger} from "./logging_service"; -import {checkAndUpdateTorrent, createTorrentEntry} from "../torrentEntries.js"; import {trackerService} from "./tracker_service"; +import {torrentEntriesService} from "./torrent_entries_service"; import {IngestedTorrentAttributes} from "../../repository/interfaces/ingested_torrent_attributes"; +import {ParsedTorrent} from "../interfaces/parsed_torrent"; class TorrentProcessingService { public async processTorrentRecord(torrent: IngestedTorrentAttributes): Promise { const { category } = torrent; const type = category === 'tv' ? TorrentType.Series : TorrentType.Movie; - const torrentInfo: TorrentInfo = await this.parseTorrent(torrent, type); + const torrentInfo: ParsedTorrent = await this.parseTorrent(torrent, type); logger.info(`Processing torrent ${torrentInfo.title} with infoHash ${torrentInfo.infoHash}`); - if (await checkAndUpdateTorrent(torrentInfo)) { + if (await torrentEntriesService.checkAndUpdateTorrent(torrentInfo)) { return; } - return createTorrentEntry(torrentInfo); + return torrentEntriesService.createTorrentEntry(torrentInfo); } private async assignTorrentTrackers(): Promise { @@ -25,14 +25,14 @@ class TorrentProcessingService { return trackers.join(','); } - private async parseTorrent(torrent: IngestedTorrentAttributes, category: string): Promise { + private async parseTorrent(torrent: IngestedTorrentAttributes, category: string): Promise { const infoHash = torrent.info_hash?.trim().toLowerCase() return { title: torrent.name, torrentId: `${torrent.name}_${infoHash}`, infoHash: infoHash, seeders: 100, - size: torrent.size, + size: parseInt(torrent.size), uploadDate: torrent.createdAt, imdbId: this.parseImdbId(torrent), type: category, diff --git a/src/node/consumer/src/lib/services/torrent_subtitle_service.ts b/src/node/consumer/src/lib/services/torrent_subtitle_service.ts index 9cbe739..e0c52a1 100644 --- a/src/node/consumer/src/lib/services/torrent_subtitle_service.ts +++ b/src/node/consumer/src/lib/services/torrent_subtitle_service.ts @@ -1,26 +1,28 @@ import { parse } from 'parse-torrent-title'; +import {TorrentFileCollection} from "../interfaces/torrent_file_collection"; +import {FileAttributes} from "../../repository/interfaces/file_attributes"; class TorrentSubtitleService { - public assignSubtitles(contents: any, videos: any[], subtitles: any[]) { - if (videos && videos.length && subtitles && subtitles.length) { - if (videos.length === 1) { - videos[0].subtitles = subtitles; - return { contents, videos, subtitles: [] }; + public assignSubtitles(fileCollection: TorrentFileCollection) : TorrentFileCollection { + if (fileCollection.videos && fileCollection.videos.length && fileCollection.subtitles && fileCollection.subtitles.length) { + if (fileCollection.videos.length === 1) { + fileCollection.videos[0].subtitles = fileCollection.subtitles; + return { ...fileCollection, subtitles: [] }; } - const parsedVideos = videos.map(video => this.parseVideo(video)); - const assignedSubs = subtitles.map(subtitle => ({ subtitle, videos: this.mostProbableSubtitleVideos(subtitle, parsedVideos) })); + const parsedVideos = fileCollection.videos.map(video => this.parseVideo(video)); + const assignedSubs = fileCollection.subtitles.map(subtitle => ({ subtitle, videos: this.mostProbableSubtitleVideos(subtitle, parsedVideos) })); const unassignedSubs = assignedSubs.filter(assignedSub => !assignedSub.videos).map(assignedSub => assignedSub.subtitle); assignedSubs .filter(assignedSub => assignedSub.videos) .forEach(assignedSub => assignedSub.videos.forEach(video => video.subtitles = (video.subtitles || []).concat(assignedSub.subtitle))); - return { contents, videos, subtitles: unassignedSubs }; + return { ...fileCollection, subtitles: unassignedSubs }; } - return { contents, videos, subtitles }; + return fileCollection; } - private parseVideo(video: any) { + private parseVideo(video: FileAttributes) { const fileName = video.title.split('/').pop().replace(/\.(\w{2,4})$/, ''); const folderName = video.title.replace(/\/?[^/]+$/, ''); return { diff --git a/src/node/consumer/src/lib/torrentEntries.js b/src/node/consumer/src/lib/torrentEntries.js deleted file mode 100644 index 0108e61..0000000 --- a/src/node/consumer/src/lib/torrentEntries.js +++ /dev/null @@ -1,183 +0,0 @@ -import { parse } from 'parse-torrent-title'; -import { repository } from '../repository/database_repository'; -import { TorrentType } from './enums/torrent_types'; -import { PromiseHelpers } from './helpers/promises_helpers'; -import { logger } from './services/logging_service'; -import { metadataService } from './services/metadata_service'; -import { parsingService } from './services/parsing_service'; -import { torrentFileService } from './services/torrent_file_service'; -import { torrentSubtitleService } from './services/torrent_subtitle_service'; - -export async function createTorrentEntry(torrent, overwrite = false) { - const titleInfo = parse(torrent.title); - - if (!torrent.imdbId && torrent.type !== TorrentType.Anime) { - const imdbQuery = { - title: titleInfo.title, - year: titleInfo.year, - type: torrent.type - }; - torrent.imdbId = await metadataService.getImdbId(imdbQuery) - .catch(() => undefined); - } - if (torrent.imdbId && torrent.imdbId.length < 9) { - // pad zeros to imdbId if missing - torrent.imdbId = 'tt' + torrent.imdbId.replace('tt', '').padStart(7, '0'); - } - if (torrent.imdbId && torrent.imdbId.length > 9 && torrent.imdbId.startsWith('tt0')) { - // sanitize imdbId from redundant zeros - torrent.imdbId = torrent.imdbId.replace(/tt0+([0-9]{7,})$/, 'tt$1'); - } - if (!torrent.kitsuId && torrent.type === TorrentType.Anime) { - const kitsuQuery = { - title: titleInfo.title, - year: titleInfo.year, - season: titleInfo.season, - }; - torrent.kitsuId = await metadataService.getKitsuId(kitsuQuery) - .catch(() => undefined); - } - - if (!torrent.imdbId && !torrent.kitsuId && !parsingService.isPackTorrent(torrent)) { - logger.warn(`imdbId or kitsuId not found: ${torrent.provider} ${torrent.title}`); - return; - } - - const { contents, videos, subtitles } = await torrentFileService.parseTorrentFiles(torrent) - .then(torrentContents => overwrite ? overwriteExistingFiles(torrent, torrentContents) : torrentContents) - .then(torrentContents => torrentSubtitleService.assignSubtitles(torrentContents)) - .catch(error => { - logger.warn(`Failed getting files for ${torrent.title}`, error.message); - return {}; - }); - if (!videos || !videos.length) { - logger.warn(`no video files found for ${torrent.provider} [${torrent.infoHash}] ${torrent.title}`); - return; - } - - return repository.createTorrent({ ...torrent, contents, subtitles }) - .then(() => PromiseHelpers.sequence(videos.map(video => () => repository.createFile(video)))) - .then(() => logger.info(`Created ${torrent.provider} entry for [${torrent.infoHash}] ${torrent.title}`)); -} - -async function overwriteExistingFiles(torrent, torrentContents) { - const videos = torrentContents && torrentContents.videos; - if (videos && videos.length) { - const existingFiles = await repository.getFiles({ infoHash: videos[0].infoHash }) - .then((existing) => existing - .reduce((map, next) => { - const fileIndex = next.fileIndex !== undefined ? next.fileIndex : null; - map[fileIndex] = (map[fileIndex] || []).concat(next); - return map; - }, {})) - .catch(() => undefined); - if (existingFiles && Object.keys(existingFiles).length) { - const overwrittenVideos = videos - .map(file => { - const mapping = videos.length === 1 && Object.keys(existingFiles).length === 1 - ? Object.values(existingFiles)[0] - : existingFiles[file.fileIndex !== undefined ? file.fileIndex : null]; - if (mapping) { - const originalFile = mapping.shift(); - return { id: originalFile.id, ...file }; - } - return file; - }); - return { ...torrentContents, videos: overwrittenVideos }; - } - return torrentContents; - } - return Promise.reject(`No video files found for: ${torrent.title}`); -} - -export async function createSkipTorrentEntry(torrent) { - return repository.createSkipTorrent(torrent); -} - -export async function getStoredTorrentEntry(torrent) { - return repository.getSkipTorrent(torrent) - .catch(() => repository.getTorrent(torrent)) - .catch(() => undefined); -} - -export async function checkAndUpdateTorrent(torrent) { - const storedTorrent = torrent.dataValues - ? torrent - : await repository.getTorrent(torrent).catch(() => undefined); - if (!storedTorrent) { - return false; - } - if (storedTorrent.provider === 'RARBG') { - return true; - } - if (storedTorrent.provider === 'KickassTorrents' && torrent.provider) { - storedTorrent.provider = torrent.provider; - storedTorrent.torrentId = torrent.torrentId; - } - if (!storedTorrent.languages && torrent.languages && storedTorrent.provider !== 'RARBG') { - storedTorrent.languages = torrent.languages; - await storedTorrent.save(); - logger.debug(`Updated [${storedTorrent.infoHash}] ${storedTorrent.title} language to ${torrent.languages}`); - } - return createTorrentContents({ ...storedTorrent.get(), torrentLink: torrent.torrentLink }) - .then(() => updateTorrentSeeders(torrent)); -} - -export async function createTorrentContents(torrent) { - if (torrent.opened) { - return; - } - const storedVideos = await repository.getFiles(torrent).catch(() => []); - if (!storedVideos || !storedVideos.length) { - return; - } - const notOpenedVideo = storedVideos.length === 1 && !Number.isInteger(storedVideos[0].fileIndex); - const imdbId = PromiseHelpers.mostCommonValue(storedVideos.map(stored => stored.imdbId)); - const kitsuId = PromiseHelpers.mostCommonValue(storedVideos.map(stored => stored.kitsuId)); - - const { contents, videos, subtitles } = await torrentFileService.parseTorrentFiles({ ...torrent, imdbId, kitsuId }) - .then(torrentContents => notOpenedVideo ? torrentContents : { ...torrentContents, videos: storedVideos }) - .then(torrentContents => torrentSubtitleService.assignSubtitles(torrentContents)) - .catch(error => { - logger.warn(`Failed getting contents for [${torrent.infoHash}] ${torrent.title}`, error.message); - return {}; - }); - - if (!contents || !contents.length) { - return; - } - if (notOpenedVideo && videos.length === 1) { - // if both have a single video and stored one was not opened, update stored one to true metadata and use that - storedVideos[0].fileIndex = videos[0].fileIndex; - storedVideos[0].title = videos[0].title; - storedVideos[0].size = videos[0].size; - storedVideos[0].subtitles = videos[0].subtitles; - videos[0] = storedVideos[0]; - } - // no videos available or more than one new videos were in the torrent - const shouldDeleteOld = notOpenedVideo && videos.every(video => !video.id); - - return repository.createTorrent({ ...torrent, contents, subtitles }) - .then(() => { - if (shouldDeleteOld) { - logger.debug(`Deleting old video for [${torrent.infoHash}] ${torrent.title}`) - return storedVideos[0].destroy(); - } - return Promise.resolve(); - }) - .then(() => PromiseHelpers.sequence(videos.map(video => () => repository.createFile(video)))) - .then(() => logger.info(`Created contents for ${torrent.provider} [${torrent.infoHash}] ${torrent.title}`)) - .catch(error => logger.error(`Failed saving contents for [${torrent.infoHash}] ${torrent.title}`, error)); -} - -export async function updateTorrentSeeders(torrent) { - if (!(torrent.infoHash || (torrent.provider && torrent.torrentId)) || !Number.isInteger(torrent.seeders)) { - return torrent; - } - - return repository.setTorrentSeeders(torrent, torrent.seeders) - .catch(error => { - logger.warn('Failed updating seeders:', error); - return undefined; - }); -} diff --git a/src/node/consumer/src/repository/database_repository.ts b/src/node/consumer/src/repository/database_repository.ts index 35a11e1..51edac1 100644 --- a/src/node/consumer/src/repository/database_repository.ts +++ b/src/node/consumer/src/repository/database_repository.ts @@ -50,7 +50,7 @@ class DatabaseRepository { } } - public async getTorrent(torrent: Torrent): Promise { + public async getTorrent(torrent: TorrentAttributes): Promise { const where = torrent.infoHash ? { infoHash: torrent.infoHash } : { provider: torrent.provider, torrentId: torrent.torrentId }; @@ -69,25 +69,6 @@ class DatabaseRepository { return await File.findAll({ where }); } - public async getUnprocessedIngestedTorrents(): Promise { - return await IngestedTorrent.findAll({ - where: { - processed: false, - category: { - [Op.or]: ['tv', 'movies'] - } - }, - }); - } - - public async setIngestedTorrentsProcessed(ingestedTorrents: IngestedTorrent[]): Promise { - await PromiseHelpers.sequence(ingestedTorrents - .map(ingestedTorrent => async () => { - ingestedTorrent.processed = true; - await ingestedTorrent.save(); - })); - } - public async getTorrentsWithoutSize(): Promise { return await Torrent.findAll({ where: literal( @@ -148,8 +129,8 @@ class DatabaseRepository { ); } - public async deleteTorrent(torrent: TorrentAttributes): Promise { - return await Torrent.destroy({ where: { infoHash: torrent.infoHash } }); + public async deleteTorrent(infoHash: string): Promise { + return await Torrent.destroy({ where: { infoHash: infoHash } }); } public async createFile(file: File): Promise { @@ -171,16 +152,16 @@ class DatabaseRepository { } } - public async getFiles(torrent: Torrent): Promise { - return File.findAll({ where: { infoHash: torrent.infoHash } }); + public async getFiles(infoHash: string): Promise { + return File.findAll({ where: { infoHash: infoHash } }); } public async getFilesBasedOnTitle(titleQuery: string): Promise { return File.findAll({ where: { title: { [Op.regexp]: `${titleQuery}` } } }); } - public async deleteFile(file: File): Promise { - return File.destroy({ where: { id: file.id } }); + public async deleteFile(id: number): Promise { + return File.destroy({ where: { id: id } }); } public async createSubtitles(infoHash: string, subtitles: Subtitle[]): Promise[]> { @@ -209,8 +190,8 @@ class DatabaseRepository { } } - public async getSubtitles(torrent: Torrent): Promise { - return Subtitle.findAll({ where: { infoHash: torrent.infoHash } }); + public async getSubtitles(infoHash: string): Promise { + return Subtitle.findAll({ where: { infoHash: infoHash } }); } public async getUnassignedSubtitles(): Promise { @@ -224,14 +205,14 @@ class DatabaseRepository { } } - public async getContents(torrent: Torrent): Promise { - return Content.findAll({ where: { infoHash: torrent.infoHash } }); + public async getContents(infoHash: string): Promise { + return Content.findAll({ where: { infoHash: infoHash } }); } - public async getSkipTorrent(torrent: Torrent): Promise { - const result = await SkipTorrent.findByPk(torrent.infoHash); + public async getSkipTorrent(infoHash: string): Promise { + const result = await SkipTorrent.findByPk(infoHash); if (!result) { - throw new Error(`torrent not found: ${torrent.infoHash}`); + throw new Error(`torrent not found: ${infoHash}`); } return result.dataValues as SkipTorrent; } diff --git a/src/node/consumer/src/repository/interfaces/file_attributes.ts b/src/node/consumer/src/repository/interfaces/file_attributes.ts index 4cb68ea..7ae8469 100644 --- a/src/node/consumer/src/repository/interfaces/file_attributes.ts +++ b/src/node/consumer/src/repository/interfaces/file_attributes.ts @@ -1,18 +1,21 @@ import {Optional} from "sequelize"; import {SubtitleAttributes} from "./subtitle_attributes"; +import {ParseTorrentTitleResult} from "../../lib/interfaces/parse_torrent_title_result"; -export interface FileAttributes { +export interface FileAttributes extends ParseTorrentTitleResult { id?: number; - infoHash: string; - fileIndex: number; - title: string; - size: number; - imdbId: string; - imdbSeason: number; - imdbEpisode: number; - kitsuId: number; - kitsuEpisode: number; + infoHash?: string; + fileIndex?: number; + title?: string; + size?: number; + imdbId?: string; + imdbSeason?: number; + imdbEpisode?: number; + kitsuId?: number; + kitsuEpisode?: number; subtitles?: SubtitleAttributes[]; + path?: string; + isMovie?: boolean; } export interface FileCreationAttributes extends Optional { diff --git a/src/node/consumer/src/repository/interfaces/torrent_attributes.ts b/src/node/consumer/src/repository/interfaces/torrent_attributes.ts index 0fcef0b..60e4bb7 100644 --- a/src/node/consumer/src/repository/interfaces/torrent_attributes.ts +++ b/src/node/consumer/src/repository/interfaces/torrent_attributes.ts @@ -5,20 +5,20 @@ import {FileAttributes} from "./file_attributes"; export interface TorrentAttributes { infoHash: string; - provider: string; - torrentId: string; - title: string; - size: number; - type: string; - uploadDate: Date; - seeders: number; - trackers: string; - languages: string; - resolution: string; - reviewed: boolean; - opened: boolean; - contents: ContentAttributes[]; - files: FileAttributes[]; + provider?: string; + torrentId?: string; + title?: string; + size?: number; + type?: string; + uploadDate?: Date; + seeders?: number; + trackers?: string; + languages?: string; + resolution?: string; + reviewed?: boolean; + opened?: boolean; + contents?: ContentAttributes[]; + files?: FileAttributes[]; subtitles?: SubtitleAttributes[]; }