diff --git a/src/node/consumer/package-lock.json b/src/node/consumer/package-lock.json index 2898608..3a41714 100644 --- a/src/node/consumer/package-lock.json +++ b/src/node/consumer/package-lock.json @@ -31,6 +31,7 @@ }, "devDependencies": { "@types/amqplib": "^0.10.4", + "@types/jaro-winkler": "^0.2.3", "@types/magnet-uri": "^5.1.5", "@types/node": "^20.11.16", "@types/stremio-addon-sdk": "^1.6.10", @@ -583,6 +584,12 @@ "@types/ms": "*" } }, + "node_modules/@types/jaro-winkler": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/@types/jaro-winkler/-/jaro-winkler-0.2.3.tgz", + "integrity": "sha512-W5qVYCDkBMP7hMM9szj4JvA52CYEyEqL/CKUy7EIulJmlzfqJy5cW0hkzOgJ50Yz8Egfo7MoLF+LUWHUxRZVrg==", + "dev": true + }, "node_modules/@types/json5": { "version": "0.0.29", "dev": true, diff --git a/src/node/consumer/package.json b/src/node/consumer/package.json index f306029..78c9826 100644 --- a/src/node/consumer/package.json +++ b/src/node/consumer/package.json @@ -32,6 +32,7 @@ }, "devDependencies": { "@types/amqplib": "^0.10.4", + "@types/jaro-winkler": "^0.2.3", "@types/magnet-uri": "^5.1.5", "@types/node": "^20.11.16", "@types/stremio-addon-sdk": "^1.6.10", diff --git a/src/node/consumer/src/lib/interfaces/cinemeta_metadata.ts b/src/node/consumer/src/lib/interfaces/cinemeta_metadata.ts index 48cdc8a..fc32250 100644 --- a/src/node/consumer/src/lib/interfaces/cinemeta_metadata.ts +++ b/src/node/consumer/src/lib/interfaces/cinemeta_metadata.ts @@ -61,8 +61,6 @@ export interface CinemetaVideo extends CommonVideoMetadata { rating?: string; overview?: string; thumbnail?: string; - id?: string; - released?: string; description?: string; } export interface CinemetaTrailerStream { diff --git a/src/node/consumer/src/lib/interfaces/common_video_metadata.ts b/src/node/consumer/src/lib/interfaces/common_video_metadata.ts index 4383417..16df177 100644 --- a/src/node/consumer/src/lib/interfaces/common_video_metadata.ts +++ b/src/node/consumer/src/lib/interfaces/common_video_metadata.ts @@ -1,4 +1,8 @@ export interface CommonVideoMetadata { season?: number; - episode?: number; + episode?: number; + released?: string; + title?: string; + name?: string; + id?: string; } \ No newline at end of file diff --git a/src/node/consumer/src/lib/interfaces/downloaded_torrent_file.ts b/src/node/consumer/src/lib/interfaces/downloaded_torrent_file.ts deleted file mode 100644 index 4a0a5a5..0000000 --- a/src/node/consumer/src/lib/interfaces/downloaded_torrent_file.ts +++ /dev/null @@ -1,3 +0,0 @@ -export interface DownloadedTorrentFile extends TorrentStream.TorrentFile { - fileIndex: number; -} \ No newline at end of file diff --git a/src/node/consumer/src/lib/interfaces/kitsu_metadata.ts b/src/node/consumer/src/lib/interfaces/kitsu_metadata.ts index fc63293..e789340 100644 --- a/src/node/consumer/src/lib/interfaces/kitsu_metadata.ts +++ b/src/node/consumer/src/lib/interfaces/kitsu_metadata.ts @@ -30,13 +30,10 @@ export interface KitsuMeta { year?: string; } export interface KitsuVideo extends CommonVideoMetadata { - id?: string; imdbEpisode?: number; imdbSeason?: number; imdb_id?: string; - released?: string; thumbnail?: string; - title?: string; } export interface KitsuTrailer { source?: string; diff --git a/src/node/consumer/src/lib/interfaces/metadata_query.ts b/src/node/consumer/src/lib/interfaces/metadata_query.ts index ecb85ea..2ae9c4e 100644 --- a/src/node/consumer/src/lib/interfaces/metadata_query.ts +++ b/src/node/consumer/src/lib/interfaces/metadata_query.ts @@ -5,4 +5,5 @@ export interface MetaDataQuery { date?: string season?: number episode?: number + id?: string | number } \ No newline at end of file diff --git a/src/node/consumer/src/lib/interfaces/metadata_response.ts b/src/node/consumer/src/lib/interfaces/metadata_response.ts index e1509c5..ce9670e 100644 --- a/src/node/consumer/src/lib/interfaces/metadata_response.ts +++ b/src/node/consumer/src/lib/interfaces/metadata_response.ts @@ -1,3 +1,5 @@ +import {CommonVideoMetadata} from "./common_video_metadata"; + export interface MetadataResponse { kitsuId?: number; imdbId?: number; @@ -7,7 +9,7 @@ export interface MetadataResponse { country?: string; genres?: string[]; status?: string; - videos?: any[]; + videos?: CommonVideoMetadata[]; episodeCount?: number[]; totalCount?: number; } \ No newline at end of file diff --git a/src/node/consumer/src/lib/interfaces/parsable_torrent.ts b/src/node/consumer/src/lib/interfaces/parsable_torrent.ts deleted file mode 100644 index 20683c2..0000000 --- a/src/node/consumer/src/lib/interfaces/parsable_torrent.ts +++ /dev/null @@ -1,9 +0,0 @@ -import {TorrentType} from "../enums/torrent_types"; - -export interface ParsableTorrent { - title: string; - type: TorrentType; - size: number; - pack?: boolean; -} - diff --git a/src/node/consumer/src/lib/interfaces/parsable_torrent_file.ts b/src/node/consumer/src/lib/interfaces/parsable_torrent_file.ts new file mode 100644 index 0000000..394570f --- /dev/null +++ b/src/node/consumer/src/lib/interfaces/parsable_torrent_file.ts @@ -0,0 +1,12 @@ +import {ParseTorrentTitleResult} from "./parse_torrent_title_result"; + +export interface ParsableTorrentFile extends ParseTorrentTitleResult { + name?: string; + path?: string; + size?: number; + length?: number; + fileIndex?: number; + isMovie?: boolean; + imdbId?: string | number; + kitsuId?: number | string; +} \ No newline at end of file diff --git a/src/node/consumer/src/lib/interfaces/parsable_torrent_video.ts b/src/node/consumer/src/lib/interfaces/parsable_torrent_video.ts deleted file mode 100644 index d57b7e7..0000000 --- a/src/node/consumer/src/lib/interfaces/parsable_torrent_video.ts +++ /dev/null @@ -1,6 +0,0 @@ -import {ParseTorrentTitleResult} from "./parse_torrent_title_result"; - -export interface ParsableTorrentVideo extends ParseTorrentTitleResult { - name: string; - path: string; -} \ No newline at end of file diff --git a/src/node/consumer/src/lib/interfaces/parse_torrent_title_result.ts b/src/node/consumer/src/lib/interfaces/parse_torrent_title_result.ts index ea85e8e..0cc445b 100644 --- a/src/node/consumer/src/lib/interfaces/parse_torrent_title_result.ts +++ b/src/node/consumer/src/lib/interfaces/parse_torrent_title_result.ts @@ -1,5 +1,5 @@ export interface ParseTorrentTitleResult { - title: string; + title?: string; date?: string; year?: number | string; resolution?: string; diff --git a/src/node/consumer/src/lib/interfaces/torrent_file_collection.ts b/src/node/consumer/src/lib/interfaces/torrent_file_collection.ts new file mode 100644 index 0000000..500c5b0 --- /dev/null +++ b/src/node/consumer/src/lib/interfaces/torrent_file_collection.ts @@ -0,0 +1,7 @@ +import {ParsableTorrentFile} from "./parsable_torrent_file"; + +export interface TorrentFileCollection { + contents?: ParsableTorrentFile[]; + videos?: ParsableTorrentFile[]; + subtitles?: ParsableTorrentFile[]; +} \ No newline at end of file diff --git a/src/node/consumer/src/lib/interfaces/torrent_info.ts b/src/node/consumer/src/lib/interfaces/torrent_info.ts index 31895bd..93f7082 100644 --- a/src/node/consumer/src/lib/interfaces/torrent_info.ts +++ b/src/node/consumer/src/lib/interfaces/torrent_info.ts @@ -1,12 +1,16 @@ +import {TorrentType} from "../enums/torrent_types"; + export interface TorrentInfo { title: string | null; torrentId: string; infoHash: string | null; seeders: number; - size: string | null; uploadDate: Date; - imdbId: string | undefined; - type: string; - provider: string | null; + imdbId?: string; + kitsuId?: string; + type: TorrentType; + provider?: string | null; trackers: string; + size?: number; + pack?: boolean; } \ No newline at end of file diff --git a/src/node/consumer/src/lib/services/metadata_service.ts b/src/node/consumer/src/lib/services/metadata_service.ts index 5667bb8..e564de2 100644 --- a/src/node/consumer/src/lib/services/metadata_service.ts +++ b/src/node/consumer/src/lib/services/metadata_service.ts @@ -53,13 +53,13 @@ class MetadataService { } } - public getMetadata(id: string | number, type: TorrentType = TorrentType.SERIES): Promise { - if (!id) { + public getMetadata(query: MetaDataQuery): Promise { + if (!query.id) { return Promise.reject("no valid id provided"); } - const key = Number.isInteger(id) || id.toString().match(/^\d+$/) ? `kitsu:${id}` : id; - const metaType = type === TorrentType.MOVIE ? TorrentType.MOVIE : TorrentType.SERIES; + const key = Number.isInteger(query.id) || query.id.toString().match(/^\d+$/) ? `kitsu:${query.id}` : query.id; + const metaType = query.type === TorrentType.MOVIE ? TorrentType.MOVIE : TorrentType.SERIES; return cacheService.cacheWrapMetadata(key.toString(), () => this.requestMetadata(`${KITSU_URL}/meta/${metaType}/${key}.json`) .catch(() => this.requestMetadata(`${CINEMETA_URL}/meta/${metaType}/${key}.json`)) .catch(() => { diff --git a/src/node/consumer/src/lib/services/parsing_service.ts b/src/node/consumer/src/lib/services/parsing_service.ts index 28ea866..dc0b2a6 100644 --- a/src/node/consumer/src/lib/services/parsing_service.ts +++ b/src/node/consumer/src/lib/services/parsing_service.ts @@ -1,20 +1,20 @@ import { parse } from 'parse-torrent-title'; import { TorrentType } from '../enums/torrent_types'; import {ParseTorrentTitleResult} from "../interfaces/parse_torrent_title_result"; -import {ParsableTorrentVideo} from "../interfaces/parsable_torrent_video"; -import {ParsableTorrent} from "../interfaces/parsable_torrent"; +import {ParsableTorrentFile} from "../interfaces/parsable_torrent_file"; +import {TorrentInfo} from "../interfaces/torrent_info"; class ParsingService { private readonly MULTIPLE_FILES_SIZE = 4 * 1024 * 1024 * 1024; // 4 GB - public parseSeriesVideos(torrent: ParsableTorrent, videos: ParsableTorrentVideo[]): ParseTorrentTitleResult[] { + public parseSeriesVideos(torrent: TorrentInfo, videos: ParsableTorrentFile[]): ParsableTorrentFile[] { const parsedTorrentName = parse(torrent.title); const hasMovies = parsedTorrentName.complete || !!torrent.title.match(/movies?(?:\W|$)/i); const parsedVideos = videos.map(video => this.parseSeriesVideo(video, parsedTorrentName)); return parsedVideos.map(video => ({ ...video, isMovie: this.isMovieVideo(video, parsedVideos, torrent.type, hasMovies) })); } - public isPackTorrent(torrent: ParsableTorrent): boolean { + public isPackTorrent(torrent: TorrentInfo): boolean { if (torrent.pack) { return true; } @@ -31,7 +31,7 @@ class ParsingService { return hasMultipleEpisodes && !hasSingleEpisode; } - private parseSeriesVideo(video: ParsableTorrentVideo, parsedTorrentName: ParseTorrentTitleResult): ParseTorrentTitleResult { + private parseSeriesVideo(video: ParsableTorrentFile, parsedTorrentName: ParseTorrentTitleResult): ParseTorrentTitleResult { const videoInfo = parse(video.name); // the episode may be in a folder containing season number if (!Number.isInteger(videoInfo.season) && video.path.includes('/')) { diff --git a/src/node/consumer/src/lib/services/torrent_download_service.ts b/src/node/consumer/src/lib/services/torrent_download_service.ts index 3af4a26..58a7241 100644 --- a/src/node/consumer/src/lib/services/torrent_download_service.ts +++ b/src/node/consumer/src/lib/services/torrent_download_service.ts @@ -3,7 +3,8 @@ import torrentStream from 'torrent-stream'; import { configurationService } from './configuration_service'; import {extensionService} from './extension_service'; import {TorrentInfo} from "../interfaces/torrent_info"; -import {DownloadedTorrentFile} from "../interfaces/downloaded_torrent_file"; +import {TorrentFileCollection} from "../interfaces/torrent_file_collection"; +import {ParsableTorrentFile} from "../interfaces/parsable_torrent_file"; class TorrentDownloadService { private engineOptions: TorrentStream.TorrentEngineOptions = { @@ -14,16 +15,16 @@ class TorrentDownloadService { tracker: true, }; - public async getTorrentFiles(torrent: TorrentInfo, timeout: number = 30000): Promise<{ contents: Array; videos: Array; subtitles: Array }> { + public async getTorrentFiles(torrent: TorrentInfo, timeout: number = 30000): Promise { return this.filesFromTorrentStream(torrent, timeout) - .then((files: Array) => ({ + .then((files: Array) => ({ contents: files, videos: this.filterVideos(files), subtitles: this.filterSubtitles(files) })); } - private async filesFromTorrentStream(torrent: TorrentInfo, timeout: number): Promise> { + private async filesFromTorrentStream(torrent: TorrentInfo, timeout: number): Promise> { if (!torrent.infoHash) { return Promise.reject(new Error("No infoHash...")); } @@ -40,7 +41,12 @@ class TorrentDownloadService { engine = torrentStream(magnet, this.engineOptions); engine.on("ready", () => { - const files: DownloadedTorrentFile[] = engine.files.map((file, fileId) => ({ ...file, fileIndex: fileId })); + const files: ParsableTorrentFile[] = engine.files.map((file, fileId) => ({ + ...file, + fileIndex: fileId, + size: file.length, + title: file.name})); + resolve(files); engine.destroy(() => {}); @@ -49,21 +55,21 @@ class TorrentDownloadService { }); } - private filterVideos(files: Array): Array { + private filterVideos(files: Array): Array { if (files.length === 1 && !Number.isInteger(files[0].fileIndex)) { return files; } const videos = files.filter(file => extensionService.isVideo(file.path || '')); - const maxSize = Math.max(...videos.map((video: DownloadedTorrentFile) => video.length)); + const maxSize = Math.max(...videos.map((video: ParsableTorrentFile) => video.length)); const minSampleRatio = videos.length <= 3 ? 3 : 10; const minAnimeExtraRatio = 5; const minRedundantRatio = videos.length <= 3 ? 30 : Number.MAX_VALUE; - const isSample = (video: DownloadedTorrentFile) => video.path?.match(/sample|bonus|promo/i) && maxSize / parseInt(video.length.toString()) > minSampleRatio; - const isRedundant = (video: DownloadedTorrentFile) => maxSize / parseInt(video.length.toString()) > minRedundantRatio; - const isExtra = (video: DownloadedTorrentFile) => video.path?.match(/extras?\//i); - const isAnimeExtra = (video: DownloadedTorrentFile) => video.path?.match(/(?:\b|_)(?:NC)?(?:ED|OP|PV)(?:v?\d\d?)?(?:\b|_)/i) + const isSample = (video: ParsableTorrentFile) => video.path?.match(/sample|bonus|promo/i) && maxSize / parseInt(video.length.toString()) > minSampleRatio; + const isRedundant = (video: ParsableTorrentFile) => maxSize / parseInt(video.length.toString()) > minRedundantRatio; + const isExtra = (video: ParsableTorrentFile) => video.path?.match(/extras?\//i); + const isAnimeExtra = (video: ParsableTorrentFile) => video.path?.match(/(?:\b|_)(?:NC)?(?:ED|OP|PV)(?:v?\d\d?)?(?:\b|_)/i) && maxSize / parseInt(video.length.toString()) > minAnimeExtraRatio; - const isWatermark = (video: DownloadedTorrentFile) => video.path?.match(/^[A-Z-]+(?:\.[A-Z]+)?\.\w{3,4}$/) + const isWatermark = (video: ParsableTorrentFile) => video.path?.match(/^[A-Z-]+(?:\.[A-Z]+)?\.\w{3,4}$/) && maxSize / parseInt(video.length.toString()) > minAnimeExtraRatio return videos .filter(video => !isSample(video)) @@ -73,7 +79,7 @@ class TorrentDownloadService { .filter(video => !isWatermark(video)); } - private filterSubtitles(files: Array): Array { + private filterSubtitles(files: Array): Array { return files.filter(file => extensionService.isSubtitle(file.path || '')); } } diff --git a/src/node/consumer/src/lib/services/torrent_file_service.ts b/src/node/consumer/src/lib/services/torrent_file_service.ts new file mode 100644 index 0000000..6744d99 --- /dev/null +++ b/src/node/consumer/src/lib/services/torrent_file_service.ts @@ -0,0 +1,589 @@ +import Bottleneck from 'bottleneck'; +import moment from 'moment'; +import { parse } from 'parse-torrent-title'; +import {PromiseHelpers} from '../helpers/promises_helpers'; +import { TorrentType } from '../enums/torrent_types'; +import {TorrentInfo} from "../interfaces/torrent_info"; +import { configurationService } from './configuration_service'; +import { extensionService } from './extension_service'; +import { metadataService } from './metadata_service'; +import { parsingService } from './parsing_service'; +import { torrentDownloadService } from "./torrent_download_service"; +import {logger} from "./logging_service"; +import {MetadataResponse} from "../interfaces/metadata_response"; +import {ParseTorrentTitleResult} from "../interfaces/parse_torrent_title_result"; +import {MetaDataQuery} from "../interfaces/metadata_query"; +import {ParsableTorrentFile} from "../interfaces/parsable_torrent_file"; +import {CommonVideoMetadata} from "../interfaces/common_video_metadata"; +import {TorrentFileCollection} from "../interfaces/torrent_file_collection"; + +class TorrentFileService { + private readonly MIN_SIZE: number = 5 * 1024 * 1024; // 5 MB + private readonly imdb_limiter: Bottleneck = new Bottleneck({ + maxConcurrent: configurationService.metadataConfig.IMDB_CONCURRENT, + minTime: configurationService.metadataConfig.IMDB_INTERVAL_MS + }); + + public async parseTorrentFiles(torrent: TorrentInfo) { + const parsedTorrentName = parse(torrent.title); + const query: MetaDataQuery = { + id: torrent.kitsuId || torrent.imdbId, + type: torrent.type || TorrentType.MOVIE, + }; + const metadata = await metadataService.getMetadata(query) + .then(meta => Object.assign({}, meta)) + .catch(() => undefined); + + if (torrent.type !== TorrentType.ANIME && metadata && metadata.type && metadata.type !== torrent.type) { + // it's actually a movie/series + torrent.type = metadata.type; + } + + if (torrent.type === TorrentType.MOVIE && (!parsedTorrentName.seasons || + parsedTorrentName.season === 5 && [1, 5].includes(parsedTorrentName.episode))) { + return this.parseMovieFiles(torrent, metadata); + } + + return this.parseSeriesFiles(torrent, metadata) + } + + private async parseMovieFiles(torrent: TorrentInfo, metadata: MetadataResponse): Promise { + const {contents, videos, subtitles} = await this.getMoviesTorrentContent(torrent); + const filteredVideos = videos + .filter(video => video.size > this.MIN_SIZE) + .filter(video => !this.isFeaturette(video)); + if (this.isSingleMovie(filteredVideos)) { + const parsedVideos = filteredVideos.map(video => ({ + infoHash: torrent.infoHash, + fileIndex: video.fileIndex, + title: video.path || torrent.title, + size: video.size || torrent.size, + imdbId: torrent.imdbId || metadata && metadata.imdbId, + kitsuId: torrent.kitsuId || metadata && metadata.kitsuId + })); + return {contents, videos: parsedVideos, subtitles}; + } + + const parsedVideos = await PromiseHelpers.sequence(filteredVideos.map(video => () => this.isFeaturette(video) + ? Promise.resolve(video) + : this.findMovieImdbId(video.name).then(imdbId => ({...video, imdbId})))) + .then(videos => videos.map(video => ({ + infoHash: torrent.infoHash, + fileIndex: video.fileIndex, + title: video.path || video.name, + size: video.size, + imdbId: video.imdbId, + }))); + return {contents, videos: parsedVideos, subtitles}; + } + + private async parseSeriesFiles(torrent: TorrentInfo, metadata: MetadataResponse) { + const {contents, videos, subtitles} = await this.getSeriesTorrentContent(torrent); + const parsedVideos = await Promise.resolve(videos) + .then(videos => videos.filter(video => videos.length === 1 || video.size > this.MIN_SIZE)) + .then(videos => parsingService.parseSeriesVideos(torrent, videos)) + .then(videos => this.decomposeEpisodes(torrent, videos, metadata)) + .then(videos => this.assignKitsuOrImdbEpisodes(torrent, videos, metadata)) + .then(videos => Promise.all(videos.map(video => video.isMovie + ? this.mapSeriesMovie(video, torrent) + : this.mapSeriesEpisode(video, torrent, videos)))) + .then(videos => videos + .map((video: ParsableTorrentFile) => this.isFeaturette(video) ? this.clearInfoFields(video) : video)) + return {contents, videos: parsedVideos, subtitles}; + } + + private async getMoviesTorrentContent(torrent: TorrentInfo) { + const files = await torrentDownloadService.getTorrentFiles(torrent) + .catch(error => { + if (!parsingService.isPackTorrent(torrent)) { + return { videos: [{name: torrent.title, path: torrent.title, size: torrent.size, fileIndex: null}], contents:[], subtitles: []} + } + return Promise.reject(error); + }); + + if (files.contents && files.contents.length && !files.videos.length && this.isDiskTorrent(files.contents)) { + files.videos = [{name: torrent.title, path: torrent.title, size: torrent.size}]; + } + return files; + } + + private async getSeriesTorrentContent(torrent: TorrentInfo) { + return torrentDownloadService.getTorrentFiles(torrent) + .catch(error => { + if (!parsingService.isPackTorrent(torrent)) { + return { videos: [{ name: torrent.title, path: torrent.title, size: torrent.size }], subtitles: [], contents: [] } + } + return Promise.reject(error); + }); + } + + private async mapSeriesEpisode(file: ParsableTorrentFile, torrent: TorrentInfo, files: ParsableTorrentFile[]) : Promise { + if (!file.episodes && !file.episodes) { + if (files.length === 1 || files.some(f => f.episodes || f.episodes) || parse(torrent.title).seasons) { + return Promise.resolve({ + infoHash: torrent.infoHash, + fileIndex: file.fileIndex, + title: file.path || file.name, + size: file.size, + imdbId: torrent.imdbId || file.imdbId, + }); + } + return Promise.resolve([]); + } + const episodeIndexes = [...(file.episodes || file.episodes).keys()]; + return Promise.resolve(episodeIndexes.map((index) => ({ + infoHash: torrent.infoHash, + fileIndex: file.fileIndex, + title: file.path || file.name, + size: file.size, + imdbId: file.imdbId || torrent.imdbId, + season: file.season, + episode: file.episodes && file.episodes[index], + episodes: file.episodes, + kitsuId: file.kitsuId || torrent.kitsuId, + }))) + } + + private async mapSeriesMovie(file: ParsableTorrentFile, torrent: TorrentInfo): Promise { + const kitsuId= torrent.type === TorrentType.ANIME ? await this.findMovieKitsuId(file) + .then(result => { + if (result instanceof Error) { + logger.warn(`Failed to retrieve kitsuId due to error: ${result.message}`); + return undefined; + } + return result; + }): undefined; + + const imdbId = !kitsuId ? await this.findMovieImdbId(file) : undefined; + + const query: MetaDataQuery = { + id: kitsuId || imdbId, + type: TorrentType.MOVIE + }; + + const metadataOrError = await metadataService.getMetadata(query); + if (metadataOrError instanceof Error) { + logger.warn(`Failed to retrieve metadata due to error: ${metadataOrError.message}`); + // return default result or throw error, depending on your use case + return [{ + infoHash: torrent.infoHash, + fileIndex: file.fileIndex, + title: file.path || file.name, + size: file.size, + imdbId: imdbId, + kitsuId: kitsuId, + episodes: undefined, + imdbSeason: undefined, + imdbEpisode: undefined, + kitsuEpisode: undefined + }]; + } + // at this point, TypeScript infers that metadataOrError is actually MetadataResponse + const metadata = metadataOrError; + const hasEpisode = metadata.videos && metadata.videos.length && (file.episode || metadata.videos.length === 1); + const episodeVideo = hasEpisode && metadata.videos[(file.episode || 1) - 1]; + return [{ + infoHash: torrent.infoHash, + fileIndex: file.fileIndex, + title: file.path || file.name, + size: file.size, + imdbId: metadata.imdbId || imdbId, + kitsuId: metadata.kitsuId || kitsuId, + season: episodeVideo && metadata.imdbId ? episodeVideo.season : undefined, + episode: episodeVideo && metadata.imdbId | metadata.kitsuId ? episodeVideo.episode || episodeVideo.episode : undefined, + }]; + } + + private async decomposeEpisodes(torrent: TorrentInfo, files: ParsableTorrentFile[], metadata: MetadataResponse = { episodeCount: [] }) { + if (files.every(file => !file.episodes && !file.date)) { + return files; + } + + this.preprocessEpisodes(files); + + if (torrent.type === TorrentType.ANIME && torrent.kitsuId) { + if (this.needsCinemetaMetadataForAnime(files, metadata)) { + // In some cases anime could be resolved to wrong kitsuId + // because of imdb season naming/absolute per series naming/multiple seasons + // So in these cases we need to fetch cinemeta based metadata and decompose episodes using that + await this.updateToCinemetaMetadata(metadata); + if (files.some(file => Number.isInteger(file.season))) { + // sometimes multi season anime torrents don't include season 1 naming + files + .filter(file => !Number.isInteger(file.season) && file.episodes) + .forEach(file => file.season = 1); + } + } else { + // otherwise for anime type episodes are always absolute and for a single season + files + .filter(file => file.episodes && file.season !== 0) + .forEach(file => file.season = 1); + return files; + } + } + + const sortedEpisodes = files + .map(file => !file.isMovie && file.episodes || []) + .reduce((a, b) => a.concat(b), []) + .sort((a, b) => a - b); + + if (this.isConcatSeasonAndEpisodeFiles(files, sortedEpisodes, metadata)) { + this.decomposeConcatSeasonAndEpisodeFiles(files, metadata); + } else if (this.isDateEpisodeFiles(files, metadata)) { + this.decomposeDateEpisodeFiles(files, metadata); + } else if (this.isAbsoluteEpisodeFiles(torrent, files, metadata)) { + this.decomposeAbsoluteEpisodeFiles(torrent, files, metadata); + } + // decomposeEpisodeTitleFiles(torrent, files, metadata); + + return files; + } + + private preprocessEpisodes(files: ParsableTorrentFile[]) { + // reverse special episode naming when they named with 0 episode, ie. S02E00 + files + .filter(file => Number.isInteger(file.season) && file.episode === 0) + .forEach(file => { + file.episode = file.season + file.episodes = [file.season] + file.season = 0; + }) + } + + private isConcatSeasonAndEpisodeFiles(files: ParsableTorrentFile[], sortedEpisodes: number[], metadata: MetadataResponse) { + if (metadata.kitsuId !== undefined) { + // anime does not use this naming scheme in 99% of cases; + return false; + } + // decompose concat season and episode files (ex. 101=S01E01) in case: + // 1. file has a season, but individual files are concatenated with that season (ex. path Season 5/511 - Prize + // Fighters.avi) + // 2. file does not have a season and the episode does not go out of range for the concat season + // episode count + const thresholdAbove = Math.max(Math.ceil(files.length * 0.05), 5); + const thresholdSorted = Math.max(Math.ceil(files.length * 0.8), 8); + const threshold = Math.max(Math.ceil(files.length * 0.8), 5); + const sortedConcatEpisodes = sortedEpisodes + .filter(ep => ep > 100) + .filter(ep => metadata.episodeCount[this.div100(ep) - 1] < ep) + .filter(ep => metadata.episodeCount[this.div100(ep) - 1] >= this.mod100(ep)); + const concatFileEpisodes = files + .filter(file => !file.isMovie && file.episodes) + .filter(file => !file.season || file.episodes.every(ep => this.div100(ep) === file.season)); + const concatAboveTotalEpisodeCount = files + .filter(file => !file.isMovie && file.episodes && file.episodes.every(ep => ep > 100)) + .filter(file => file.episodes.every(ep => ep > metadata.totalCount)); + return sortedConcatEpisodes.length >= thresholdSorted && concatFileEpisodes.length >= threshold + || concatAboveTotalEpisodeCount.length >= thresholdAbove; + } + + private isDateEpisodeFiles(files: ParsableTorrentFile[], metadata: MetadataResponse) { + return files.every(file => (!file.season || !metadata.episodeCount[file.season - 1]) && file.date); + } + + private isAbsoluteEpisodeFiles(torrent: TorrentInfo, files: ParsableTorrentFile[], metadata: MetadataResponse) { + const threshold = Math.ceil(files.length / 5); + const isAnime = torrent.type === TorrentType.ANIME && torrent.kitsuId; + const nonMovieEpisodes = files + .filter(file => !file.isMovie && file.episodes); + const absoluteEpisodes = files + .filter(file => file.season && file.episodes) + .filter(file => file.episodes.every(ep => metadata.episodeCount[file.season - 1] < ep)) + return nonMovieEpisodes.every(file => !file.season) + || (isAnime && nonMovieEpisodes.every(file => file.season > metadata.episodeCount.length)) + || absoluteEpisodes.length >= threshold; + } + + private isNewEpisodeNotInMetadata(torrent: TorrentInfo, file: ParsableTorrentFile, metadata: MetadataResponse) { + // new episode might not yet been indexed by cinemeta. + // detect this if episode number is larger than the last episode or season is larger than the last one + // only for non anime metas + const isAnime = torrent.type === TorrentType.ANIME && torrent.kitsuId; + return !isAnime && !file.isMovie && file.episodes && file.season !== 1 + && /continuing|current/i.test(metadata.status) + && file.season >= metadata.episodeCount.length + && file.episodes.every(ep => ep > (metadata.episodeCount[file.season - 1] || 0)); + } + + private decomposeConcatSeasonAndEpisodeFiles(files: ParsableTorrentFile[], metadata: MetadataResponse) { + files + .filter(file => file.episodes && file.season !== 0 && file.episodes.every(ep => ep > 100)) + .filter(file => metadata.episodeCount[(file.season || this.div100(file.episodes[0])) - 1] < 100) + .filter(file => file.season && file.episodes.every(ep => this.div100(ep) === file.season) || !file.season) + .forEach(file => { + file.season = this.div100(file.episodes[0]); + file.episodes = file.episodes.map(ep => this.mod100(ep)) + }); + + } + + private decomposeAbsoluteEpisodeFiles(torrent: TorrentInfo, files: ParsableTorrentFile[], metadata: MetadataResponse) { + if (metadata.episodeCount.length === 0) { + files + .filter(file => !Number.isInteger(file.season) && file.episodes && !file.isMovie) + .forEach(file => { + file.season = 1; + }); + return; + } + files + .filter(file => file.episodes && !file.isMovie && file.season !== 0) + .filter(file => !this.isNewEpisodeNotInMetadata(torrent, file, metadata)) + .filter(file => !file.season || (metadata.episodeCount[file.season - 1] || 0) < file.episodes[0]) + .forEach(file => { + const seasonIdx = ([...metadata.episodeCount.keys()] + .find((i) => metadata.episodeCount.slice(0, i + 1).reduce((a, b) => a + b) >= file.episodes[0]) + + 1 || metadata.episodeCount.length) - 1; + + file.season = seasonIdx + 1; + file.episodes = file.episodes + .map(ep => ep - metadata.episodeCount.slice(0, seasonIdx).reduce((a, b) => a + b, 0)) + }); + } + + private decomposeDateEpisodeFiles(files: ParsableTorrentFile[], metadata: MetadataResponse) { + if (!metadata || !metadata.videos || !metadata.videos.length) { + return; + } + + const timeZoneOffset = this.getTimeZoneOffset(metadata.country); + const offsetVideos = metadata.videos + .reduce((map, video) => { + const releaseDate = moment(video.released).utcOffset(timeZoneOffset).format('YYYY-MM-DD'); + map[releaseDate] = video; + return map; + }, {}); + + files + .filter(file => file.date) + .forEach(file => { + const video = offsetVideos[file.date]; + if (video) { + file.season = video.season; + file.episodes = [video.episode]; + } + }); + } + + private getTimeZoneOffset(country: string | undefined) { + switch (country) { + case 'United States': + case 'USA': + return '-08:00'; + default: + return '00:00'; + } + } + + private assignKitsuOrImdbEpisodes(torrent: TorrentInfo, files: ParsableTorrentFile[], metadata: MetadataResponse) { + if (!metadata || !metadata.videos || !metadata.videos.length) { + if (torrent.type === TorrentType.ANIME) { + // assign episodes as kitsu episodes for anime when no metadata available for imdb mapping + files + .filter(file => file.season && file.episodes) + .forEach(file => { + file.season = undefined; + file.episodes = undefined; + }) + if (metadata.type === TorrentType.MOVIE && files.every(file => !file.imdbId)) { + // sometimes a movie has episode naming, thus not recognized as a movie and imdbId not assigned + files.forEach(file => file.imdbId = metadata.imdbId); + } + } + return files; + } + + const seriesMapping: CommonVideoMetadata = metadata.videos + .reduce((map, video) => { + const episodeMap = map[video.season] || {}; + episodeMap[video.episode] = video; + map[video.season] = episodeMap; + return map; + }, {}); + + if (metadata.videos.some(video => Number.isInteger(video.season)) || !metadata.imdbId) { + files.filter((file => Number.isInteger(file.season) && file.episodes)) + .map(file => { + const seasonMapping = seriesMapping[file.season]; + const episodeMapping = seasonMapping && seasonMapping[file.episodes[0]]; + if (episodeMapping && Number.isInteger(episodeMapping.imdbSeason)) { + file.imdbId = metadata.imdbId; + file.season = episodeMapping.imdbSeason; + file.episodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].imdbEpisode); + } else { + // no imdb mapping available for episode + file.season = undefined; + file.episodes = undefined; + } + }); + } else if (metadata.videos.some(video => video.episode)) { + // imdb episode info is base + files + .filter(file => Number.isInteger(file.season) && file.episodes) + .forEach(file => { + if (seriesMapping[file.season]) { + const seasonMapping = seriesMapping[file.season]; + file.imdbId = metadata.imdbId; + file.kitsuId = seasonMapping[file.episodes[0]] && seasonMapping[file.episodes[0]].kitsuId; + file.episodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].kitsuEpisode); + } else if (seriesMapping[file.season - 1]) { + // sometimes a second season might be a continuation of the previous season + const seasonMapping = seriesMapping[file.season - 1] as CommonVideoMetadata; + const episodes = Object.values(seasonMapping); + const firstKitsuId = episodes.length && episodes[0]; + const differentTitlesCount = new Set(episodes.map(ep => ep.id)).size + const skippedCount = episodes.filter(ep => ep.id === firstKitsuId).length; + const seasonEpisodes = files + .filter(otherFile => otherFile.season === file.season) + .reduce((a, b) => a.episodes.concat(b.episodes), []); + const isAbsoluteOrder = seasonEpisodes.episodes.every(ep => ep > skippedCount && ep <= episodes.length) + const isNormalOrder = seasonEpisodes.episodes.every(ep => ep + skippedCount <= episodes.length) + if (differentTitlesCount >= 1 && (isAbsoluteOrder || isNormalOrder)) { + file.imdbId = metadata.imdbId; + file.season = file.season - 1; + file.episodes = file.episodes.map(ep => isAbsoluteOrder ? ep : ep + skippedCount); + file.kitsuId = seasonMapping[file.episodes[0]].kitsuId; + file.episodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].kitsuEpisode); + } + } else if (Object.values(seriesMapping).length === 1 && seriesMapping[1]) { + // sometimes series might be named with sequel season but it's not a season on imdb and a new title + const seasonMapping = seriesMapping[1]; + file.imdbId = metadata.imdbId; + file.season = 1; + file.kitsuId = seasonMapping[file.episodes[0]].kitsuId; + file.episodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].kitsuEpisode); + } + }); + } + return files; + } + + private needsCinemetaMetadataForAnime(files: ParsableTorrentFile[], metadata: MetadataResponse) { + if (!metadata || !metadata.imdbId || !metadata.videos || !metadata.videos.length) { + return false; + } + + const minSeason = Math.min(...metadata.videos.map(video => video.season)) || Number.MAX_VALUE; + const maxSeason = Math.max(...metadata.videos.map(video => video.season)) || Number.MAX_VALUE; + const differentSeasons = new Set(metadata.videos + .map(video => video.season) + .filter(season => Number.isInteger(season))).size; + const total = metadata.totalCount || Number.MAX_VALUE; + return differentSeasons > 1 || files + .filter(file => !file.isMovie && file.episodes) + .some(file => file.season < minSeason || file.season > maxSeason || file.episodes.every(ep => ep > total)); + } + + private async updateToCinemetaMetadata(metadata: MetadataResponse) { + const query: MetaDataQuery = { + id: metadata.imdbId, + type: metadata.type + }; + + return await metadataService.getMetadata(query) + .then((newMetadataOrError) => { + if (newMetadataOrError instanceof Error) { + // handle error + logger.warn(`Failed ${metadata.imdbId} metadata cinemeta update due: ${newMetadataOrError.message}`); + return metadata; // or throw newMetadataOrError to propagate error up the call stack + } + // At this point TypeScript infers newMetadataOrError to be of type MetadataResponse + let newMetadata = newMetadataOrError; + if (!newMetadata.videos || !newMetadata.videos.length) { + return metadata; + } else { + metadata.videos = newMetadata.videos; + metadata.episodeCount = newMetadata.episodeCount; + metadata.totalCount = newMetadata.totalCount; + return metadata; + } + }) + } + + private findMovieImdbId(title: ParseTorrentTitleResult | string) { + const parsedTitle = typeof title === 'string' ? parse(title) : title; + logger.debug(`Finding movie imdbId for ${title}`); + return this.imdb_limiter.schedule(async () => { + const imdbQuery = { + title: parsedTitle.title, + year: parsedTitle.year, + type: TorrentType.MOVIE + }; + try { + return await metadataService.getImdbId(imdbQuery); + } catch (e) { + return undefined; + } + }); + } + + private async findMovieKitsuId(title: ParseTorrentTitleResult | string) { + const parsedTitle = typeof title === 'string' ? parse(title) : title; + const kitsuQuery = { + title: parsedTitle.title, + year: parsedTitle.year, + season: parsedTitle.season, + type: TorrentType.MOVIE + }; + try { + return await metadataService.getKitsuId(kitsuQuery); + } catch (e) { + return undefined; + } + } + + private isDiskTorrent(contents: ParsableTorrentFile[]) { + return contents.some(content => extensionService.isDisk(content.path)); + } + + private isSingleMovie(videos: ParsableTorrentFile[]) { + return videos.length === 1 || + (videos.length === 2 && + videos.find(v => /\b(?:part|disc|cd)[ ._-]?0?1\b|^0?1\.\w{2,4}$/i.test(v.path)) && + videos.find(v => /\b(?:part|disc|cd)[ ._-]?0?2\b|^0?2\.\w{2,4}$/i.test(v.path))); + } + + private isFeaturette(video: ParsableTorrentFile) { + return /featurettes?\/|extras-grym/i.test(video.path); + } + + private clearInfoFields(video: ParsableTorrentFile) { + video.imdbId = undefined; + video.season = undefined; + video.episode = undefined; + video.kitsuId = undefined; + return video; + } + + private div100(episode: number) { + return (episode / 100 >> 0); // floor to nearest int + } + + private mod100(episode: number) { + return episode % 100; + } +} + +export const torrentFileService = new TorrentFileService(); + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/node/consumer/src/lib/services/torrent_subtitle_service.ts b/src/node/consumer/src/lib/services/torrent_subtitle_service.ts new file mode 100644 index 0000000..9cbe739 --- /dev/null +++ b/src/node/consumer/src/lib/services/torrent_subtitle_service.ts @@ -0,0 +1,88 @@ +import { parse } from 'parse-torrent-title'; + +class TorrentSubtitleService { + public assignSubtitles(contents: any, videos: any[], subtitles: any[]) { + if (videos && videos.length && subtitles && subtitles.length) { + if (videos.length === 1) { + videos[0].subtitles = subtitles; + return { contents, videos, subtitles: [] }; + } + + const parsedVideos = videos.map(video => this.parseVideo(video)); + const assignedSubs = subtitles.map(subtitle => ({ subtitle, videos: this.mostProbableSubtitleVideos(subtitle, parsedVideos) })); + const unassignedSubs = assignedSubs.filter(assignedSub => !assignedSub.videos).map(assignedSub => assignedSub.subtitle); + + assignedSubs + .filter(assignedSub => assignedSub.videos) + .forEach(assignedSub => assignedSub.videos.forEach(video => video.subtitles = (video.subtitles || []).concat(assignedSub.subtitle))); + return { contents, videos, subtitles: unassignedSubs }; + } + return { contents, videos, subtitles }; + } + + private parseVideo(video: any) { + const fileName = video.title.split('/').pop().replace(/\.(\w{2,4})$/, ''); + const folderName = video.title.replace(/\/?[^/]+$/, ''); + return { + videoFile: video, + fileName: fileName, + folderName: folderName, + ...this.parseFilename(video.title) + }; + } + + private mostProbableSubtitleVideos(subtitle: any, parsedVideos: any[]) { + const subTitle = (subtitle.title || subtitle.path).split('/').pop().replace(/\.(\w{2,4})$/, ''); + const parsedSub = this.parsePath(subtitle.title || subtitle.path); + const byFileName = parsedVideos.filter(video => subTitle.includes(video.fileName)); + if (byFileName.length === 1) { + return byFileName.map(v => v.videoFile); + } + const byTitleSeasonEpisode = parsedVideos.filter(video => video.title === parsedSub.title + && this.arrayEquals(video.seasons, parsedSub.seasons) + && this.arrayEquals(video.episodes, parsedSub.episodes)); + if (this.singleVideoFile(byTitleSeasonEpisode)) { + return byTitleSeasonEpisode.map(v => v.videoFile); + } + const bySeasonEpisode = parsedVideos.filter(video => this.arrayEquals(video.seasons, parsedSub.seasons) + && this.arrayEquals(video.episodes, parsedSub.episodes)); + if (this.singleVideoFile(bySeasonEpisode)) { + return bySeasonEpisode.map(v => v.videoFile); + } + const byTitle = parsedVideos.filter(video => video.title && video.title === parsedSub.title); + if (this.singleVideoFile(byTitle)) { + return byTitle.map(v => v.videoFile); + } + const byEpisode = parsedVideos.filter(video => this.arrayEquals(video.episodes, parsedSub.episodes)); + if (this.singleVideoFile(byEpisode)) { + return byEpisode.map(v => v.videoFile); + } + return undefined; + } + + private singleVideoFile(videos: any[]) { + return new Set(videos.map(v => v.videoFile.fileIndex)).size === 1; + } + + private parsePath(path: string) { + const pathParts = path.split('/').map(part => this.parseFilename(part)); + const parsedWithEpisode = pathParts.find(parsed => parsed.season && parsed.episodes); + return parsedWithEpisode || pathParts[pathParts.length - 1]; + } + + private parseFilename(filename: string) { + const parsedInfo = parse(filename) + const titleEpisode = parsedInfo.title.match(/(\d+)$/); + if (!parsedInfo.episodes && titleEpisode) { + parsedInfo.episodes = [parseInt(titleEpisode[1], 10)]; + } + return parsedInfo; + } + + private arrayEquals(array1: any[], array2: any[]) { + if (!array1 || !array2) return array1 === array2; + return array1.length === array2.length && array1.every((value, index) => value === array2[index]) + } +} + +export const torrentSubtitleService = new TorrentSubtitleService(); diff --git a/src/node/consumer/src/lib/torrentEntries.js b/src/node/consumer/src/lib/torrentEntries.js index 0212c1a..ed4933e 100644 --- a/src/node/consumer/src/lib/torrentEntries.js +++ b/src/node/consumer/src/lib/torrentEntries.js @@ -3,8 +3,8 @@ import { metadataService } from './services/metadata_service'; import { parsingService } from './services/parsing_service'; import {PromiseHelpers} from './helpers/promises_helpers'; import { repository } from '../repository/database_repository'; -import { parseTorrentFiles } from './torrentFiles.js'; -import { assignSubtitles } from './torrentSubtitles.js'; +import { torrentFileService } from './services/torrent_file_service'; +import { torrentSubtitleService } from './services/torrent_subtitle_service'; import { TorrentType } from './enums/torrent_types'; import {logger} from './services/logging_service'; @@ -43,9 +43,9 @@ export async function createTorrentEntry(torrent, overwrite = false) { return; } - const { contents, videos, subtitles } = await parseTorrentFiles(torrent) + const { contents, videos, subtitles } = await torrentFileService.parseTorrentFiles(torrent) .then(torrentContents => overwrite ? overwriteExistingFiles(torrent, torrentContents) : torrentContents) - .then(torrentContents => assignSubtitles(torrentContents)) + .then(torrentContents => torrentSubtitleService.assignSubtitles(torrentContents)) .catch(error => { logger.warn(`Failed getting files for ${torrent.title}`, error.message); return {}; @@ -135,9 +135,9 @@ export async function createTorrentContents(torrent) { const imdbId = PromiseHelpers.mostCommonValue(storedVideos.map(stored => stored.imdbId)); const kitsuId = PromiseHelpers.mostCommonValue(storedVideos.map(stored => stored.kitsuId)); - const { contents, videos, subtitles } = await parseTorrentFiles({ ...torrent, imdbId, kitsuId }) + const { contents, videos, subtitles } = await torrentFileService.parseTorrentFiles({ ...torrent, imdbId, kitsuId }) .then(torrentContents => notOpenedVideo ? torrentContents : { ...torrentContents, videos: storedVideos }) - .then(torrentContents => assignSubtitles(torrentContents)) + .then(torrentContents => torrentSubtitleService.assignSubtitles(torrentContents)) .catch(error => { logger.warn(`Failed getting contents for [${torrent.infoHash}] ${torrent.title}`, error.message); return {}; diff --git a/src/node/consumer/src/lib/torrentFiles.js b/src/node/consumer/src/lib/torrentFiles.js deleted file mode 100644 index c2fda7c..0000000 --- a/src/node/consumer/src/lib/torrentFiles.js +++ /dev/null @@ -1,526 +0,0 @@ -import Bottleneck from 'bottleneck'; -import distance from 'jaro-winkler'; -import moment from 'moment'; -import { parse } from 'parse-torrent-title'; -import { configurationService } from './services/configuration_service'; -import { extensionService } from './services/extension_service'; -import { metadataService } from './services/metadata_service'; -import { parsingService } from './services/parsing_service'; -import {PromiseHelpers} from './helpers/promises_helpers'; -import {torrentDownloadService} from "./services/torrent_download_service"; -import { TorrentType } from './enums/torrent_types'; -import {logger} from "./services/logging_service"; - -const MIN_SIZE = 5 * 1024 * 1024; // 5 MB -const imdb_limiter = new Bottleneck({ maxConcurrent: configurationService.metadataConfig.IMDB_CONCURRENT, minTime: configurationService.metadataConfig.IMDB_INTERVAL_MS }); - -export async function parseTorrentFiles(torrent) { - const parsedTorrentName = parse(torrent.title); - const metadata = await metadataService.getMetadata(torrent.kitsuId || torrent.imdbId, torrent.type || TorrentType.MOVIE) - .then(meta => Object.assign({}, meta)) - .catch(() => undefined); - - // if (metadata && metadata.type !== torrent.type && torrent.type !== Type.ANIME) { - // throw new Error(`Mismatching entry type for ${torrent.name}: ${torrent.type}!=${metadata.type}`); - // } - if (torrent.type !== TorrentType.ANIME && metadata && metadata.type && metadata.type !== torrent.type) { - // it's actually a movie/series - torrent.type = metadata.type; - } - - if (torrent.type === TorrentType.MOVIE && (!parsedTorrentName.seasons || - parsedTorrentName.season === 5 && [1, 5].includes(parsedTorrentName.episode))) { - return parseMovieFiles(torrent, parsedTorrentName, metadata); - } - - return parseSeriesFiles(torrent, parsedTorrentName, metadata) -} - -async function parseMovieFiles(torrent, parsedName, metadata) { - const { contents, videos, subtitles } = await getMoviesTorrentContent(torrent); - const filteredVideos = videos - .filter(video => video.size > MIN_SIZE) - .filter(video => !isFeaturette(video)); - if (isSingleMovie(filteredVideos)) { - const parsedVideos = filteredVideos.map(video => ({ - infoHash: torrent.infoHash, - fileIndex: video.fileIndex, - title: video.path || torrent.title, - size: video.size || torrent.size, - imdbId: torrent.imdbId || metadata && metadata.imdbId, - kitsuId: torrent.kitsuId || metadata && metadata.kitsuId - })); - return { contents, videos: parsedVideos, subtitles }; - } - - const parsedVideos = await PromiseHelpers.sequence(filteredVideos.map(video => () => isFeaturette(video) - ? Promise.resolve(video) - : findMovieImdbId(video.name).then(imdbId => ({ ...video, imdbId })))) - .then(videos => videos.map(video => ({ - infoHash: torrent.infoHash, - fileIndex: video.fileIndex, - title: video.path || video.name, - size: video.size, - imdbId: video.imdbId, - }))); - return { contents, videos: parsedVideos, subtitles }; -} - -async function parseSeriesFiles(torrent, parsedName, metadata) { - const { contents, videos, subtitles } = await getSeriesTorrentContent(torrent); - const parsedVideos = await Promise.resolve(videos) - .then(videos => videos.filter(video => videos.length === 1 || video.size > MIN_SIZE)) - .then(videos => parsingService.parseSeriesVideos(torrent, videos)) - .then(videos => decomposeEpisodes(torrent, videos, metadata)) - .then(videos => assignKitsuOrImdbEpisodes(torrent, videos, metadata)) - .then(videos => Promise.all(videos.map(video => video.isMovie - ? mapSeriesMovie(video, torrent) - : mapSeriesEpisode(video, torrent, videos)))) - .then(videos => videos - .reduce((a, b) => a.concat(b), []) - .map(video => isFeaturette(video) ? clearInfoFields(video) : video)) - return { contents, videos: parsedVideos, subtitles }; -} - -async function getMoviesTorrentContent(torrent) { - const files = await torrentFiles(torrent) - .catch(error => { - if (!isPackTorrent(torrent)) { - return { videos: [{ name: torrent.title, path: torrent.title, size: torrent.size }] } - } - return Promise.reject(error); - }); - if (files.contents && files.contents.length && !files.videos.length && isDiskTorrent(files.contents)) { - files.videos = [{ name: torrent.title, path: torrent.title, size: torrent.size }]; - } - return files; -} - -async function getSeriesTorrentContent(torrent) { - return torrentDownloadService.getTorrentFiles(torrent) - .catch(error => { - if (!parsingService.isPackTorrent(torrent)) { - return { videos: [{ name: torrent.title, path: torrent.title, size: torrent.size }] } - } - return Promise.reject(error); - }); -} - -async function mapSeriesEpisode(file, torrent, files) { - if (!file.episodes && !file.kitsuEpisodes) { - if (files.length === 1 || files.some(f => f.episodes || f.kitsuEpisodes) || parse(torrent.title).seasons) { - return Promise.resolve({ - infoHash: torrent.infoHash, - fileIndex: file.fileIndex, - title: file.path || file.name, - size: file.size, - imdbId: torrent.imdbId || file.imdbId, - }); - } - return Promise.resolve([]); - } - const episodeIndexes = [...(file.episodes || file.kitsuEpisodes).keys()]; - return Promise.resolve(episodeIndexes.map((index) => ({ - infoHash: torrent.infoHash, - fileIndex: file.fileIndex, - title: file.path || file.name, - size: file.size, - imdbId: file.imdbId || torrent.imdbId, - imdbSeason: file.season, - imdbEpisode: file.episodes && file.episodes[index], - kitsuId: file.kitsuId || torrent.kitsuId, - kitsuEpisode: file.kitsuEpisodes && file.kitsuEpisodes[index] - }))) -} - -async function mapSeriesMovie(file, torrent) { - const kitsuId = torrent.type === TorrentType.ANIME ? await findMovieKitsuId(file) : undefined; - const imdbId = !kitsuId ? await findMovieImdbId(file) : undefined; - const metadata = await metadataService.getMetadata(kitsuId || imdbId, TorrentType.MOVIE).catch(() => ({})); - const hasEpisode = metadata.videos && metadata.videos.length && (file.episode || metadata.videos.length === 1); - const episodeVideo = hasEpisode && metadata.videos[(file.episode || 1) - 1]; - return [{ - infoHash: torrent.infoHash, - fileIndex: file.fileIndex, - title: file.path || file.name, - size: file.size, - imdbId: metadata.imdbId || imdbId, - kitsuId: metadata.kitsuId || kitsuId, - imdbSeason: episodeVideo && metadata.imdbId ? episodeVideo.imdbSeason : undefined, - imdbEpisode: episodeVideo && metadata.imdbId ? episodeVideo.imdbEpisode || episodeVideo.episode : undefined, - kitsuEpisode: episodeVideo && metadata.kitsuId ? episodeVideo.kitsuEpisode || episodeVideo.episode : undefined - }]; -} - -async function decomposeEpisodes(torrent, files, metadata = { episodeCount: [] }) { - if (files.every(file => !file.episodes && !file.date)) { - return files; - } - - preprocessEpisodes(files); - - if (torrent.type === TorrentType.ANIME && torrent.kitsuId) { - if (needsCinemetaMetadataForAnime(files, metadata)) { - // In some cases anime could be resolved to wrong kitsuId - // because of imdb season naming/absolute per series naming/multiple seasons - // So in these cases we need to fetch cinemeta based metadata and decompose episodes using that - await updateToCinemetaMetadata(metadata); - if (files.some(file => Number.isInteger(file.season))) { - // sometimes multi season anime torrents don't include season 1 naming - files - .filter(file => !Number.isInteger(file.season) && file.episodes) - .forEach(file => file.season = 1); - } - } else { - // otherwise for anime type episodes are always absolute and for a single season - files - .filter(file => file.episodes && file.season !== 0) - .forEach(file => file.season = 1); - return files; - } - } - - const sortedEpisodes = files - .map(file => !file.isMovie && file.episodes || []) - .reduce((a, b) => a.concat(b), []) - .sort((a, b) => a - b); - - if (isConcatSeasonAndEpisodeFiles(files, sortedEpisodes, metadata)) { - decomposeConcatSeasonAndEpisodeFiles(torrent, files, metadata); - } else if (isDateEpisodeFiles(files, metadata)) { - decomposeDateEpisodeFiles(torrent, files, metadata); - } else if (isAbsoluteEpisodeFiles(torrent, files, metadata)) { - decomposeAbsoluteEpisodeFiles(torrent, files, metadata); - } - // decomposeEpisodeTitleFiles(torrent, files, metadata); - - return files; -} - -function preprocessEpisodes(files) { - // reverse special episode naming when they named with 0 episode, ie. S02E00 - files - .filter(file => Number.isInteger(file.season) && file.episode === 0) - .forEach(file => { - file.episode = file.season - file.episodes = [file.season] - file.season = 0; - }) -} - -function isConcatSeasonAndEpisodeFiles(files, sortedEpisodes, metadata) { - if (metadata.kitsuId !== undefined) { - // anime does not use this naming scheme in 99% of cases; - return false; - } - // decompose concat season and episode files (ex. 101=S01E01) in case: - // 1. file has a season, but individual files are concatenated with that season (ex. path Season 5/511 - Prize - // Fighters.avi) - // 2. file does not have a season and the episode does not go out of range for the concat season - // episode count - const thresholdAbove = Math.max(Math.ceil(files.length * 0.05), 5); - const thresholdSorted = Math.max(Math.ceil(files.length * 0.8), 8); - const threshold = Math.max(Math.ceil(files.length * 0.8), 5); - const sortedConcatEpisodes = sortedEpisodes - .filter(ep => ep > 100) - .filter(ep => metadata.episodeCount[div100(ep) - 1] < ep) - .filter(ep => metadata.episodeCount[div100(ep) - 1] >= mod100(ep)); - const concatFileEpisodes = files - .filter(file => !file.isMovie && file.episodes) - .filter(file => !file.season || file.episodes.every(ep => div100(ep) === file.season)); - const concatAboveTotalEpisodeCount = files - .filter(file => !file.isMovie && file.episodes && file.episodes.every(ep => ep > 100)) - .filter(file => file.episodes.every(ep => ep > metadata.totalCount)); - return sortedConcatEpisodes.length >= thresholdSorted && concatFileEpisodes.length >= threshold - || concatAboveTotalEpisodeCount.length >= thresholdAbove; -} - -function isDateEpisodeFiles(files, metadata) { - return files.every(file => (!file.season || !metadata.episodeCount[file.season - 1]) && file.date); -} - -function isAbsoluteEpisodeFiles(torrent, files, metadata) { - const threshold = Math.ceil(files.length / 5); - const isAnime = torrent.type === TorrentType.ANIME && torrent.kitsuId; - const nonMovieEpisodes = files - .filter(file => !file.isMovie && file.episodes); - const absoluteEpisodes = files - .filter(file => file.season && file.episodes) - .filter(file => file.episodes.every(ep => metadata.episodeCount[file.season - 1] < ep)) - return nonMovieEpisodes.every(file => !file.season) - || (isAnime && nonMovieEpisodes.every(file => file.season > metadata.episodeCount.length)) - || absoluteEpisodes.length >= threshold; -} - -function isNewEpisodeNotInMetadata(torrent, file, metadata) { - // new episode might not yet been indexed by cinemeta. - // detect this if episode number is larger than the last episode or season is larger than the last one - // only for non anime metas - const isAnime = torrent.type === TorrentType.ANIME && torrent.kitsuId; - return !isAnime && !file.isMovie && file.episodes && file.season !== 1 - && /continuing|current/i.test(metadata.status) - && file.season >= metadata.episodeCount.length - && file.episodes.every(ep => ep > (metadata.episodeCount[file.season - 1] || 0)); -} - -function decomposeConcatSeasonAndEpisodeFiles(torrent, files, metadata) { - files - .filter(file => file.episodes && file.season !== 0 && file.episodes.every(ep => ep > 100)) - .filter(file => metadata.episodeCount[(file.season || div100(file.episodes[0])) - 1] < 100) - .filter(file => file.season && file.episodes.every(ep => div100(ep) === file.season) || !file.season) - .forEach(file => { - file.season = div100(file.episodes[0]); - file.episodes = file.episodes.map(ep => mod100(ep)) - }); - -} - -function decomposeAbsoluteEpisodeFiles(torrent, files, metadata) { - if (metadata.episodeCount.length === 0) { - files - .filter(file => !Number.isInteger(file.season) && file.episodes && !file.isMovie) - .forEach(file => { - file.season = 1; - }); - return; - } - files - .filter(file => file.episodes && !file.isMovie && file.season !== 0) - .filter(file => !isNewEpisodeNotInMetadata(torrent, file, metadata)) - .filter(file => !file.season || (metadata.episodeCount[file.season - 1] || 0) < file.episodes[0]) - .forEach(file => { - const seasonIdx = ([...metadata.episodeCount.keys()] - .find((i) => metadata.episodeCount.slice(0, i + 1).reduce((a, b) => a + b) >= file.episodes[0]) - + 1 || metadata.episodeCount.length) - 1; - - file.season = seasonIdx + 1; - file.episodes = file.episodes - .map(ep => ep - metadata.episodeCount.slice(0, seasonIdx).reduce((a, b) => a + b, 0)) - }); -} - -function decomposeDateEpisodeFiles(torrent, files, metadata) { - if (!metadata || !metadata.videos || !metadata.videos.length) { - return; - } - - const timeZoneOffset = getTimeZoneOffset(metadata.country); - const offsetVideos = metadata.videos - .reduce((map, video) => { - const releaseDate = moment(video.released).utcOffset(timeZoneOffset).format('YYYY-MM-DD'); - map[releaseDate] = video; - return map; - }, {}); - - files - .filter(file => file.date) - .forEach(file => { - const video = offsetVideos[file.date]; - if (video) { - file.season = video.season; - file.episodes = [video.episode]; - } - }); -} - - -/* eslint-disable no-unused-vars */ -function decomposeEpisodeTitleFiles(torrent, files, metadata) { - files - // .filter(file => !file.season) - .map(file => { - const episodeTitle = file.name.replace('_', ' ') - .replace(/^.*(?:E\d+[abc]?|- )\s?(.+)\.\w{1,4}$/, '$1') - .trim(); - const foundEpisode = metadata.videos - .map(video => ({ ...video, distance: distance(episodeTitle, video.name) })) - .sort((a, b) => b.distance - a.distance)[0]; - if (foundEpisode) { - file.isMovie = false; - file.season = foundEpisode.season; - file.episodes = [foundEpisode.episode]; - } - }) -} -/* eslint-enable no-unused-vars */ - -function getTimeZoneOffset(country) { - switch (country) { - case 'United States': - case 'USA': - return '-08:00'; - default: - return '00:00'; - } -} - -function assignKitsuOrImdbEpisodes(torrent, files, metadata) { - if (!metadata || !metadata.videos || !metadata.videos.length) { - if (torrent.type === TorrentType.ANIME) { - // assign episodes as kitsu episodes for anime when no metadata available for imdb mapping - files - .filter(file => file.season && file.episodes) - .forEach(file => { - file.kitsuEpisodes = file.episodes; - file.season = undefined; - file.episodes = undefined; - }) - if (metadata.type === TorrentType.MOVIE && files.every(file => !file.imdbId)) { - // sometimes a movie has episode naming, thus not recognized as a movie and imdbId not assigned - files.forEach(file => file.imdbId = metadata.imdbId); - } - } - return files; - } - - const seriesMapping = metadata.videos - .reduce((map, video) => { - const episodeMap = map[video.season] || {}; - episodeMap[video.episode] = video; - map[video.season] = episodeMap; - return map; - }, {}); - - if (metadata.videos.some(video => Number.isInteger(video.imdbSeason)) || !metadata.imdbId) { - // kitsu episode info is the base - files - .filter(file => Number.isInteger(file.season) && file.episodes) - .map(file => { - const seasonMapping = seriesMapping[file.season]; - const episodeMapping = seasonMapping && seasonMapping[file.episodes[0]]; - file.kitsuEpisodes = file.episodes; - if (episodeMapping && Number.isInteger(episodeMapping.imdbSeason)) { - file.imdbId = metadata.imdbId; - file.season = episodeMapping.imdbSeason; - file.episodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].imdbEpisode); - } else { - // no imdb mapping available for episode - file.season = undefined; - file.episodes = undefined; - } - }); - } else if (metadata.videos.some(video => video.kitsuEpisode)) { - // imdb episode info is base - files - .filter(file => Number.isInteger(file.season) && file.episodes) - .forEach(file => { - if (seriesMapping[file.season]) { - const seasonMapping = seriesMapping[file.season]; - file.imdbId = metadata.imdbId; - file.kitsuId = seasonMapping[file.episodes[0]] && seasonMapping[file.episodes[0]].kitsuId; - file.kitsuEpisodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].kitsuEpisode); - } else if (seriesMapping[file.season - 1]) { - // sometimes a second season might be a continuation of the previous season - const seasonMapping = seriesMapping[file.season - 1]; - const episodes = Object.values(seasonMapping); - const firstKitsuId = episodes.length && episodes[0].kitsuId; - const differentTitlesCount = new Set(episodes.map(ep => ep.kitsuId)).size - const skippedCount = episodes.filter(ep => ep.kitsuId === firstKitsuId).length; - const seasonEpisodes = files - .filter(otherFile => otherFile.season === file.season) - .reduce((a, b) => a.concat(b.episodes), []); - const isAbsoluteOrder = seasonEpisodes.every(ep => ep > skippedCount && ep <= episodes.length) - const isNormalOrder = seasonEpisodes.every(ep => ep + skippedCount <= episodes.length) - if (differentTitlesCount >= 1 && (isAbsoluteOrder || isNormalOrder)) { - file.imdbId = metadata.imdbId; - file.season = file.season - 1; - file.episodes = file.episodes.map(ep => isAbsoluteOrder ? ep : ep + skippedCount); - file.kitsuId = seasonMapping[file.episodes[0]].kitsuId; - file.kitsuEpisodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].kitsuEpisode); - } - } else if (Object.values(seriesMapping).length === 1 && seriesMapping[1]) { - // sometimes series might be named with sequel season but it's not a season on imdb and a new title - const seasonMapping = seriesMapping[1]; - file.imdbId = metadata.imdbId; - file.season = 1; - file.kitsuId = seasonMapping[file.episodes[0]].kitsuId; - file.kitsuEpisodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].kitsuEpisode); - } - }); - } - return files; -} - -function needsCinemetaMetadataForAnime(files, metadata) { - if (!metadata || !metadata.imdbId || !metadata.videos || !metadata.videos.length) { - return false; - } - - const minSeason = Math.min(...metadata.videos.map(video => video.imdbSeason)) || Number.MAX_VALUE; - const maxSeason = Math.max(...metadata.videos.map(video => video.imdbSeason)) || Number.MAX_VALUE; - const differentSeasons = new Set(metadata.videos - .map(video => video.imdbSeason) - .filter(season => Number.isInteger(season))).size; - const total = metadata.totalCount || Number.MAX_VALUE; - return differentSeasons > 1 || files - .filter(file => !file.isMovie && file.episodes) - .some(file => file.season < minSeason || file.season > maxSeason || file.episodes.every(ep => ep > total)); -} - -async function updateToCinemetaMetadata(metadata) { - return metadataService.getMetadata(metadata.imdbId, metadata.type) - .then(newMetadata => !newMetadata.videos || !newMetadata.videos.length ? metadata : newMetadata) - .then(newMetadata => { - metadata.videos = newMetadata.videos; - metadata.episodeCount = newMetadata.episodeCount; - metadata.totalCount = newMetadata.totalCount; - return metadata; - }) - .catch(error => logger.warn(`Failed ${metadata.imdbId} metadata cinemeta update due: ${error.message}`)); -} - -function findMovieImdbId(title) { - const parsedTitle = typeof title === 'string' ? parse(title) : title; - logger.debug(`Finding movie imdbId for ${title}`); - return imdb_limiter.schedule(() => { - const imdbQuery = { - title: parsedTitle.title, - year: parsedTitle.year, - type: TorrentType.MOVIE - }; - return metadataService.getImdbId(imdbQuery).catch(() => undefined); - }); -} - -function findMovieKitsuId(title) { - const parsedTitle = typeof title === 'string' ? parse(title) : title; - const kitsuQuery = { - title: parsedTitle.title, - year: parsedTitle.year, - season: parsedTitle.season, - type: TorrentType.MOVIE - }; - return metadataService.getKitsuId(kitsuQuery).catch(() => undefined); -} - -function isDiskTorrent(contents) { - return contents.some(content => extensionService.isDisk(content.path)); -} - -function isSingleMovie(videos) { - return videos.length === 1 || - (videos.length === 2 && - videos.find(v => /\b(?:part|disc|cd)[ ._-]?0?1\b|^0?1\.\w{2,4}$/i.test(v.path)) && - videos.find(v => /\b(?:part|disc|cd)[ ._-]?0?2\b|^0?2\.\w{2,4}$/i.test(v.path))); -} - -function isFeaturette(video) { - return /featurettes?\/|extras-grym/i.test(video.path); -} - -function clearInfoFields(video) { - video.imdbId = undefined; - video.imdbSeason = undefined; - video.imdbEpisode = undefined; - video.kitsuId = undefined; - video.kitsuEpisode = undefined; - return video; -} - -function div100(episode) { - return (episode / 100 >> 0); // floor to nearest int -} - -function mod100(episode) { - return episode % 100; -} \ No newline at end of file diff --git a/src/node/consumer/src/lib/torrentSubtitles.js b/src/node/consumer/src/lib/torrentSubtitles.js deleted file mode 100644 index 7bd9ab4..0000000 --- a/src/node/consumer/src/lib/torrentSubtitles.js +++ /dev/null @@ -1,89 +0,0 @@ -import { parse } from 'parse-torrent-title'; - -export function assignSubtitles({ contents, videos, subtitles }) { - if (videos && videos.length && subtitles && subtitles.length) { - if (videos.length === 1) { - videos[0].subtitles = subtitles; - return { contents, videos, subtitles: [] }; - } - - const parsedVideos = videos - .map(video => _parseVideo(video)); - const assignedSubs = subtitles - .map(subtitle => ({ subtitle, videos: _mostProbableSubtitleVideos(subtitle, parsedVideos) })); - const unassignedSubs = assignedSubs - .filter(assignedSub => !assignedSub.videos) - .map(assignedSub => assignedSub.subtitle); - - assignedSubs - .filter(assignedSub => assignedSub.videos) - .forEach(assignedSub => assignedSub.videos - .forEach(video => video.subtitles = (video.subtitles || []).concat(assignedSub.subtitle))); - return { contents, videos, subtitles: unassignedSubs }; - } - return { contents, videos, subtitles }; -} - -function _parseVideo(video) { - const fileName = video.title.split('/').pop().replace(/\.(\w{2,4})$/, ''); - const folderName = video.title.replace(/\/?[^/]+$/, ''); - return { - videoFile: video, - fileName: fileName, - folderName: folderName, - ...parseFilename(video.title) - }; -} - -function _mostProbableSubtitleVideos(subtitle, parsedVideos) { - const subTitle = (subtitle.title || subtitle.path).split('/').pop().replace(/\.(\w{2,4})$/, ''); - const parsedSub = parsePath(subtitle.title || subtitle.path); - const byFileName = parsedVideos.filter(video => subTitle.includes(video.fileName)); - if (byFileName.length === 1) { - return byFileName.map(v => v.videoFile); - } - const byTitleSeasonEpisode = parsedVideos.filter(video => video.title === parsedSub.title - && arrayEquals(video.seasons, parsedSub.seasons) - && arrayEquals(video.episodes, parsedSub.episodes)); - if (singleVideoFile(byTitleSeasonEpisode)) { - return byTitleSeasonEpisode.map(v => v.videoFile); - } - const bySeasonEpisode = parsedVideos.filter(video => arrayEquals(video.seasons, parsedSub.seasons) - && arrayEquals(video.episodes, parsedSub.episodes)); - if (singleVideoFile(bySeasonEpisode)) { - return bySeasonEpisode.map(v => v.videoFile); - } - const byTitle = parsedVideos.filter(video => video.title && video.title === parsedSub.title); - if (singleVideoFile(byTitle)) { - return byTitle.map(v => v.videoFile); - } - const byEpisode = parsedVideos.filter(video => arrayEquals(video.episodes, parsedSub.episodes)); - if (singleVideoFile(byEpisode)) { - return byEpisode.map(v => v.videoFile); - } - return undefined; -} - -function singleVideoFile(videos) { - return new Set(videos.map(v => v.videoFile.fileIndex)).size === 1; -} - -function parsePath(path) { - const pathParts = path.split('/').map(part => parseFilename(part)); - const parsedWithEpisode = pathParts.find(parsed => parsed.season && parsed.episodes); - return parsedWithEpisode || pathParts[pathParts.length - 1]; -} - -function parseFilename(filename) { - const parsedInfo = parse(filename) - const titleEpisode = parsedInfo.title.match(/(\d+)$/); - if (!parsedInfo.episodes && titleEpisode) { - parsedInfo.episodes = [parseInt(titleEpisode[1], 10)]; - } - return parsedInfo; -} - -function arrayEquals(array1, array2) { - if (!array1 || !array2) return array1 === array2; - return array1.length === array2.length && array1.every((value, index) => value === array2[index]) -} \ No newline at end of file