tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

torrent file service done - bound to be some issues in this..

Browse Source
This commit is contained in:
iPromKnight
2024-02-05 15:09:31 +00:00
committed by iPromKnight
parent aeebe88aca
commit 3f8fee518e
22 changed files with 756 additions and 673 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/node/consumer/package-lock.json generated
View File

@@ -31,6 +31,7 @@
}, },
"devDependencies": { "devDependencies": {
"@types/amqplib": "^0.10.4", "@types/amqplib": "^0.10.4",
"@types/jaro-winkler": "^0.2.3",
"@types/magnet-uri": "^5.1.5", "@types/magnet-uri": "^5.1.5",
"@types/node": "^20.11.16", "@types/node": "^20.11.16",
"@types/stremio-addon-sdk": "^1.6.10", "@types/stremio-addon-sdk": "^1.6.10",
@@ -583,6 +584,12 @@
"@types/ms": "*" "@types/ms": "*"
} }
}, },
"node_modules/@types/jaro-winkler": {
"version": "0.2.3",
"resolved": "https://registry.npmjs.org/@types/jaro-winkler/-/jaro-winkler-0.2.3.tgz",
"integrity": "sha512-W5qVYCDkBMP7hMM9szj4JvA52CYEyEqL/CKUy7EIulJmlzfqJy5cW0hkzOgJ50Yz8Egfo7MoLF+LUWHUxRZVrg==",
"dev": true
},
"node_modules/@types/json5": { "node_modules/@types/json5": {
"version": "0.0.29", "version": "0.0.29",
"dev": true, "dev": true,

1
src/node/consumer/package.json
View File

@@ -32,6 +32,7 @@
}, },
"devDependencies": { "devDependencies": {
"@types/amqplib": "^0.10.4", "@types/amqplib": "^0.10.4",
"@types/jaro-winkler": "^0.2.3",
"@types/magnet-uri": "^5.1.5", "@types/magnet-uri": "^5.1.5",
"@types/node": "^20.11.16", "@types/node": "^20.11.16",
"@types/stremio-addon-sdk": "^1.6.10", "@types/stremio-addon-sdk": "^1.6.10",

2
src/node/consumer/src/lib/interfaces/cinemeta_metadata.ts
View File

@@ -61,8 +61,6 @@ export interface CinemetaVideo extends CommonVideoMetadata {
rating?: string; rating?: string;
overview?: string; overview?: string;
thumbnail?: string; thumbnail?: string;
id?: string;
released?: string;
description?: string; description?: string;
} }
export interface CinemetaTrailerStream { export interface CinemetaTrailerStream {

6
src/node/consumer/src/lib/interfaces/common_video_metadata.ts
View File

@@ -1,4 +1,8 @@
export interface CommonVideoMetadata { export interface CommonVideoMetadata {
season?: number; season?: number;
episode?: number; episode?: number;
released?: string;
title?: string;
name?: string;
id?: string;
} }

3
src/node/consumer/src/lib/interfaces/downloaded_torrent_file.ts
View File

@@ -1,3 +0,0 @@
export interface DownloadedTorrentFile extends TorrentStream.TorrentFile {
fileIndex: number;
}

3
src/node/consumer/src/lib/interfaces/kitsu_metadata.ts
View File

@@ -30,13 +30,10 @@ export interface KitsuMeta {
year?: string; year?: string;
} }
export interface KitsuVideo extends CommonVideoMetadata { export interface KitsuVideo extends CommonVideoMetadata {
id?: string;
imdbEpisode?: number; imdbEpisode?: number;
imdbSeason?: number; imdbSeason?: number;
imdb_id?: string; imdb_id?: string;
released?: string;
thumbnail?: string; thumbnail?: string;
title?: string;
} }
export interface KitsuTrailer { export interface KitsuTrailer {
source?: string; source?: string;

1
src/node/consumer/src/lib/interfaces/metadata_query.ts
View File

@@ -5,4 +5,5 @@ export interface MetaDataQuery {
date?: string date?: string
season?: number season?: number
episode?: number episode?: number
id?: string | number
} }

4
src/node/consumer/src/lib/interfaces/metadata_response.ts
View File

@@ -1,3 +1,5 @@
import {CommonVideoMetadata} from "./common_video_metadata";
export interface MetadataResponse { export interface MetadataResponse {
kitsuId?: number; kitsuId?: number;
imdbId?: number; imdbId?: number;
@@ -7,7 +9,7 @@ export interface MetadataResponse {
country?: string; country?: string;
genres?: string[]; genres?: string[];
status?: string; status?: string;
videos?: any[]; videos?: CommonVideoMetadata[];
episodeCount?: number[]; episodeCount?: number[];
totalCount?: number; totalCount?: number;
} }

9
src/node/consumer/src/lib/interfaces/parsable_torrent.ts
View File

@@ -1,9 +0,0 @@
import {TorrentType} from "../enums/torrent_types";
export interface ParsableTorrent {
title: string;
type: TorrentType;
size: number;
pack?: boolean;
}

12
src/node/consumer/src/lib/interfaces/parsable_torrent_file.ts Normal file
View File

@@ -0,0 +1,12 @@
import {ParseTorrentTitleResult} from "./parse_torrent_title_result";
export interface ParsableTorrentFile extends ParseTorrentTitleResult {
name?: string;
path?: string;
size?: number;
length?: number;
fileIndex?: number;
isMovie?: boolean;
imdbId?: string | number;
kitsuId?: number | string;
}

6
src/node/consumer/src/lib/interfaces/parsable_torrent_video.ts
View File

@@ -1,6 +0,0 @@
import {ParseTorrentTitleResult} from "./parse_torrent_title_result";
export interface ParsableTorrentVideo extends ParseTorrentTitleResult {
name: string;
path: string;
}

2
src/node/consumer/src/lib/interfaces/parse_torrent_title_result.ts
View File

@@ -1,5 +1,5 @@
export interface ParseTorrentTitleResult { export interface ParseTorrentTitleResult {
title: string; title?: string;
date?: string; date?: string;
year?: number | string; year?: number | string;
resolution?: string; resolution?: string;

7
src/node/consumer/src/lib/interfaces/torrent_file_collection.ts Normal file
View File

@@ -0,0 +1,7 @@
import {ParsableTorrentFile} from "./parsable_torrent_file";
export interface TorrentFileCollection {
contents?: ParsableTorrentFile[];
videos?: ParsableTorrentFile[];
subtitles?: ParsableTorrentFile[];
}

12
src/node/consumer/src/lib/interfaces/torrent_info.ts
View File

@@ -1,12 +1,16 @@
import {TorrentType} from "../enums/torrent_types";
export interface TorrentInfo { export interface TorrentInfo {
title: string | null; title: string | null;
torrentId: string; torrentId: string;
infoHash: string | null; infoHash: string | null;
seeders: number; seeders: number;
size: string | null;
uploadDate: Date; uploadDate: Date;
imdbId: string | undefined; imdbId?: string;
type: string; kitsuId?: string;
provider: string | null; type: TorrentType;
provider?: string | null;
trackers: string; trackers: string;
size?: number;
pack?: boolean;
} }

8
src/node/consumer/src/lib/services/metadata_service.ts
View File

@@ -53,13 +53,13 @@ class MetadataService {
} }
} }
public getMetadata(id: string | number, type: TorrentType = TorrentType.SERIES): Promise<MetadataResponse | Error> { public getMetadata(query: MetaDataQuery): Promise<MetadataResponse | Error> {
if (!id) { if (!query.id) {
return Promise.reject("no valid id provided"); return Promise.reject("no valid id provided");
} }
const key = Number.isInteger(id) || id.toString().match(/^\d+$/) ? `kitsu:${id}` : id; const key = Number.isInteger(query.id) || query.id.toString().match(/^\d+$/) ? `kitsu:${query.id}` : query.id;
const metaType = type === TorrentType.MOVIE ? TorrentType.MOVIE : TorrentType.SERIES; const metaType = query.type === TorrentType.MOVIE ? TorrentType.MOVIE : TorrentType.SERIES;
return cacheService.cacheWrapMetadata(key.toString(), () => this.requestMetadata(`${KITSU_URL}/meta/${metaType}/${key}.json`) return cacheService.cacheWrapMetadata(key.toString(), () => this.requestMetadata(`${KITSU_URL}/meta/${metaType}/${key}.json`)
.catch(() => this.requestMetadata(`${CINEMETA_URL}/meta/${metaType}/${key}.json`)) .catch(() => this.requestMetadata(`${CINEMETA_URL}/meta/${metaType}/${key}.json`))
.catch(() => { .catch(() => {

10
src/node/consumer/src/lib/services/parsing_service.ts
View File

@@ -1,20 +1,20 @@
import { parse } from 'parse-torrent-title'; import { parse } from 'parse-torrent-title';
import { TorrentType } from '../enums/torrent_types'; import { TorrentType } from '../enums/torrent_types';
import {ParseTorrentTitleResult} from "../interfaces/parse_torrent_title_result"; import {ParseTorrentTitleResult} from "../interfaces/parse_torrent_title_result";
import {ParsableTorrentVideo} from "../interfaces/parsable_torrent_video"; import {ParsableTorrentFile} from "../interfaces/parsable_torrent_file";
import {ParsableTorrent} from "../interfaces/parsable_torrent"; import {TorrentInfo} from "../interfaces/torrent_info";
class ParsingService { class ParsingService {
private readonly MULTIPLE_FILES_SIZE = 4 * 1024 * 1024 * 1024; // 4 GB private readonly MULTIPLE_FILES_SIZE = 4 * 1024 * 1024 * 1024; // 4 GB
public parseSeriesVideos(torrent: ParsableTorrent, videos: ParsableTorrentVideo[]): ParseTorrentTitleResult[] { public parseSeriesVideos(torrent: TorrentInfo, videos: ParsableTorrentFile[]): ParsableTorrentFile[] {
const parsedTorrentName = parse(torrent.title); const parsedTorrentName = parse(torrent.title);
const hasMovies = parsedTorrentName.complete || !!torrent.title.match(/movies?(?:\W|$)/i); const hasMovies = parsedTorrentName.complete || !!torrent.title.match(/movies?(?:\W|$)/i);
const parsedVideos = videos.map(video => this.parseSeriesVideo(video, parsedTorrentName)); const parsedVideos = videos.map(video => this.parseSeriesVideo(video, parsedTorrentName));
return parsedVideos.map(video => ({ ...video, isMovie: this.isMovieVideo(video, parsedVideos, torrent.type, hasMovies) })); return parsedVideos.map(video => ({ ...video, isMovie: this.isMovieVideo(video, parsedVideos, torrent.type, hasMovies) }));
} }
public isPackTorrent(torrent: ParsableTorrent): boolean { public isPackTorrent(torrent: TorrentInfo): boolean {
if (torrent.pack) { if (torrent.pack) {
return true; return true;
} }
@@ -31,7 +31,7 @@ class ParsingService {
return hasMultipleEpisodes && !hasSingleEpisode; return hasMultipleEpisodes && !hasSingleEpisode;
} }
private parseSeriesVideo(video: ParsableTorrentVideo, parsedTorrentName: ParseTorrentTitleResult): ParseTorrentTitleResult { private parseSeriesVideo(video: ParsableTorrentFile, parsedTorrentName: ParseTorrentTitleResult): ParseTorrentTitleResult {
const videoInfo = parse(video.name); const videoInfo = parse(video.name);
// the episode may be in a folder containing season number // the episode may be in a folder containing season number
if (!Number.isInteger(videoInfo.season) && video.path.includes('/')) { if (!Number.isInteger(videoInfo.season) && video.path.includes('/')) {

32
src/node/consumer/src/lib/services/torrent_download_service.ts
View File

@@ -3,7 +3,8 @@ import torrentStream from 'torrent-stream';
import { configurationService } from './configuration_service'; import { configurationService } from './configuration_service';
import {extensionService} from './extension_service'; import {extensionService} from './extension_service';
import {TorrentInfo} from "../interfaces/torrent_info"; import {TorrentInfo} from "../interfaces/torrent_info";
import {DownloadedTorrentFile} from "../interfaces/downloaded_torrent_file"; import {TorrentFileCollection} from "../interfaces/torrent_file_collection";
import {ParsableTorrentFile} from "../interfaces/parsable_torrent_file";
class TorrentDownloadService { class TorrentDownloadService {
private engineOptions: TorrentStream.TorrentEngineOptions = { private engineOptions: TorrentStream.TorrentEngineOptions = {
@@ -14,16 +15,16 @@ class TorrentDownloadService {
tracker: true, tracker: true,
}; };
public async getTorrentFiles(torrent: TorrentInfo, timeout: number = 30000): Promise<{ contents: Array<DownloadedTorrentFile>; videos: Array<DownloadedTorrentFile>; subtitles: Array<DownloadedTorrentFile> }> { public async getTorrentFiles(torrent: TorrentInfo, timeout: number = 30000): Promise<TorrentFileCollection> {
return this.filesFromTorrentStream(torrent, timeout) return this.filesFromTorrentStream(torrent, timeout)
.then((files: Array<DownloadedTorrentFile>) => ({ .then((files: Array<ParsableTorrentFile>) => ({
contents: files, contents: files,
videos: this.filterVideos(files), videos: this.filterVideos(files),
subtitles: this.filterSubtitles(files) subtitles: this.filterSubtitles(files)
})); }));
} }
private async filesFromTorrentStream(torrent: TorrentInfo, timeout: number): Promise<Array<DownloadedTorrentFile>> { private async filesFromTorrentStream(torrent: TorrentInfo, timeout: number): Promise<Array<ParsableTorrentFile>> {
if (!torrent.infoHash) { if (!torrent.infoHash) {
return Promise.reject(new Error("No infoHash...")); return Promise.reject(new Error("No infoHash..."));
} }
@@ -40,7 +41,12 @@ class TorrentDownloadService {
engine = torrentStream(magnet, this.engineOptions); engine = torrentStream(magnet, this.engineOptions);
engine.on("ready", () => { engine.on("ready", () => {
const files: DownloadedTorrentFile[] = engine.files.map((file, fileId) => ({ ...file, fileIndex: fileId })); const files: ParsableTorrentFile[] = engine.files.map((file, fileId) => ({
...file,
fileIndex: fileId,
size: file.length,
title: file.name}));
resolve(files); resolve(files);
engine.destroy(() => {}); engine.destroy(() => {});
@@ -49,21 +55,21 @@ class TorrentDownloadService {
}); });
} }
private filterVideos(files: Array<DownloadedTorrentFile>): Array<DownloadedTorrentFile> { private filterVideos(files: Array<ParsableTorrentFile>): Array<ParsableTorrentFile> {
if (files.length === 1 && !Number.isInteger(files[0].fileIndex)) { if (files.length === 1 && !Number.isInteger(files[0].fileIndex)) {
return files; return files;
} }
const videos = files.filter(file => extensionService.isVideo(file.path || '')); const videos = files.filter(file => extensionService.isVideo(file.path || ''));
const maxSize = Math.max(...videos.map((video: DownloadedTorrentFile) => video.length)); const maxSize = Math.max(...videos.map((video: ParsableTorrentFile) => video.length));
const minSampleRatio = videos.length <= 3 ? 3 : 10; const minSampleRatio = videos.length <= 3 ? 3 : 10;
const minAnimeExtraRatio = 5; const minAnimeExtraRatio = 5;
const minRedundantRatio = videos.length <= 3 ? 30 : Number.MAX_VALUE; const minRedundantRatio = videos.length <= 3 ? 30 : Number.MAX_VALUE;
const isSample = (video: DownloadedTorrentFile) => video.path?.match(/sample|bonus|promo/i) && maxSize / parseInt(video.length.toString()) > minSampleRatio; const isSample = (video: ParsableTorrentFile) => video.path?.match(/sample|bonus|promo/i) && maxSize / parseInt(video.length.toString()) > minSampleRatio;
const isRedundant = (video: DownloadedTorrentFile) => maxSize / parseInt(video.length.toString()) > minRedundantRatio; const isRedundant = (video: ParsableTorrentFile) => maxSize / parseInt(video.length.toString()) > minRedundantRatio;
const isExtra = (video: DownloadedTorrentFile) => video.path?.match(/extras?\//i); const isExtra = (video: ParsableTorrentFile) => video.path?.match(/extras?\//i);
const isAnimeExtra = (video: DownloadedTorrentFile) => video.path?.match(/(?:\b|_)(?:NC)?(?:ED|OP|PV)(?:v?\d\d?)?(?:\b|_)/i) const isAnimeExtra = (video: ParsableTorrentFile) => video.path?.match(/(?:\b|_)(?:NC)?(?:ED|OP|PV)(?:v?\d\d?)?(?:\b|_)/i)
&& maxSize / parseInt(video.length.toString()) > minAnimeExtraRatio; && maxSize / parseInt(video.length.toString()) > minAnimeExtraRatio;
const isWatermark = (video: DownloadedTorrentFile) => video.path?.match(/^[A-Z-]+(?:\.[A-Z]+)?\.\w{3,4}$/) const isWatermark = (video: ParsableTorrentFile) => video.path?.match(/^[A-Z-]+(?:\.[A-Z]+)?\.\w{3,4}$/)
&& maxSize / parseInt(video.length.toString()) > minAnimeExtraRatio && maxSize / parseInt(video.length.toString()) > minAnimeExtraRatio
return videos return videos
.filter(video => !isSample(video)) .filter(video => !isSample(video))
@@ -73,7 +79,7 @@ class TorrentDownloadService {
.filter(video => !isWatermark(video)); .filter(video => !isWatermark(video));
} }
private filterSubtitles(files: Array<DownloadedTorrentFile>): Array<DownloadedTorrentFile> { private filterSubtitles(files: Array<ParsableTorrentFile>): Array<ParsableTorrentFile> {
return files.filter(file => extensionService.isSubtitle(file.path || '')); return files.filter(file => extensionService.isSubtitle(file.path || ''));
} }
} }

589
src/node/consumer/src/lib/services/torrent_file_service.ts Normal file
View File

@@ -0,0 +1,589 @@
import Bottleneck from 'bottleneck';
import moment from 'moment';
import { parse } from 'parse-torrent-title';
import {PromiseHelpers} from '../helpers/promises_helpers';
import { TorrentType } from '../enums/torrent_types';
import {TorrentInfo} from "../interfaces/torrent_info";
import { configurationService } from './configuration_service';
import { extensionService } from './extension_service';
import { metadataService } from './metadata_service';
import { parsingService } from './parsing_service';
import { torrentDownloadService } from "./torrent_download_service";
import {logger} from "./logging_service";
import {MetadataResponse} from "../interfaces/metadata_response";
import {ParseTorrentTitleResult} from "../interfaces/parse_torrent_title_result";
import {MetaDataQuery} from "../interfaces/metadata_query";
import {ParsableTorrentFile} from "../interfaces/parsable_torrent_file";
import {CommonVideoMetadata} from "../interfaces/common_video_metadata";
import {TorrentFileCollection} from "../interfaces/torrent_file_collection";
class TorrentFileService {
private readonly MIN_SIZE: number = 5 * 1024 * 1024; // 5 MB
private readonly imdb_limiter: Bottleneck = new Bottleneck({
maxConcurrent: configurationService.metadataConfig.IMDB_CONCURRENT,
minTime: configurationService.metadataConfig.IMDB_INTERVAL_MS
});
public async parseTorrentFiles(torrent: TorrentInfo) {
const parsedTorrentName = parse(torrent.title);
const query: MetaDataQuery = {
id: torrent.kitsuId || torrent.imdbId,
type: torrent.type || TorrentType.MOVIE,
};
const metadata = await metadataService.getMetadata(query)
.then(meta => Object.assign({}, meta))
.catch(() => undefined);
if (torrent.type !== TorrentType.ANIME && metadata && metadata.type && metadata.type !== torrent.type) {
// it's actually a movie/series
torrent.type = metadata.type;
}
if (torrent.type === TorrentType.MOVIE && (!parsedTorrentName.seasons ||
parsedTorrentName.season === 5 && [1, 5].includes(parsedTorrentName.episode))) {
return this.parseMovieFiles(torrent, metadata);
}
return this.parseSeriesFiles(torrent, metadata)
}
private async parseMovieFiles(torrent: TorrentInfo, metadata: MetadataResponse): Promise<TorrentFileCollection> {
const {contents, videos, subtitles} = await this.getMoviesTorrentContent(torrent);
const filteredVideos = videos
.filter(video => video.size > this.MIN_SIZE)
.filter(video => !this.isFeaturette(video));
if (this.isSingleMovie(filteredVideos)) {
const parsedVideos = filteredVideos.map(video => ({
infoHash: torrent.infoHash,
fileIndex: video.fileIndex,
title: video.path || torrent.title,
size: video.size || torrent.size,
imdbId: torrent.imdbId || metadata && metadata.imdbId,
kitsuId: torrent.kitsuId || metadata && metadata.kitsuId
}));
return {contents, videos: parsedVideos, subtitles};
}
const parsedVideos = await PromiseHelpers.sequence(filteredVideos.map(video => () => this.isFeaturette(video)
? Promise.resolve(video)
: this.findMovieImdbId(video.name).then(imdbId => ({...video, imdbId}))))
.then(videos => videos.map(video => ({
infoHash: torrent.infoHash,
fileIndex: video.fileIndex,
title: video.path || video.name,
size: video.size,
imdbId: video.imdbId,
})));
return {contents, videos: parsedVideos, subtitles};
}
private async parseSeriesFiles(torrent: TorrentInfo, metadata: MetadataResponse) {
const {contents, videos, subtitles} = await this.getSeriesTorrentContent(torrent);
const parsedVideos = await Promise.resolve(videos)
.then(videos => videos.filter(video => videos.length === 1 || video.size > this.MIN_SIZE))
.then(videos => parsingService.parseSeriesVideos(torrent, videos))
.then(videos => this.decomposeEpisodes(torrent, videos, metadata))
.then(videos => this.assignKitsuOrImdbEpisodes(torrent, videos, metadata))
.then(videos => Promise.all(videos.map(video => video.isMovie
? this.mapSeriesMovie(video, torrent)
: this.mapSeriesEpisode(video, torrent, videos))))
.then(videos => videos
.map((video: ParsableTorrentFile) => this.isFeaturette(video) ? this.clearInfoFields(video) : video))
return {contents, videos: parsedVideos, subtitles};
}
private async getMoviesTorrentContent(torrent: TorrentInfo) {
const files = await torrentDownloadService.getTorrentFiles(torrent)
.catch(error => {
if (!parsingService.isPackTorrent(torrent)) {
return { videos: [{name: torrent.title, path: torrent.title, size: torrent.size, fileIndex: null}], contents:[], subtitles: []}
}
return Promise.reject(error);
});
if (files.contents && files.contents.length && !files.videos.length && this.isDiskTorrent(files.contents)) {
files.videos = [{name: torrent.title, path: torrent.title, size: torrent.size}];
}
return files;
}
private async getSeriesTorrentContent(torrent: TorrentInfo) {
return torrentDownloadService.getTorrentFiles(torrent)
.catch(error => {
if (!parsingService.isPackTorrent(torrent)) {
return { videos: [{ name: torrent.title, path: torrent.title, size: torrent.size }], subtitles: [], contents: [] }
}
return Promise.reject(error);
});
}
private async mapSeriesEpisode(file: ParsableTorrentFile, torrent: TorrentInfo, files: ParsableTorrentFile[]) : Promise<ParsableTorrentFile> {
if (!file.episodes && !file.episodes) {
if (files.length === 1 || files.some(f => f.episodes || f.episodes) || parse(torrent.title).seasons) {
return Promise.resolve({
infoHash: torrent.infoHash,
fileIndex: file.fileIndex,
title: file.path || file.name,
size: file.size,
imdbId: torrent.imdbId || file.imdbId,
});
}
return Promise.resolve([]);
}
const episodeIndexes = [...(file.episodes || file.episodes).keys()];
return Promise.resolve(episodeIndexes.map((index) => ({
infoHash: torrent.infoHash,
fileIndex: file.fileIndex,
title: file.path || file.name,
size: file.size,
imdbId: file.imdbId || torrent.imdbId,
season: file.season,
episode: file.episodes && file.episodes[index],
episodes: file.episodes,
kitsuId: file.kitsuId || torrent.kitsuId,
})))
}
private async mapSeriesMovie(file: ParsableTorrentFile, torrent: TorrentInfo): Promise<ParsableTorrentFile> {
const kitsuId= torrent.type === TorrentType.ANIME ? await this.findMovieKitsuId(file)
.then(result => {
if (result instanceof Error) {
logger.warn(`Failed to retrieve kitsuId due to error: ${result.message}`);
return undefined;
}
return result;
}): undefined;
const imdbId = !kitsuId ? await this.findMovieImdbId(file) : undefined;
const query: MetaDataQuery = {
id: kitsuId || imdbId,
type: TorrentType.MOVIE
};
const metadataOrError = await metadataService.getMetadata(query);
if (metadataOrError instanceof Error) {
logger.warn(`Failed to retrieve metadata due to error: ${metadataOrError.message}`);
// return default result or throw error, depending on your use case
return [{
infoHash: torrent.infoHash,
fileIndex: file.fileIndex,
title: file.path || file.name,
size: file.size,
imdbId: imdbId,
kitsuId: kitsuId,
episodes: undefined,
imdbSeason: undefined,
imdbEpisode: undefined,
kitsuEpisode: undefined
}];
}
// at this point, TypeScript infers that metadataOrError is actually MetadataResponse
const metadata = metadataOrError;
const hasEpisode = metadata.videos && metadata.videos.length && (file.episode || metadata.videos.length === 1);
const episodeVideo = hasEpisode && metadata.videos[(file.episode || 1) - 1];
return [{
infoHash: torrent.infoHash,
fileIndex: file.fileIndex,
title: file.path || file.name,
size: file.size,
imdbId: metadata.imdbId || imdbId,
kitsuId: metadata.kitsuId || kitsuId,
season: episodeVideo && metadata.imdbId ? episodeVideo.season : undefined,
episode: episodeVideo && metadata.imdbId | metadata.kitsuId ? episodeVideo.episode || episodeVideo.episode : undefined,
}];
}
private async decomposeEpisodes(torrent: TorrentInfo, files: ParsableTorrentFile[], metadata: MetadataResponse = { episodeCount: [] }) {
if (files.every(file => !file.episodes && !file.date)) {
return files;
}
this.preprocessEpisodes(files);
if (torrent.type === TorrentType.ANIME && torrent.kitsuId) {
if (this.needsCinemetaMetadataForAnime(files, metadata)) {
// In some cases anime could be resolved to wrong kitsuId
// because of imdb season naming/absolute per series naming/multiple seasons
// So in these cases we need to fetch cinemeta based metadata and decompose episodes using that
await this.updateToCinemetaMetadata(metadata);
if (files.some(file => Number.isInteger(file.season))) {
// sometimes multi season anime torrents don't include season 1 naming
files
.filter(file => !Number.isInteger(file.season) && file.episodes)
.forEach(file => file.season = 1);
}
} else {
// otherwise for anime type episodes are always absolute and for a single season
files
.filter(file => file.episodes && file.season !== 0)
.forEach(file => file.season = 1);
return files;
}
}
const sortedEpisodes = files
.map(file => !file.isMovie && file.episodes || [])
.reduce((a, b) => a.concat(b), [])
.sort((a, b) => a - b);
if (this.isConcatSeasonAndEpisodeFiles(files, sortedEpisodes, metadata)) {
this.decomposeConcatSeasonAndEpisodeFiles(files, metadata);
} else if (this.isDateEpisodeFiles(files, metadata)) {
this.decomposeDateEpisodeFiles(files, metadata);
} else if (this.isAbsoluteEpisodeFiles(torrent, files, metadata)) {
this.decomposeAbsoluteEpisodeFiles(torrent, files, metadata);
}
// decomposeEpisodeTitleFiles(torrent, files, metadata);
return files;
}
private preprocessEpisodes(files: ParsableTorrentFile[]) {
// reverse special episode naming when they named with 0 episode, ie. S02E00
files
.filter(file => Number.isInteger(file.season) && file.episode === 0)
.forEach(file => {
file.episode = file.season
file.episodes = [file.season]
file.season = 0;
})
}
private isConcatSeasonAndEpisodeFiles(files: ParsableTorrentFile[], sortedEpisodes: number[], metadata: MetadataResponse) {
if (metadata.kitsuId !== undefined) {
// anime does not use this naming scheme in 99% of cases;
return false;
}
// decompose concat season and episode files (ex. 101=S01E01) in case:
// 1. file has a season, but individual files are concatenated with that season (ex. path Season 5/511 - Prize
// Fighters.avi)
// 2. file does not have a season and the episode does not go out of range for the concat season
// episode count
const thresholdAbove = Math.max(Math.ceil(files.length * 0.05), 5);
const thresholdSorted = Math.max(Math.ceil(files.length * 0.8), 8);
const threshold = Math.max(Math.ceil(files.length * 0.8), 5);
const sortedConcatEpisodes = sortedEpisodes
.filter(ep => ep > 100)
.filter(ep => metadata.episodeCount[this.div100(ep) - 1] < ep)
.filter(ep => metadata.episodeCount[this.div100(ep) - 1] >= this.mod100(ep));
const concatFileEpisodes = files
.filter(file => !file.isMovie && file.episodes)
.filter(file => !file.season || file.episodes.every(ep => this.div100(ep) === file.season));
const concatAboveTotalEpisodeCount = files
.filter(file => !file.isMovie && file.episodes && file.episodes.every(ep => ep > 100))
.filter(file => file.episodes.every(ep => ep > metadata.totalCount));
return sortedConcatEpisodes.length >= thresholdSorted && concatFileEpisodes.length >= threshold
|| concatAboveTotalEpisodeCount.length >= thresholdAbove;
}
private isDateEpisodeFiles(files: ParsableTorrentFile[], metadata: MetadataResponse) {
return files.every(file => (!file.season || !metadata.episodeCount[file.season - 1]) && file.date);
}
private isAbsoluteEpisodeFiles(torrent: TorrentInfo, files: ParsableTorrentFile[], metadata: MetadataResponse) {
const threshold = Math.ceil(files.length / 5);
const isAnime = torrent.type === TorrentType.ANIME && torrent.kitsuId;
const nonMovieEpisodes = files
.filter(file => !file.isMovie && file.episodes);
const absoluteEpisodes = files
.filter(file => file.season && file.episodes)
.filter(file => file.episodes.every(ep => metadata.episodeCount[file.season - 1] < ep))
return nonMovieEpisodes.every(file => !file.season)
|| (isAnime && nonMovieEpisodes.every(file => file.season > metadata.episodeCount.length))
|| absoluteEpisodes.length >= threshold;
}
private isNewEpisodeNotInMetadata(torrent: TorrentInfo, file: ParsableTorrentFile, metadata: MetadataResponse) {
// new episode might not yet been indexed by cinemeta.
// detect this if episode number is larger than the last episode or season is larger than the last one
// only for non anime metas
const isAnime = torrent.type === TorrentType.ANIME && torrent.kitsuId;
return !isAnime && !file.isMovie && file.episodes && file.season !== 1
&& /continuing|current/i.test(metadata.status)
&& file.season >= metadata.episodeCount.length
&& file.episodes.every(ep => ep > (metadata.episodeCount[file.season - 1] || 0));
}
private decomposeConcatSeasonAndEpisodeFiles(files: ParsableTorrentFile[], metadata: MetadataResponse) {
files
.filter(file => file.episodes && file.season !== 0 && file.episodes.every(ep => ep > 100))
.filter(file => metadata.episodeCount[(file.season || this.div100(file.episodes[0])) - 1] < 100)
.filter(file => file.season && file.episodes.every(ep => this.div100(ep) === file.season) || !file.season)
.forEach(file => {
file.season = this.div100(file.episodes[0]);
file.episodes = file.episodes.map(ep => this.mod100(ep))
});
}
private decomposeAbsoluteEpisodeFiles(torrent: TorrentInfo, files: ParsableTorrentFile[], metadata: MetadataResponse) {
if (metadata.episodeCount.length === 0) {
files
.filter(file => !Number.isInteger(file.season) && file.episodes && !file.isMovie)
.forEach(file => {
file.season = 1;
});
return;
}
files
.filter(file => file.episodes && !file.isMovie && file.season !== 0)
.filter(file => !this.isNewEpisodeNotInMetadata(torrent, file, metadata))
.filter(file => !file.season || (metadata.episodeCount[file.season - 1] || 0) < file.episodes[0])
.forEach(file => {
const seasonIdx = ([...metadata.episodeCount.keys()]
.find((i) => metadata.episodeCount.slice(0, i + 1).reduce((a, b) => a + b) >= file.episodes[0])
+ 1 || metadata.episodeCount.length) - 1;
file.season = seasonIdx + 1;
file.episodes = file.episodes
.map(ep => ep - metadata.episodeCount.slice(0, seasonIdx).reduce((a, b) => a + b, 0))
});
}
private decomposeDateEpisodeFiles(files: ParsableTorrentFile[], metadata: MetadataResponse) {
if (!metadata || !metadata.videos || !metadata.videos.length) {
return;
}
const timeZoneOffset = this.getTimeZoneOffset(metadata.country);
const offsetVideos = metadata.videos
.reduce((map, video) => {
const releaseDate = moment(video.released).utcOffset(timeZoneOffset).format('YYYY-MM-DD');
map[releaseDate] = video;
return map;
}, {});
files
.filter(file => file.date)
.forEach(file => {
const video = offsetVideos[file.date];
if (video) {
file.season = video.season;
file.episodes = [video.episode];
}
});
}
private getTimeZoneOffset(country: string | undefined) {
switch (country) {
case 'United States':
case 'USA':
return '-08:00';
default:
return '00:00';
}
}
private assignKitsuOrImdbEpisodes(torrent: TorrentInfo, files: ParsableTorrentFile[], metadata: MetadataResponse) {
if (!metadata || !metadata.videos || !metadata.videos.length) {
if (torrent.type === TorrentType.ANIME) {
// assign episodes as kitsu episodes for anime when no metadata available for imdb mapping
files
.filter(file => file.season && file.episodes)
.forEach(file => {
file.season = undefined;
file.episodes = undefined;
})
if (metadata.type === TorrentType.MOVIE && files.every(file => !file.imdbId)) {
// sometimes a movie has episode naming, thus not recognized as a movie and imdbId not assigned
files.forEach(file => file.imdbId = metadata.imdbId);
}
}
return files;
}
const seriesMapping: CommonVideoMetadata = metadata.videos
.reduce((map, video) => {
const episodeMap = map[video.season] || {};
episodeMap[video.episode] = video;
map[video.season] = episodeMap;
return map;
}, {});
if (metadata.videos.some(video => Number.isInteger(video.season)) || !metadata.imdbId) {
files.filter((file => Number.isInteger(file.season) && file.episodes))
.map(file => {
const seasonMapping = seriesMapping[file.season];
const episodeMapping = seasonMapping && seasonMapping[file.episodes[0]];
if (episodeMapping && Number.isInteger(episodeMapping.imdbSeason)) {
file.imdbId = metadata.imdbId;
file.season = episodeMapping.imdbSeason;
file.episodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].imdbEpisode);
} else {
// no imdb mapping available for episode
file.season = undefined;
file.episodes = undefined;
}
});
} else if (metadata.videos.some(video => video.episode)) {
// imdb episode info is base
files
.filter(file => Number.isInteger(file.season) && file.episodes)
.forEach(file => {
if (seriesMapping[file.season]) {
const seasonMapping = seriesMapping[file.season];
file.imdbId = metadata.imdbId;
file.kitsuId = seasonMapping[file.episodes[0]] && seasonMapping[file.episodes[0]].kitsuId;
file.episodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].kitsuEpisode);
} else if (seriesMapping[file.season - 1]) {
// sometimes a second season might be a continuation of the previous season
const seasonMapping = seriesMapping[file.season - 1] as CommonVideoMetadata;
const episodes = Object.values(seasonMapping);
const firstKitsuId = episodes.length && episodes[0];
const differentTitlesCount = new Set(episodes.map(ep => ep.id)).size
const skippedCount = episodes.filter(ep => ep.id === firstKitsuId).length;
const seasonEpisodes = files
.filter(otherFile => otherFile.season === file.season)
.reduce((a, b) => a.episodes.concat(b.episodes), []);
const isAbsoluteOrder = seasonEpisodes.episodes.every(ep => ep > skippedCount && ep <= episodes.length)
const isNormalOrder = seasonEpisodes.episodes.every(ep => ep + skippedCount <= episodes.length)
if (differentTitlesCount >= 1 && (isAbsoluteOrder || isNormalOrder)) {
file.imdbId = metadata.imdbId;
file.season = file.season - 1;
file.episodes = file.episodes.map(ep => isAbsoluteOrder ? ep : ep + skippedCount);
file.kitsuId = seasonMapping[file.episodes[0]].kitsuId;
file.episodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].kitsuEpisode);
}
} else if (Object.values(seriesMapping).length === 1 && seriesMapping[1]) {
// sometimes series might be named with sequel season but it's not a season on imdb and a new title
const seasonMapping = seriesMapping[1];
file.imdbId = metadata.imdbId;
file.season = 1;
file.kitsuId = seasonMapping[file.episodes[0]].kitsuId;
file.episodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].kitsuEpisode);
}
});
}
return files;
}
private needsCinemetaMetadataForAnime(files: ParsableTorrentFile[], metadata: MetadataResponse) {
if (!metadata || !metadata.imdbId || !metadata.videos || !metadata.videos.length) {
return false;
}
const minSeason = Math.min(...metadata.videos.map(video => video.season)) || Number.MAX_VALUE;
const maxSeason = Math.max(...metadata.videos.map(video => video.season)) || Number.MAX_VALUE;
const differentSeasons = new Set(metadata.videos
.map(video => video.season)
.filter(season => Number.isInteger(season))).size;
const total = metadata.totalCount || Number.MAX_VALUE;
return differentSeasons > 1 || files
.filter(file => !file.isMovie && file.episodes)
.some(file => file.season < minSeason || file.season > maxSeason || file.episodes.every(ep => ep > total));
}
private async updateToCinemetaMetadata(metadata: MetadataResponse) {
const query: MetaDataQuery = {
id: metadata.imdbId,
type: metadata.type
};
return await metadataService.getMetadata(query)
.then((newMetadataOrError) => {
if (newMetadataOrError instanceof Error) {
// handle error
logger.warn(`Failed ${metadata.imdbId} metadata cinemeta update due: ${newMetadataOrError.message}`);
return metadata; // or throw newMetadataOrError to propagate error up the call stack
}
// At this point TypeScript infers newMetadataOrError to be of type MetadataResponse
let newMetadata = newMetadataOrError;
if (!newMetadata.videos || !newMetadata.videos.length) {
return metadata;
} else {
metadata.videos = newMetadata.videos;
metadata.episodeCount = newMetadata.episodeCount;
metadata.totalCount = newMetadata.totalCount;
return metadata;
}
})
}
private findMovieImdbId(title: ParseTorrentTitleResult | string) {
const parsedTitle = typeof title === 'string' ? parse(title) : title;
logger.debug(`Finding movie imdbId for ${title}`);
return this.imdb_limiter.schedule(async () => {
const imdbQuery = {
title: parsedTitle.title,
year: parsedTitle.year,
type: TorrentType.MOVIE
};
try {
return await metadataService.getImdbId(imdbQuery);
} catch (e) {
return undefined;
}
});
}
private async findMovieKitsuId(title: ParseTorrentTitleResult | string) {
const parsedTitle = typeof title === 'string' ? parse(title) : title;
const kitsuQuery = {
title: parsedTitle.title,
year: parsedTitle.year,
season: parsedTitle.season,
type: TorrentType.MOVIE
};
try {
return await metadataService.getKitsuId(kitsuQuery);
} catch (e) {
return undefined;
}
}
private isDiskTorrent(contents: ParsableTorrentFile[]) {
return contents.some(content => extensionService.isDisk(content.path));
}
private isSingleMovie(videos: ParsableTorrentFile[]) {
return videos.length === 1 ||
(videos.length === 2 &&
videos.find(v => /\b(?:part|disc|cd)[ ._-]?0?1\b|^0?1\.\w{2,4}$/i.test(v.path)) &&
videos.find(v => /\b(?:part|disc|cd)[ ._-]?0?2\b|^0?2\.\w{2,4}$/i.test(v.path)));
}
private isFeaturette(video: ParsableTorrentFile) {
return /featurettes?\/|extras-grym/i.test(video.path);
}
private clearInfoFields(video: ParsableTorrentFile) {
video.imdbId = undefined;
video.season = undefined;
video.episode = undefined;
video.kitsuId = undefined;
return video;
}
private div100(episode: number) {
return (episode / 100 >> 0); // floor to nearest int
}
private mod100(episode: number) {
return episode % 100;
}
}
export const torrentFileService = new TorrentFileService();

88
src/node/consumer/src/lib/services/torrent_subtitle_service.ts Normal file
View File

@@ -0,0 +1,88 @@
import { parse } from 'parse-torrent-title';
class TorrentSubtitleService {
public assignSubtitles(contents: any, videos: any[], subtitles: any[]) {
if (videos && videos.length && subtitles && subtitles.length) {
if (videos.length === 1) {
videos[0].subtitles = subtitles;
return { contents, videos, subtitles: [] };
}
const parsedVideos = videos.map(video => this.parseVideo(video));
const assignedSubs = subtitles.map(subtitle => ({ subtitle, videos: this.mostProbableSubtitleVideos(subtitle, parsedVideos) }));
const unassignedSubs = assignedSubs.filter(assignedSub => !assignedSub.videos).map(assignedSub => assignedSub.subtitle);
assignedSubs
.filter(assignedSub => assignedSub.videos)
.forEach(assignedSub => assignedSub.videos.forEach(video => video.subtitles = (video.subtitles || []).concat(assignedSub.subtitle)));
return { contents, videos, subtitles: unassignedSubs };
}
return { contents, videos, subtitles };
}
private parseVideo(video: any) {
const fileName = video.title.split('/').pop().replace(/\.(\w{2,4})$/, '');
const folderName = video.title.replace(/\/?[^/]+$/, '');
return {
videoFile: video,
fileName: fileName,
folderName: folderName,
...this.parseFilename(video.title)
};
}
private mostProbableSubtitleVideos(subtitle: any, parsedVideos: any[]) {
const subTitle = (subtitle.title || subtitle.path).split('/').pop().replace(/\.(\w{2,4})$/, '');
const parsedSub = this.parsePath(subtitle.title || subtitle.path);
const byFileName = parsedVideos.filter(video => subTitle.includes(video.fileName));
if (byFileName.length === 1) {
return byFileName.map(v => v.videoFile);
}
const byTitleSeasonEpisode = parsedVideos.filter(video => video.title === parsedSub.title
&& this.arrayEquals(video.seasons, parsedSub.seasons)
&& this.arrayEquals(video.episodes, parsedSub.episodes));
if (this.singleVideoFile(byTitleSeasonEpisode)) {
return byTitleSeasonEpisode.map(v => v.videoFile);
}
const bySeasonEpisode = parsedVideos.filter(video => this.arrayEquals(video.seasons, parsedSub.seasons)
&& this.arrayEquals(video.episodes, parsedSub.episodes));
if (this.singleVideoFile(bySeasonEpisode)) {
return bySeasonEpisode.map(v => v.videoFile);
}
const byTitle = parsedVideos.filter(video => video.title && video.title === parsedSub.title);
if (this.singleVideoFile(byTitle)) {
return byTitle.map(v => v.videoFile);
}
const byEpisode = parsedVideos.filter(video => this.arrayEquals(video.episodes, parsedSub.episodes));
if (this.singleVideoFile(byEpisode)) {
return byEpisode.map(v => v.videoFile);
}
return undefined;
}
private singleVideoFile(videos: any[]) {
return new Set(videos.map(v => v.videoFile.fileIndex)).size === 1;
}
private parsePath(path: string) {
const pathParts = path.split('/').map(part => this.parseFilename(part));
const parsedWithEpisode = pathParts.find(parsed => parsed.season && parsed.episodes);
return parsedWithEpisode || pathParts[pathParts.length - 1];
}
private parseFilename(filename: string) {
const parsedInfo = parse(filename)
const titleEpisode = parsedInfo.title.match(/(\d+)$/);
if (!parsedInfo.episodes && titleEpisode) {
parsedInfo.episodes = [parseInt(titleEpisode[1], 10)];
}
return parsedInfo;
}
private arrayEquals(array1: any[], array2: any[]) {
if (!array1 || !array2) return array1 === array2;
return array1.length === array2.length && array1.every((value, index) => value === array2[index])
}
}
export const torrentSubtitleService = new TorrentSubtitleService();

12
src/node/consumer/src/lib/torrentEntries.js
View File

@@ -3,8 +3,8 @@ import { metadataService } from './services/metadata_service';
import { parsingService } from './services/parsing_service'; import { parsingService } from './services/parsing_service';
import {PromiseHelpers} from './helpers/promises_helpers'; import {PromiseHelpers} from './helpers/promises_helpers';
import { repository } from '../repository/database_repository'; import { repository } from '../repository/database_repository';
import { parseTorrentFiles } from './torrentFiles.js'; import { torrentFileService } from './services/torrent_file_service';
import { assignSubtitles } from './torrentSubtitles.js'; import { torrentSubtitleService } from './services/torrent_subtitle_service';
import { TorrentType } from './enums/torrent_types'; import { TorrentType } from './enums/torrent_types';
import {logger} from './services/logging_service'; import {logger} from './services/logging_service';
@@ -43,9 +43,9 @@ export async function createTorrentEntry(torrent, overwrite = false) {
return; return;
} }
const { contents, videos, subtitles } = await parseTorrentFiles(torrent) const { contents, videos, subtitles } = await torrentFileService.parseTorrentFiles(torrent)
.then(torrentContents => overwrite ? overwriteExistingFiles(torrent, torrentContents) : torrentContents) .then(torrentContents => overwrite ? overwriteExistingFiles(torrent, torrentContents) : torrentContents)
.then(torrentContents => assignSubtitles(torrentContents)) .then(torrentContents => torrentSubtitleService.assignSubtitles(torrentContents))
.catch(error => { .catch(error => {
logger.warn(`Failed getting files for ${torrent.title}`, error.message); logger.warn(`Failed getting files for ${torrent.title}`, error.message);
return {}; return {};
@@ -135,9 +135,9 @@ export async function createTorrentContents(torrent) {
const imdbId = PromiseHelpers.mostCommonValue(storedVideos.map(stored => stored.imdbId)); const imdbId = PromiseHelpers.mostCommonValue(storedVideos.map(stored => stored.imdbId));
const kitsuId = PromiseHelpers.mostCommonValue(storedVideos.map(stored => stored.kitsuId)); const kitsuId = PromiseHelpers.mostCommonValue(storedVideos.map(stored => stored.kitsuId));
const { contents, videos, subtitles } = await parseTorrentFiles({ ...torrent, imdbId, kitsuId }) const { contents, videos, subtitles } = await torrentFileService.parseTorrentFiles({ ...torrent, imdbId, kitsuId })
.then(torrentContents => notOpenedVideo ? torrentContents : { ...torrentContents, videos: storedVideos }) .then(torrentContents => notOpenedVideo ? torrentContents : { ...torrentContents, videos: storedVideos })
.then(torrentContents => assignSubtitles(torrentContents)) .then(torrentContents => torrentSubtitleService.assignSubtitles(torrentContents))
.catch(error => { .catch(error => {
logger.warn(`Failed getting contents for [${torrent.infoHash}] ${torrent.title}`, error.message); logger.warn(`Failed getting contents for [${torrent.infoHash}] ${torrent.title}`, error.message);
return {}; return {};

526
src/node/consumer/src/lib/torrentFiles.js
View File

@@ -1,526 +0,0 @@
import Bottleneck from 'bottleneck';
import distance from 'jaro-winkler';
import moment from 'moment';
import { parse } from 'parse-torrent-title';
import { configurationService } from './services/configuration_service';
import { extensionService } from './services/extension_service';
import { metadataService } from './services/metadata_service';
import { parsingService } from './services/parsing_service';
import {PromiseHelpers} from './helpers/promises_helpers';
import {torrentDownloadService} from "./services/torrent_download_service";
import { TorrentType } from './enums/torrent_types';
import {logger} from "./services/logging_service";
const MIN_SIZE = 5 * 1024 * 1024; // 5 MB
const imdb_limiter = new Bottleneck({ maxConcurrent: configurationService.metadataConfig.IMDB_CONCURRENT, minTime: configurationService.metadataConfig.IMDB_INTERVAL_MS });
export async function parseTorrentFiles(torrent) {
const parsedTorrentName = parse(torrent.title);
const metadata = await metadataService.getMetadata(torrent.kitsuId || torrent.imdbId, torrent.type || TorrentType.MOVIE)
.then(meta => Object.assign({}, meta))
.catch(() => undefined);
// if (metadata && metadata.type !== torrent.type && torrent.type !== Type.ANIME) {
// throw new Error(`Mismatching entry type for ${torrent.name}: ${torrent.type}!=${metadata.type}`);
// }
if (torrent.type !== TorrentType.ANIME && metadata && metadata.type && metadata.type !== torrent.type) {
// it's actually a movie/series
torrent.type = metadata.type;
}
if (torrent.type === TorrentType.MOVIE && (!parsedTorrentName.seasons ||
parsedTorrentName.season === 5 && [1, 5].includes(parsedTorrentName.episode))) {
return parseMovieFiles(torrent, parsedTorrentName, metadata);
}
return parseSeriesFiles(torrent, parsedTorrentName, metadata)
}
async function parseMovieFiles(torrent, parsedName, metadata) {
const { contents, videos, subtitles } = await getMoviesTorrentContent(torrent);
const filteredVideos = videos
.filter(video => video.size > MIN_SIZE)
.filter(video => !isFeaturette(video));
if (isSingleMovie(filteredVideos)) {
const parsedVideos = filteredVideos.map(video => ({
infoHash: torrent.infoHash,
fileIndex: video.fileIndex,
title: video.path || torrent.title,
size: video.size || torrent.size,
imdbId: torrent.imdbId || metadata && metadata.imdbId,
kitsuId: torrent.kitsuId || metadata && metadata.kitsuId
}));
return { contents, videos: parsedVideos, subtitles };
}
const parsedVideos = await PromiseHelpers.sequence(filteredVideos.map(video => () => isFeaturette(video)
? Promise.resolve(video)
: findMovieImdbId(video.name).then(imdbId => ({ ...video, imdbId }))))
.then(videos => videos.map(video => ({
infoHash: torrent.infoHash,
fileIndex: video.fileIndex,
title: video.path || video.name,
size: video.size,
imdbId: video.imdbId,
})));
return { contents, videos: parsedVideos, subtitles };
}
async function parseSeriesFiles(torrent, parsedName, metadata) {
const { contents, videos, subtitles } = await getSeriesTorrentContent(torrent);
const parsedVideos = await Promise.resolve(videos)
.then(videos => videos.filter(video => videos.length === 1 || video.size > MIN_SIZE))
.then(videos => parsingService.parseSeriesVideos(torrent, videos))
.then(videos => decomposeEpisodes(torrent, videos, metadata))
.then(videos => assignKitsuOrImdbEpisodes(torrent, videos, metadata))
.then(videos => Promise.all(videos.map(video => video.isMovie
? mapSeriesMovie(video, torrent)
: mapSeriesEpisode(video, torrent, videos))))
.then(videos => videos
.reduce((a, b) => a.concat(b), [])
.map(video => isFeaturette(video) ? clearInfoFields(video) : video))
return { contents, videos: parsedVideos, subtitles };
}
async function getMoviesTorrentContent(torrent) {
const files = await torrentFiles(torrent)
.catch(error => {
if (!isPackTorrent(torrent)) {
return { videos: [{ name: torrent.title, path: torrent.title, size: torrent.size }] }
}
return Promise.reject(error);
});
if (files.contents && files.contents.length && !files.videos.length && isDiskTorrent(files.contents)) {
files.videos = [{ name: torrent.title, path: torrent.title, size: torrent.size }];
}
return files;
}
async function getSeriesTorrentContent(torrent) {
return torrentDownloadService.getTorrentFiles(torrent)
.catch(error => {
if (!parsingService.isPackTorrent(torrent)) {
return { videos: [{ name: torrent.title, path: torrent.title, size: torrent.size }] }
}
return Promise.reject(error);
});
}
async function mapSeriesEpisode(file, torrent, files) {
if (!file.episodes && !file.kitsuEpisodes) {
if (files.length === 1 || files.some(f => f.episodes || f.kitsuEpisodes) || parse(torrent.title).seasons) {
return Promise.resolve({
infoHash: torrent.infoHash,
fileIndex: file.fileIndex,
title: file.path || file.name,
size: file.size,
imdbId: torrent.imdbId || file.imdbId,
});
}
return Promise.resolve([]);
}
const episodeIndexes = [...(file.episodes || file.kitsuEpisodes).keys()];
return Promise.resolve(episodeIndexes.map((index) => ({
infoHash: torrent.infoHash,
fileIndex: file.fileIndex,
title: file.path || file.name,
size: file.size,
imdbId: file.imdbId || torrent.imdbId,
imdbSeason: file.season,
imdbEpisode: file.episodes && file.episodes[index],
kitsuId: file.kitsuId || torrent.kitsuId,
kitsuEpisode: file.kitsuEpisodes && file.kitsuEpisodes[index]
})))
}
async function mapSeriesMovie(file, torrent) {
const kitsuId = torrent.type === TorrentType.ANIME ? await findMovieKitsuId(file) : undefined;
const imdbId = !kitsuId ? await findMovieImdbId(file) : undefined;
const metadata = await metadataService.getMetadata(kitsuId || imdbId, TorrentType.MOVIE).catch(() => ({}));
const hasEpisode = metadata.videos && metadata.videos.length && (file.episode || metadata.videos.length === 1);
const episodeVideo = hasEpisode && metadata.videos[(file.episode || 1) - 1];
return [{
infoHash: torrent.infoHash,
fileIndex: file.fileIndex,
title: file.path || file.name,
size: file.size,
imdbId: metadata.imdbId || imdbId,
kitsuId: metadata.kitsuId || kitsuId,
imdbSeason: episodeVideo && metadata.imdbId ? episodeVideo.imdbSeason : undefined,
imdbEpisode: episodeVideo && metadata.imdbId ? episodeVideo.imdbEpisode || episodeVideo.episode : undefined,
kitsuEpisode: episodeVideo && metadata.kitsuId ? episodeVideo.kitsuEpisode || episodeVideo.episode : undefined
}];
}
async function decomposeEpisodes(torrent, files, metadata = { episodeCount: [] }) {
if (files.every(file => !file.episodes && !file.date)) {
return files;
}
preprocessEpisodes(files);
if (torrent.type === TorrentType.ANIME && torrent.kitsuId) {
if (needsCinemetaMetadataForAnime(files, metadata)) {
// In some cases anime could be resolved to wrong kitsuId
// because of imdb season naming/absolute per series naming/multiple seasons
// So in these cases we need to fetch cinemeta based metadata and decompose episodes using that
await updateToCinemetaMetadata(metadata);
if (files.some(file => Number.isInteger(file.season))) {
// sometimes multi season anime torrents don't include season 1 naming
files
.filter(file => !Number.isInteger(file.season) && file.episodes)
.forEach(file => file.season = 1);
}
} else {
// otherwise for anime type episodes are always absolute and for a single season
files
.filter(file => file.episodes && file.season !== 0)
.forEach(file => file.season = 1);
return files;
}
}
const sortedEpisodes = files
.map(file => !file.isMovie && file.episodes || [])
.reduce((a, b) => a.concat(b), [])
.sort((a, b) => a - b);
if (isConcatSeasonAndEpisodeFiles(files, sortedEpisodes, metadata)) {
decomposeConcatSeasonAndEpisodeFiles(torrent, files, metadata);
} else if (isDateEpisodeFiles(files, metadata)) {
decomposeDateEpisodeFiles(torrent, files, metadata);
} else if (isAbsoluteEpisodeFiles(torrent, files, metadata)) {
decomposeAbsoluteEpisodeFiles(torrent, files, metadata);
}
// decomposeEpisodeTitleFiles(torrent, files, metadata);
return files;
}
function preprocessEpisodes(files) {
// reverse special episode naming when they named with 0 episode, ie. S02E00
files
.filter(file => Number.isInteger(file.season) && file.episode === 0)
.forEach(file => {
file.episode = file.season
file.episodes = [file.season]
file.season = 0;
})
}
function isConcatSeasonAndEpisodeFiles(files, sortedEpisodes, metadata) {
if (metadata.kitsuId !== undefined) {
// anime does not use this naming scheme in 99% of cases;
return false;
}
// decompose concat season and episode files (ex. 101=S01E01) in case:
// 1. file has a season, but individual files are concatenated with that season (ex. path Season 5/511 - Prize
// Fighters.avi)
// 2. file does not have a season and the episode does not go out of range for the concat season
// episode count
const thresholdAbove = Math.max(Math.ceil(files.length * 0.05), 5);
const thresholdSorted = Math.max(Math.ceil(files.length * 0.8), 8);
const threshold = Math.max(Math.ceil(files.length * 0.8), 5);
const sortedConcatEpisodes = sortedEpisodes
.filter(ep => ep > 100)
.filter(ep => metadata.episodeCount[div100(ep) - 1] < ep)
.filter(ep => metadata.episodeCount[div100(ep) - 1] >= mod100(ep));
const concatFileEpisodes = files
.filter(file => !file.isMovie && file.episodes)
.filter(file => !file.season || file.episodes.every(ep => div100(ep) === file.season));
const concatAboveTotalEpisodeCount = files
.filter(file => !file.isMovie && file.episodes && file.episodes.every(ep => ep > 100))
.filter(file => file.episodes.every(ep => ep > metadata.totalCount));
return sortedConcatEpisodes.length >= thresholdSorted && concatFileEpisodes.length >= threshold
|| concatAboveTotalEpisodeCount.length >= thresholdAbove;
}
function isDateEpisodeFiles(files, metadata) {
return files.every(file => (!file.season || !metadata.episodeCount[file.season - 1]) && file.date);
}
function isAbsoluteEpisodeFiles(torrent, files, metadata) {
const threshold = Math.ceil(files.length / 5);
const isAnime = torrent.type === TorrentType.ANIME && torrent.kitsuId;
const nonMovieEpisodes = files
.filter(file => !file.isMovie && file.episodes);
const absoluteEpisodes = files
.filter(file => file.season && file.episodes)
.filter(file => file.episodes.every(ep => metadata.episodeCount[file.season - 1] < ep))
return nonMovieEpisodes.every(file => !file.season)
|| (isAnime && nonMovieEpisodes.every(file => file.season > metadata.episodeCount.length))
|| absoluteEpisodes.length >= threshold;
}
function isNewEpisodeNotInMetadata(torrent, file, metadata) {
// new episode might not yet been indexed by cinemeta.
// detect this if episode number is larger than the last episode or season is larger than the last one
// only for non anime metas
const isAnime = torrent.type === TorrentType.ANIME && torrent.kitsuId;
return !isAnime && !file.isMovie && file.episodes && file.season !== 1
&& /continuing|current/i.test(metadata.status)
&& file.season >= metadata.episodeCount.length
&& file.episodes.every(ep => ep > (metadata.episodeCount[file.season - 1] || 0));
}
function decomposeConcatSeasonAndEpisodeFiles(torrent, files, metadata) {
files
.filter(file => file.episodes && file.season !== 0 && file.episodes.every(ep => ep > 100))
.filter(file => metadata.episodeCount[(file.season || div100(file.episodes[0])) - 1] < 100)
.filter(file => file.season && file.episodes.every(ep => div100(ep) === file.season) || !file.season)
.forEach(file => {
file.season = div100(file.episodes[0]);
file.episodes = file.episodes.map(ep => mod100(ep))
});
}
function decomposeAbsoluteEpisodeFiles(torrent, files, metadata) {
if (metadata.episodeCount.length === 0) {
files
.filter(file => !Number.isInteger(file.season) && file.episodes && !file.isMovie)
.forEach(file => {
file.season = 1;
});
return;
}
files
.filter(file => file.episodes && !file.isMovie && file.season !== 0)
.filter(file => !isNewEpisodeNotInMetadata(torrent, file, metadata))
.filter(file => !file.season || (metadata.episodeCount[file.season - 1] || 0) < file.episodes[0])
.forEach(file => {
const seasonIdx = ([...metadata.episodeCount.keys()]
.find((i) => metadata.episodeCount.slice(0, i + 1).reduce((a, b) => a + b) >= file.episodes[0])
+ 1 || metadata.episodeCount.length) - 1;
file.season = seasonIdx + 1;
file.episodes = file.episodes
.map(ep => ep - metadata.episodeCount.slice(0, seasonIdx).reduce((a, b) => a + b, 0))
});
}
function decomposeDateEpisodeFiles(torrent, files, metadata) {
if (!metadata || !metadata.videos || !metadata.videos.length) {
return;
}
const timeZoneOffset = getTimeZoneOffset(metadata.country);
const offsetVideos = metadata.videos
.reduce((map, video) => {
const releaseDate = moment(video.released).utcOffset(timeZoneOffset).format('YYYY-MM-DD');
map[releaseDate] = video;
return map;
}, {});
files
.filter(file => file.date)
.forEach(file => {
const video = offsetVideos[file.date];
if (video) {
file.season = video.season;
file.episodes = [video.episode];
}
});
}
/* eslint-disable no-unused-vars */
function decomposeEpisodeTitleFiles(torrent, files, metadata) {
files
// .filter(file => !file.season)
.map(file => {
const episodeTitle = file.name.replace('_', ' ')
.replace(/^.*(?:E\d+[abc]?|- )\s?(.+)\.\w{1,4}$/, '$1')
.trim();
const foundEpisode = metadata.videos
.map(video => ({ ...video, distance: distance(episodeTitle, video.name) }))
.sort((a, b) => b.distance - a.distance)[0];
if (foundEpisode) {
file.isMovie = false;
file.season = foundEpisode.season;
file.episodes = [foundEpisode.episode];
}
})
}
/* eslint-enable no-unused-vars */
function getTimeZoneOffset(country) {
switch (country) {
case 'United States':
case 'USA':
return '-08:00';
default:
return '00:00';
}
}
function assignKitsuOrImdbEpisodes(torrent, files, metadata) {
if (!metadata || !metadata.videos || !metadata.videos.length) {
if (torrent.type === TorrentType.ANIME) {
// assign episodes as kitsu episodes for anime when no metadata available for imdb mapping
files
.filter(file => file.season && file.episodes)
.forEach(file => {
file.kitsuEpisodes = file.episodes;
file.season = undefined;
file.episodes = undefined;
})
if (metadata.type === TorrentType.MOVIE && files.every(file => !file.imdbId)) {
// sometimes a movie has episode naming, thus not recognized as a movie and imdbId not assigned
files.forEach(file => file.imdbId = metadata.imdbId);
}
}
return files;
}
const seriesMapping = metadata.videos
.reduce((map, video) => {
const episodeMap = map[video.season] || {};
episodeMap[video.episode] = video;
map[video.season] = episodeMap;
return map;
}, {});
if (metadata.videos.some(video => Number.isInteger(video.imdbSeason)) || !metadata.imdbId) {
// kitsu episode info is the base
files
.filter(file => Number.isInteger(file.season) && file.episodes)
.map(file => {
const seasonMapping = seriesMapping[file.season];
const episodeMapping = seasonMapping && seasonMapping[file.episodes[0]];
file.kitsuEpisodes = file.episodes;
if (episodeMapping && Number.isInteger(episodeMapping.imdbSeason)) {
file.imdbId = metadata.imdbId;
file.season = episodeMapping.imdbSeason;
file.episodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].imdbEpisode);
} else {
// no imdb mapping available for episode
file.season = undefined;
file.episodes = undefined;
}
});
} else if (metadata.videos.some(video => video.kitsuEpisode)) {
// imdb episode info is base
files
.filter(file => Number.isInteger(file.season) && file.episodes)
.forEach(file => {
if (seriesMapping[file.season]) {
const seasonMapping = seriesMapping[file.season];
file.imdbId = metadata.imdbId;
file.kitsuId = seasonMapping[file.episodes[0]] && seasonMapping[file.episodes[0]].kitsuId;
file.kitsuEpisodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].kitsuEpisode);
} else if (seriesMapping[file.season - 1]) {
// sometimes a second season might be a continuation of the previous season
const seasonMapping = seriesMapping[file.season - 1];
const episodes = Object.values(seasonMapping);
const firstKitsuId = episodes.length && episodes[0].kitsuId;
const differentTitlesCount = new Set(episodes.map(ep => ep.kitsuId)).size
const skippedCount = episodes.filter(ep => ep.kitsuId === firstKitsuId).length;
const seasonEpisodes = files
.filter(otherFile => otherFile.season === file.season)
.reduce((a, b) => a.concat(b.episodes), []);
const isAbsoluteOrder = seasonEpisodes.every(ep => ep > skippedCount && ep <= episodes.length)
const isNormalOrder = seasonEpisodes.every(ep => ep + skippedCount <= episodes.length)
if (differentTitlesCount >= 1 && (isAbsoluteOrder || isNormalOrder)) {
file.imdbId = metadata.imdbId;
file.season = file.season - 1;
file.episodes = file.episodes.map(ep => isAbsoluteOrder ? ep : ep + skippedCount);
file.kitsuId = seasonMapping[file.episodes[0]].kitsuId;
file.kitsuEpisodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].kitsuEpisode);
}
} else if (Object.values(seriesMapping).length === 1 && seriesMapping[1]) {
// sometimes series might be named with sequel season but it's not a season on imdb and a new title
const seasonMapping = seriesMapping[1];
file.imdbId = metadata.imdbId;
file.season = 1;
file.kitsuId = seasonMapping[file.episodes[0]].kitsuId;
file.kitsuEpisodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].kitsuEpisode);
}
});
}
return files;
}
function needsCinemetaMetadataForAnime(files, metadata) {
if (!metadata || !metadata.imdbId || !metadata.videos || !metadata.videos.length) {
return false;
}
const minSeason = Math.min(...metadata.videos.map(video => video.imdbSeason)) || Number.MAX_VALUE;
const maxSeason = Math.max(...metadata.videos.map(video => video.imdbSeason)) || Number.MAX_VALUE;
const differentSeasons = new Set(metadata.videos
.map(video => video.imdbSeason)
.filter(season => Number.isInteger(season))).size;
const total = metadata.totalCount || Number.MAX_VALUE;
return differentSeasons > 1 || files
.filter(file => !file.isMovie && file.episodes)
.some(file => file.season < minSeason || file.season > maxSeason || file.episodes.every(ep => ep > total));
}
async function updateToCinemetaMetadata(metadata) {
return metadataService.getMetadata(metadata.imdbId, metadata.type)
.then(newMetadata => !newMetadata.videos || !newMetadata.videos.length ? metadata : newMetadata)
.then(newMetadata => {
metadata.videos = newMetadata.videos;
metadata.episodeCount = newMetadata.episodeCount;
metadata.totalCount = newMetadata.totalCount;
return metadata;
})
.catch(error => logger.warn(`Failed ${metadata.imdbId} metadata cinemeta update due: ${error.message}`));
}
function findMovieImdbId(title) {
const parsedTitle = typeof title === 'string' ? parse(title) : title;
logger.debug(`Finding movie imdbId for ${title}`);
return imdb_limiter.schedule(() => {
const imdbQuery = {
title: parsedTitle.title,
year: parsedTitle.year,
type: TorrentType.MOVIE
};
return metadataService.getImdbId(imdbQuery).catch(() => undefined);
});
}
function findMovieKitsuId(title) {
const parsedTitle = typeof title === 'string' ? parse(title) : title;
const kitsuQuery = {
title: parsedTitle.title,
year: parsedTitle.year,
season: parsedTitle.season,
type: TorrentType.MOVIE
};
return metadataService.getKitsuId(kitsuQuery).catch(() => undefined);
}
function isDiskTorrent(contents) {
return contents.some(content => extensionService.isDisk(content.path));
}
function isSingleMovie(videos) {
return videos.length === 1 ||
(videos.length === 2 &&
videos.find(v => /\b(?:part|disc|cd)[ ._-]?0?1\b|^0?1\.\w{2,4}$/i.test(v.path)) &&
videos.find(v => /\b(?:part|disc|cd)[ ._-]?0?2\b|^0?2\.\w{2,4}$/i.test(v.path)));
}
function isFeaturette(video) {
return /featurettes?\/|extras-grym/i.test(video.path);
}
function clearInfoFields(video) {
video.imdbId = undefined;
video.imdbSeason = undefined;
video.imdbEpisode = undefined;
video.kitsuId = undefined;
video.kitsuEpisode = undefined;
return video;
}
function div100(episode) {
return (episode / 100 >> 0); // floor to nearest int
}
function mod100(episode) {
return episode % 100;
}

89
src/node/consumer/src/lib/torrentSubtitles.js
View File

@@ -1,89 +0,0 @@
import { parse } from 'parse-torrent-title';
export function assignSubtitles({ contents, videos, subtitles }) {
if (videos && videos.length && subtitles && subtitles.length) {
if (videos.length === 1) {
videos[0].subtitles = subtitles;
return { contents, videos, subtitles: [] };
}
const parsedVideos = videos
.map(video => _parseVideo(video));
const assignedSubs = subtitles
.map(subtitle => ({ subtitle, videos: _mostProbableSubtitleVideos(subtitle, parsedVideos) }));
const unassignedSubs = assignedSubs
.filter(assignedSub => !assignedSub.videos)
.map(assignedSub => assignedSub.subtitle);
assignedSubs
.filter(assignedSub => assignedSub.videos)
.forEach(assignedSub => assignedSub.videos
.forEach(video => video.subtitles = (video.subtitles || []).concat(assignedSub.subtitle)));
return { contents, videos, subtitles: unassignedSubs };
}
return { contents, videos, subtitles };
}
function _parseVideo(video) {
const fileName = video.title.split('/').pop().replace(/\.(\w{2,4})$/, '');
const folderName = video.title.replace(/\/?[^/]+$/, '');
return {
videoFile: video,
fileName: fileName,
folderName: folderName,
...parseFilename(video.title)
};
}
function _mostProbableSubtitleVideos(subtitle, parsedVideos) {
const subTitle = (subtitle.title || subtitle.path).split('/').pop().replace(/\.(\w{2,4})$/, '');
const parsedSub = parsePath(subtitle.title || subtitle.path);
const byFileName = parsedVideos.filter(video => subTitle.includes(video.fileName));
if (byFileName.length === 1) {
return byFileName.map(v => v.videoFile);
}
const byTitleSeasonEpisode = parsedVideos.filter(video => video.title === parsedSub.title
&& arrayEquals(video.seasons, parsedSub.seasons)
&& arrayEquals(video.episodes, parsedSub.episodes));
if (singleVideoFile(byTitleSeasonEpisode)) {
return byTitleSeasonEpisode.map(v => v.videoFile);
}
const bySeasonEpisode = parsedVideos.filter(video => arrayEquals(video.seasons, parsedSub.seasons)
&& arrayEquals(video.episodes, parsedSub.episodes));
if (singleVideoFile(bySeasonEpisode)) {
return bySeasonEpisode.map(v => v.videoFile);
}
const byTitle = parsedVideos.filter(video => video.title && video.title === parsedSub.title);
if (singleVideoFile(byTitle)) {
return byTitle.map(v => v.videoFile);
}
const byEpisode = parsedVideos.filter(video => arrayEquals(video.episodes, parsedSub.episodes));
if (singleVideoFile(byEpisode)) {
return byEpisode.map(v => v.videoFile);
}
return undefined;
}
function singleVideoFile(videos) {
return new Set(videos.map(v => v.videoFile.fileIndex)).size === 1;
}
function parsePath(path) {
const pathParts = path.split('/').map(part => parseFilename(part));
const parsedWithEpisode = pathParts.find(parsed => parsed.season && parsed.episodes);
return parsedWithEpisode || pathParts[pathParts.length - 1];
}
function parseFilename(filename) {
const parsedInfo = parse(filename)
const titleEpisode = parsedInfo.title.match(/(\d+)$/);
if (!parsedInfo.episodes && titleEpisode) {
parsedInfo.episodes = [parseInt(titleEpisode[1], 10)];
}
return parsedInfo;
}
function arrayEquals(array1, array2) {
if (!array1 || !array2) return array1 === array2;
return array1.length === array2.length && array1.every((value, index) => value === array2[index])
}