brock/torrentio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wip torrent_file_service, not completed yet

Browse Source
This commit is contained in:
iPromKnight
2024-02-06 15:29:40 +00:00
committed by iPromKnight
parent dcf62fe651
commit 555097d40b
14 changed files with 592 additions and 502 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/node/consumer/src/lib/interfaces/parsable_torrent_file.ts
View File

@@ -1,12 +0,0 @@
import {ParseTorrentTitleResult} from "./parse_torrent_title_result";
export interface ParsableTorrentFile extends ParseTorrentTitleResult {
name?: string;
path?: string;
size?: number;
length?: number;
fileIndex?: number;
isMovie?: boolean;
imdbId?: string | number;
kitsuId?: number | string;
}

18
src/node/consumer/src/lib/interfaces/parsed_torrent.ts Normal file
View File

@@ -0,0 +1,18 @@
import {ParseTorrentTitleResult} from "./parse_torrent_title_result";
import {TorrentType} from "../enums/torrent_types";
import {TorrentFileCollection} from "./torrent_file_collection";
export interface ParsedTorrent extends ParseTorrentTitleResult {
size?: number;
isPack?: boolean;
imdbId?: string | number;
kitsuId?: string | number;
trackers?: string;
provider?: string | null;
infoHash: string | null;
type: string | TorrentType;
uploadDate?: Date;
seeders?: number;
torrentId?: string;
fileCollection?: TorrentFileCollection;
}

10
src/node/consumer/src/lib/interfaces/torrent_file_collection.ts
View File

@@ -1,7 +1,9 @@
import {ParsableTorrentFile} from "./parsable_torrent_file";
import {ContentAttributes} from "../../repository/interfaces/content_attributes";
import {FileAttributes} from "../../repository/interfaces/file_attributes";
import {SubtitleAttributes} from "../../repository/interfaces/subtitle_attributes";
export interface TorrentFileCollection {
contents?: ParsableTorrentFile[];
videos?: ParsableTorrentFile[];
subtitles?: ParsableTorrentFile[];
contents?: ContentAttributes[];
videos?: FileAttributes[];
subtitles?: SubtitleAttributes[];
}

16
src/node/consumer/src/lib/interfaces/torrent_info.ts
View File

@@ -1,16 +0,0 @@
import {TorrentType} from "../enums/torrent_types";
export interface TorrentInfo {
title: string | null;
torrentId: string;
infoHash: string | null;
seeders: number;
uploadDate: Date;
imdbId?: string;
kitsuId?: string;
type: TorrentType;
provider?: string | null;
trackers: string;
size?: number;
pack?: boolean;
}

106
src/node/consumer/src/lib/services/parsing_service.ts
View File

@@ -1,106 +0,0 @@
import { parse } from 'parse-torrent-title';
import { TorrentType } from '../enums/torrent_types';
import {ParseTorrentTitleResult} from "../interfaces/parse_torrent_title_result";
import {ParsableTorrentFile} from "../interfaces/parsable_torrent_file";
import {TorrentInfo} from "../interfaces/torrent_info";
class ParsingService {
private readonly MULTIPLE_FILES_SIZE = 4 * 1024 * 1024 * 1024; // 4 GB
public parseSeriesVideos(torrent: TorrentInfo, videos: ParsableTorrentFile[]): ParsableTorrentFile[] {
const parsedTorrentName = parse(torrent.title);
const hasMovies = parsedTorrentName.complete || !!torrent.title.match(/movies?(?:\W|$)/i);
const parsedVideos = videos.map(video => this.parseSeriesVideo(video, parsedTorrentName));
return parsedVideos.map(video => ({ ...video, isMovie: this.isMovieVideo(video, parsedVideos, torrent.type, hasMovies) }));
}
public isPackTorrent(torrent: TorrentInfo): boolean {
if (torrent.pack) {
return true;
}
const parsedInfo = parse(torrent.title);
if (torrent.type === TorrentType.Movie) {
return parsedInfo.complete || typeof parsedInfo.year === 'string' || /movies/i.test(torrent.title);
}
const hasMultipleEpisodes = parsedInfo.complete ||
torrent.size > this.MULTIPLE_FILES_SIZE ||
(parsedInfo.seasons && parsedInfo.seasons.length > 1) ||
(parsedInfo.episodes && parsedInfo.episodes.length > 1) ||
(parsedInfo.seasons && !parsedInfo.episodes);
const hasSingleEpisode = Number.isInteger(parsedInfo.episode) || (!parsedInfo.episodes && parsedInfo.date);
return hasMultipleEpisodes && !hasSingleEpisode;
}
private parseSeriesVideo(video: ParsableTorrentFile, parsedTorrentName: ParseTorrentTitleResult): ParseTorrentTitleResult {
const videoInfo = parse(video.name);
// the episode may be in a folder containing season number
if (!Number.isInteger(videoInfo.season) && video.path.includes('/')) {
const folders = video.path.split('/');
const pathInfo = parse(folders[folders.length - 2]);
videoInfo.season = pathInfo.season;
}
if (!Number.isInteger(videoInfo.season) && parsedTorrentName.season) {
videoInfo.season = parsedTorrentName.season;
}
if (!Number.isInteger(videoInfo.season) && videoInfo.seasons && videoInfo.seasons.length > 1) {
// in case single file was interpreted as having multiple seasons
videoInfo.season = videoInfo.seasons[0];
}
if (!Number.isInteger(videoInfo.season) && video.path.includes('/') && parsedTorrentName.seasons
&& parsedTorrentName.seasons.length > 1) {
// russian season are usually named with 'series name-2` i.e. Улицы разбитых фонарей-6/22. Одиночный выстрел.mkv
const folderPathSeasonMatch = video.path.match(/[\u0400-\u04ff]-(\d{1,2})(?=.*\/)/);
videoInfo.season = folderPathSeasonMatch && parseInt(folderPathSeasonMatch[1], 10) || undefined;
}
// sometimes video file does not have correct date format as in torrent title
if (!videoInfo.episodes && !videoInfo.date && parsedTorrentName.date) {
videoInfo.date = parsedTorrentName.date;
}
// limit number of episodes in case of incorrect parsing
if (videoInfo.episodes && videoInfo.episodes.length > 20) {
videoInfo.episodes = [videoInfo.episodes[0]];
videoInfo.episode = videoInfo.episodes[0];
}
// force episode to any found number if it was not parsed
if (!videoInfo.episodes && !videoInfo.date) {
const epMatcher = videoInfo.title.match(
/(?<!season\W*|disk\W*|movie\W*|film\W*)(?:^|\W|_)(\d{1,4})(?:a|b|c|v\d)?(?:_|\W|$)(?!disk|movie|film)/i);
videoInfo.episodes = epMatcher && [parseInt(epMatcher[1], 10)];
videoInfo.episode = videoInfo.episodes && videoInfo.episodes[0];
}
if (!videoInfo.episodes && !videoInfo.date) {
const epMatcher = video.name.match(new RegExp(`(?:\\(${videoInfo.year}\\)|part)[._ ]?(\\d{1,3})(?:\\b|_)`, "i"));
videoInfo.episodes = epMatcher && [parseInt(epMatcher[1], 10)];
videoInfo.episode = videoInfo.episodes && videoInfo.episodes[0];
}
return { ...video, ...videoInfo };
}
private isMovieVideo(video: ParseTorrentTitleResult, otherVideos: ParseTorrentTitleResult[], type: TorrentType, hasMovies: boolean): boolean {
if (Number.isInteger(video.season) && Array.isArray(video.episodes)) {
// not movie if video has season
return false;
}
if (video.title.match(/\b(?:\d+[ .]movie|movie[ .]\d+)\b/i)) {
// movie if video explicitly has numbered movie keyword in the name, ie. 1 Movie or Movie 1
return true;
}
if (!hasMovies && type !== TorrentType.Anime) {
// not movie if torrent name does not contain movies keyword or is not a pack torrent and is not anime
return false;
}
if (!video.episodes) {
// movie if there's no episode info it could be a movie
return true;
}
// movie if contains year info and there aren't more than 3 video with same title and year
// as some series titles might contain year in it.
return !!video.year
&& otherVideos.length > 3
&& otherVideos.filter(other => other.title === video.title && other.year === video.year).length < 3;
}
}
export const parsingService = new ParsingService();

125
src/node/consumer/src/lib/services/torrent_download_service.ts
View File

@@ -1,10 +1,20 @@
import { encode } from 'magnet-uri';
import {encode} from 'magnet-uri';
import torrentStream from 'torrent-stream';
import { configurationService } from './configuration_service';
import {configurationService} from './configuration_service';
import {extensionService} from './extension_service';
import {TorrentInfo} from "../interfaces/torrent_info";
import {TorrentFileCollection} from "../interfaces/torrent_file_collection";
import {ParsableTorrentFile} from "../interfaces/parsable_torrent_file";
import {ParsedTorrent} from "../interfaces/parsed_torrent";
import {FileAttributes} from "../../repository/interfaces/file_attributes";
import {SubtitleAttributes} from "../../repository/interfaces/subtitle_attributes";
import {ContentAttributes} from "../../repository/interfaces/content_attributes";
import {parse} from "parse-torrent-title";
interface TorrentFile {
name: string;
path: string;
length: number;
fileIndex: number;
}
class TorrentDownloadService {
private engineOptions: TorrentStream.TorrentEngineOptions = {
@@ -14,73 +24,124 @@ class TorrentDownloadService {
dht: false,
tracker: true,
};
public async getTorrentFiles(torrent: TorrentInfo, timeout: number = 30000): Promise<TorrentFileCollection> {
return this.filesFromTorrentStream(torrent, timeout)
.then((files: Array<ParsableTorrentFile>) => ({
contents: files,
videos: this.filterVideos(files),
subtitles: this.filterSubtitles(files)
}));
public async getTorrentFiles(torrent: ParsedTorrent, timeout: number = 30000): Promise<TorrentFileCollection> {
const torrentFiles: TorrentFile[] = await this.filesFromTorrentStream(torrent, timeout);
const videos = this.filterVideos(torrent, torrentFiles);
const subtitles = this.filterSubtitles(torrent, torrentFiles);
const contents = this.createContent(torrent, torrentFiles);
return {
contents: contents,
videos: videos,
subtitles: subtitles,
};
}
private async filesFromTorrentStream(torrent: TorrentInfo, timeout: number): Promise<Array<ParsableTorrentFile>> {
private async filesFromTorrentStream(torrent: ParsedTorrent, timeout: number): Promise<TorrentFile[]> {
if (!torrent.infoHash) {
return Promise.reject(new Error("No infoHash..."));
}
const magnet = encode({ infoHash: torrent.infoHash, announce: torrent.trackers.split(',') });
const magnet = encode({infoHash: torrent.infoHash, announce: torrent.trackers.split(',')});
return new Promise((resolve, reject) => {
let engine: TorrentStream.TorrentEngine;
const timeoutId = setTimeout(() => {
engine.destroy(() => {});
engine.destroy(() => {
});
reject(new Error('No available connections for torrent!'));
}, timeout);
engine = torrentStream(magnet, this.engineOptions);
engine.on("ready", () => {
const files: ParsableTorrentFile[] = engine.files.map((file, fileId) => ({
const files: TorrentFile[] = engine.files.map((file, fileId) => ({
...file,
fileIndex: fileId,
size: file.length,
title: file.name}));
title: file.name
}));
resolve(files);
engine.destroy(() => {});
engine.destroy(() => {
});
clearTimeout(timeoutId);
});
});
}
private filterVideos(files: Array<ParsableTorrentFile>): Array<ParsableTorrentFile> {
if (files.length === 1 && !Number.isInteger(files[0].fileIndex)) {
return files;
private filterVideos(torrent: ParsedTorrent, torrentFiles: TorrentFile[]): FileAttributes[] {
if (torrentFiles.length === 1 && !Number.isInteger(torrentFiles[0].fileIndex)) {
return [this.mapTorrentFileToFileAttributes(torrent, torrentFiles[0])];
}
const videos = files.filter(file => extensionService.isVideo(file.path || ''));
const maxSize = Math.max(...videos.map((video: ParsableTorrentFile) => video.length));
const videos = torrentFiles.filter(file => extensionService.isVideo(file.path || ''));
const maxSize = Math.max(...videos.map((video: TorrentFile) => video.length));
const minSampleRatio = videos.length <= 3 ? 3 : 10;
const minAnimeExtraRatio = 5;
const minRedundantRatio = videos.length <= 3 ? 30 : Number.MAX_VALUE;
const isSample = (video: ParsableTorrentFile) => video.path?.match(/sample|bonus|promo/i) && maxSize / parseInt(video.length.toString()) > minSampleRatio;
const isRedundant = (video: ParsableTorrentFile) => maxSize / parseInt(video.length.toString()) > minRedundantRatio;
const isExtra = (video: ParsableTorrentFile) => video.path?.match(/extras?\//i);
const isAnimeExtra = (video: ParsableTorrentFile) => video.path?.match(/(?:\b|_)(?:NC)?(?:ED|OP|PV)(?:v?\d\d?)?(?:\b|_)/i)
const isSample = (video: TorrentFile) => video.path?.match(/sample|bonus|promo/i) && maxSize / parseInt(video.path.toString()) > minSampleRatio;
const isRedundant = (video: TorrentFile) => maxSize / parseInt(video.path.toString()) > minRedundantRatio;
const isExtra = (video: TorrentFile) => video.path?.match(/extras?\//i);
const isAnimeExtra = (video: TorrentFile) => video.path?.match(/(?:\b|_)(?:NC)?(?:ED|OP|PV)(?:v?\d\d?)?(?:\b|_)/i)
&& maxSize / parseInt(video.length.toString()) > minAnimeExtraRatio;
const isWatermark = (video: ParsableTorrentFile) => video.path?.match(/^[A-Z-]+(?:\.[A-Z]+)?\.\w{3,4}$/)
const isWatermark = (video: TorrentFile) => video.path?.match(/^[A-Z-]+(?:\.[A-Z]+)?\.\w{3,4}$/)
&& maxSize / parseInt(video.length.toString()) > minAnimeExtraRatio
return videos
.filter(video => !isSample(video))
.filter(video => !isExtra(video))
.filter(video => !isAnimeExtra(video))
.filter(video => !isRedundant(video))
.filter(video => !isWatermark(video));
.filter(video => !isWatermark(video))
.map(video => this.mapTorrentFileToFileAttributes(torrent, video));
}
private filterSubtitles(files: Array<ParsableTorrentFile>): Array<ParsableTorrentFile> {
return files.filter(file => extensionService.isSubtitle(file.path || ''));
private filterSubtitles(torrent: ParsedTorrent, torrentFiles: TorrentFile[]): SubtitleAttributes[] {
return torrentFiles.filter(file => extensionService.isSubtitle(file.name || ''))
.map(file => this.mapTorrentFileToSubtitleAttributes(torrent, file));
}
private createContent(torrent: ParsedTorrent, torrentFiles: TorrentFile[]): ContentAttributes[] {
return torrentFiles.map(file => this.mapTorrentFileToContentAttributes(torrent, file));
}
private mapTorrentFileToFileAttributes(torrent: ParsedTorrent, file: TorrentFile): FileAttributes {
const videoFile: FileAttributes = {
title: file.name,
size: file.length,
fileIndex: file.fileIndex || 0,
infoHash: torrent.infoHash,
imdbId: torrent.imdbId.toString(),
imdbSeason: torrent.season || 0,
imdbEpisode: torrent.episode || 0,
kitsuId: parseInt(torrent.kitsuId.toString()) || 0,
kitsuEpisode: torrent.episode || 0
};
return {...videoFile, ...parse(file.name)};
}
private mapTorrentFileToSubtitleAttributes(torrent: ParsedTorrent, file: TorrentFile): SubtitleAttributes {
return {
title: file.name,
infoHash: torrent.infoHash,
fileIndex: file.fileIndex,
fileId: file.fileIndex,
path: file.path,
};
}
private mapTorrentFileToContentAttributes(torrent: ParsedTorrent, file: TorrentFile): ContentAttributes {
return {
infoHash: torrent.infoHash,
fileIndex: file.fileIndex,
path: file.path,
size: file.length,
};
}
}

233
src/node/consumer/src/lib/services/torrent_entries_service.ts Normal file
View File

@@ -0,0 +1,233 @@
import {parse} from 'parse-torrent-title';
import {ParsedTorrent} from "../interfaces/parsed_torrent";
import {repository} from '../../repository/database_repository';
import {TorrentType} from '../enums/torrent_types';
import {TorrentFileCollection} from "../interfaces/torrent_file_collection";
import {Torrent} from "../../repository/models/torrent";
import {PromiseHelpers} from '../helpers/promises_helpers';
import {logger} from './logging_service';
import {metadataService} from './metadata_service';
import {torrentFileService} from './torrent_file_service';
import {torrentSubtitleService} from './torrent_subtitle_service';
import {TorrentAttributes} from "../../repository/interfaces/torrent_attributes";
import {File} from "../../repository/models/file";
import {Subtitle} from "../../repository/models/subtitle";
class TorrentEntriesService {
public async createTorrentEntry(torrent: ParsedTorrent, overwrite = false): Promise<void> {
const titleInfo = parse(torrent.title);
if (!torrent.imdbId && torrent.type !== TorrentType.Anime) {
const imdbQuery = {
title: titleInfo.title,
year: titleInfo.year,
type: torrent.type
};
torrent.imdbId = await metadataService.getImdbId(imdbQuery)
.catch(() => undefined);
}
if (torrent.imdbId && torrent.imdbId.toString().length < 9) {
// pad zeros to imdbId if missing
torrent.imdbId = 'tt' + torrent.imdbId.toString().replace('tt', '').padStart(7, '0');
}
if (torrent.imdbId && torrent.imdbId.toString().length > 9 && torrent.imdbId.toString().startsWith('tt0')) {
// sanitize imdbId from redundant zeros
torrent.imdbId = torrent.imdbId.toString().replace(/tt0+([0-9]{7,})$/, 'tt$1');
}
if (!torrent.kitsuId && torrent.type === TorrentType.Anime) {
const kitsuQuery = {
title: titleInfo.title,
year: titleInfo.year,
season: titleInfo.season,
};
torrent.kitsuId = await metadataService.getKitsuId(kitsuQuery)
.catch(() => undefined);
}
if (!torrent.imdbId && !torrent.kitsuId && !torrentFileService.isPackTorrent(torrent)) {
logger.warn(`imdbId or kitsuId not found: ${torrent.provider} ${torrent.title}`);
return;
}
const fileCollection: TorrentFileCollection = await torrentFileService.parseTorrentFiles(torrent)
.then((torrentContents: TorrentFileCollection) => overwrite ? this.overwriteExistingFiles(torrent, torrentContents) : torrentContents)
.then((torrentContents: TorrentFileCollection) => torrentSubtitleService.assignSubtitles(torrentContents))
.catch(error => {
logger.warn(`Failed getting files for ${torrent.title}`, error.message);
return {};
});
if (!fileCollection.videos || !fileCollection.videos.length) {
logger.warn(`no video files found for ${torrent.provider} [${torrent.infoHash}] ${torrent.title}`);
return;
}
const newTorrent: Torrent = Torrent.build({
...torrent,
contents: fileCollection.contents,
subtitles: fileCollection.subtitles
});
return repository.createTorrent(newTorrent)
.then(() => PromiseHelpers.sequence(fileCollection.videos.map(video => () => {
const newVideo = File.build(video);
return repository.createFile(newVideo)
})))
.then(() => logger.info(`Created ${torrent.provider} entry for [${torrent.infoHash}] ${torrent.title}`));
}
public async createSkipTorrentEntry(torrent: Torrent) {
return repository.createSkipTorrent(torrent);
}
public async getStoredTorrentEntry(torrent: Torrent) {
return repository.getSkipTorrent(torrent.infoHash)
.catch(() => repository.getTorrent(torrent))
.catch(() => undefined);
}
public async checkAndUpdateTorrent(torrent: ParsedTorrent): Promise<boolean> {
const query: TorrentAttributes = {
infoHash: torrent.infoHash,
provider: torrent.provider,
}
const existingTorrent = await repository.getTorrent(query).catch(() => undefined);
if (!existingTorrent) {
return false;
}
if (existingTorrent.provider === 'RARBG') {
return true;
}
if (existingTorrent.provider === 'KickassTorrents' && torrent.provider) {
existingTorrent.provider = torrent.provider;
existingTorrent.torrentId = torrent.torrentId;
}
if (!existingTorrent.languages && torrent.languages && existingTorrent.provider !== 'RARBG') {
existingTorrent.languages = torrent.languages;
await existingTorrent.save();
logger.debug(`Updated [${existingTorrent.infoHash}] ${existingTorrent.title} language to ${torrent.languages}`);
}
return this.createTorrentContents(existingTorrent)
.then(() => this.updateTorrentSeeders(existingTorrent));
}
public async createTorrentContents(torrent: Torrent) {
if (torrent.opened) {
return;
}
const storedVideos: File[] = await repository.getFiles(torrent.infoHash).catch(() => []);
if (!storedVideos || !storedVideos.length) {
return;
}
const notOpenedVideo = storedVideos.length === 1 && !Number.isInteger(storedVideos[0].fileIndex);
const imdbId: string | undefined = PromiseHelpers.mostCommonValue(storedVideos.map(stored => stored.imdbId));
const kitsuId: number | undefined = PromiseHelpers.mostCommonValue(storedVideos.map(stored => stored.kitsuId));
const fileCollection: TorrentFileCollection = await torrentFileService.parseTorrentFiles(torrent)
.then(torrentContents => notOpenedVideo ? torrentContents : {...torrentContents, videos: storedVideos})
.then(torrentContents => torrentSubtitleService.assignSubtitles(torrentContents))
.then(torrentContents => this.assignMetaIds(torrentContents, imdbId, kitsuId))
.catch(error => {
logger.warn(`Failed getting contents for [${torrent.infoHash}] ${torrent.title}`, error.message);
return {};
});
this.assignMetaIds(fileCollection, imdbId, kitsuId);
if (!fileCollection.contents || !fileCollection.contents.length) {
return;
}
if (notOpenedVideo && fileCollection.videos.length === 1) {
// if both have a single video and stored one was not opened, update stored one to true metadata and use that
storedVideos[0].fileIndex = fileCollection.videos[0].fileIndex;
storedVideos[0].title = fileCollection.videos[0].title;
storedVideos[0].size = fileCollection.videos[0].size;
storedVideos[0].subtitles = fileCollection.videos[0].subtitles.map(subtitle => Subtitle.build(subtitle));
fileCollection.videos[0] = storedVideos[0];
}
// no videos available or more than one new videos were in the torrent
const shouldDeleteOld = notOpenedVideo && fileCollection.videos.every(video => !video.id);
const newTorrent: Torrent = Torrent.build({
...torrent,
contents: fileCollection.contents,
subtitles: fileCollection.subtitles
});
return repository.createTorrent(newTorrent)
.then(() => {
if (shouldDeleteOld) {
logger.debug(`Deleting old video for [${torrent.infoHash}] ${torrent.title}`)
return storedVideos[0].destroy();
}
return Promise.resolve();
})
.then(() => PromiseHelpers.sequence(fileCollection.videos.map(video => () => {
const newVideo = File.build(video);
return repository.createFile(newVideo)
})))
.then(() => logger.info(`Created contents for ${torrent.provider} [${torrent.infoHash}] ${torrent.title}`))
.catch(error => logger.error(`Failed saving contents for [${torrent.infoHash}] ${torrent.title}`, error));
}
public async updateTorrentSeeders(torrent: TorrentAttributes) {
if (!(torrent.infoHash || (torrent.provider && torrent.torrentId)) || !Number.isInteger(torrent.seeders)) {
return torrent;
}
return repository.setTorrentSeeders(torrent, torrent.seeders)
.catch(error => {
logger.warn('Failed updating seeders:', error);
return undefined;
});
}
private assignMetaIds(fileCollection: TorrentFileCollection, imdbId: string, kitsuId: number): TorrentFileCollection {
if (fileCollection.videos && fileCollection.videos.length) {
fileCollection.videos.forEach(video => {
video.imdbId = imdbId;
video.kitsuId = kitsuId;
});
}
return fileCollection;
}
private async overwriteExistingFiles(torrent: ParsedTorrent, torrentContents: TorrentFileCollection) {
const videos = torrentContents && torrentContents.videos;
if (videos && videos.length) {
const existingFiles = await repository.getFiles(torrent.infoHash)
.then((existing) => existing
.reduce((map, next) => {
const fileIndex = next.fileIndex !== undefined ? next.fileIndex : null;
map[fileIndex] = (map[fileIndex] || []).concat(next);
return map;
}, {}))
.catch(() => undefined);
if (existingFiles && Object.keys(existingFiles).length) {
const overwrittenVideos = videos
.map(file => {
const mapping = videos.length === 1 && Object.keys(existingFiles).length === 1
? Object.values(existingFiles)[0]
: existingFiles[file.fileIndex !== undefined ? file.fileIndex : null];
if (mapping) {
const originalFile = mapping.shift();
return {id: originalFile.id, ...file};
}
return file;
});
return {...torrentContents, videos: overwrittenVideos};
}
return torrentContents;
}
return Promise.reject(`No video files found for: ${torrent.title}`);
}
}
export const torrentEntriesService = new TorrentEntriesService();

257
src/node/consumer/src/lib/services/torrent_file_service.ts
View File

@@ -1,30 +1,32 @@
import Bottleneck from 'bottleneck';
import moment from 'moment';
import { parse } from 'parse-torrent-title';
import {parse} from 'parse-torrent-title';
import {PromiseHelpers} from '../helpers/promises_helpers';
import { TorrentType } from '../enums/torrent_types';
import {TorrentInfo} from "../interfaces/torrent_info";
import { configurationService } from './configuration_service';
import { extensionService } from './extension_service';
import { metadataService } from './metadata_service';
import { parsingService } from './parsing_service';
import { torrentDownloadService } from "./torrent_download_service";
import {TorrentType} from '../enums/torrent_types';
import {configurationService} from './configuration_service';
import {extensionService} from './extension_service';
import {metadataService} from './metadata_service';
import {torrentDownloadService} from "./torrent_download_service";
import {logger} from "./logging_service";
import {MetadataResponse} from "../interfaces/metadata_response";
import {ParseTorrentTitleResult} from "../interfaces/parse_torrent_title_result";
import {MetaDataQuery} from "../interfaces/metadata_query";
import {ParsableTorrentFile} from "../interfaces/parsable_torrent_file";
import {CommonVideoMetadata} from "../interfaces/common_video_metadata";
import {TorrentFileCollection} from "../interfaces/torrent_file_collection";
import {ParsedTorrent} from "../interfaces/parsed_torrent";
import {FileAttributes} from "../../repository/interfaces/file_attributes";
import {ContentAttributes} from "../../repository/interfaces/content_attributes";
class TorrentFileService {
private readonly MIN_SIZE: number = 5 * 1024 * 1024; // 5 MB
private readonly MULTIPLE_FILES_SIZE = 4 * 1024 * 1024 * 1024; // 4 GB
private readonly imdb_limiter: Bottleneck = new Bottleneck({
maxConcurrent: configurationService.metadataConfig.IMDB_CONCURRENT,
minTime: configurationService.metadataConfig.IMDB_INTERVAL_MS
});
public async parseTorrentFiles(torrent: TorrentInfo) {
public async parseTorrentFiles(torrent: ParsedTorrent): Promise<TorrentFileCollection> {
const parsedTorrentName = parse(torrent.title);
const query: MetaDataQuery = {
id: torrent.kitsuId || torrent.imdbId,
@@ -47,7 +49,32 @@ class TorrentFileService {
return this.parseSeriesFiles(torrent, metadata)
}
private async parseMovieFiles(torrent: TorrentInfo, metadata: MetadataResponse): Promise<TorrentFileCollection> {
public isPackTorrent(torrent: ParsedTorrent): boolean {
if (torrent.isPack) {
return true;
}
const parsedInfo = parse(torrent.title);
if (torrent.type === TorrentType.Movie) {
return parsedInfo.complete || typeof parsedInfo.year === 'string' || /movies/i.test(torrent.title);
}
const hasMultipleEpisodes = parsedInfo.complete ||
torrent.size > this.MULTIPLE_FILES_SIZE ||
(parsedInfo.seasons && parsedInfo.seasons.length > 1) ||
(parsedInfo.episodes && parsedInfo.episodes.length > 1) ||
(parsedInfo.seasons && !parsedInfo.episodes);
const hasSingleEpisode = Number.isInteger(parsedInfo.episode) || (!parsedInfo.episodes && parsedInfo.date);
return hasMultipleEpisodes && !hasSingleEpisode;
}
private parseSeriesVideos(torrent: ParsedTorrent): FileAttributes[] {
const parsedTorrentName = parse(torrent.title);
const hasMovies = parsedTorrentName.complete || !!torrent.title.match(/movies?(?:\W|$)/i);
const parsedVideos = torrent.fileCollection.videos.map(video => this.parseSeriesVideo(video));
return parsedVideos.map(video => ({ ...video, isMovie: this.isMovieVideo(torrent, video, parsedVideos, hasMovies) }));
}
private async parseMovieFiles(torrent: ParsedTorrent, metadata: MetadataResponse): Promise<TorrentFileCollection> {
const {contents, videos, subtitles} = await this.getMoviesTorrentContent(torrent);
const filteredVideos = videos
.filter(video => video.size > this.MIN_SIZE)
@@ -66,7 +93,7 @@ class TorrentFileService {
const parsedVideos = await PromiseHelpers.sequence(filteredVideos.map(video => () => this.isFeaturette(video)
? Promise.resolve(video)
: this.findMovieImdbId(video.name).then(imdbId => ({...video, imdbId}))))
: this.findMovieImdbId(video.title).then(imdbId => ({...video, imdbId}))))
.then(videos => videos.map(video => ({
infoHash: torrent.infoHash,
fileIndex: video.fileIndex,
@@ -77,53 +104,58 @@ class TorrentFileService {
return {contents, videos: parsedVideos, subtitles};
}
private async parseSeriesFiles(torrent: TorrentInfo, metadata: MetadataResponse) {
const {contents, videos, subtitles} = await this.getSeriesTorrentContent(torrent);
const parsedVideos = await Promise.resolve(videos)
private async parseSeriesFiles(torrent: ParsedTorrent, metadata: MetadataResponse): TorrentFileCollection {
const fileCollection: TorrentFileCollection = await this.getSeriesTorrentContent(torrent);
const parsedVideos: FileAttributes[] = await Promise.resolve(fileCollection.videos)
.then(videos => videos.filter(video => videos.length === 1 || video.size > this.MIN_SIZE))
.then(videos => parsingService.parseSeriesVideos(torrent, videos))
.then(videos => this.decomposeEpisodes(torrent, videos, metadata))
.then(videos => this.assignKitsuOrImdbEpisodes(torrent, videos, metadata))
.then(videos => this.parseSeriesVideos(torrent))
.then(videos => this.decomposeEpisodes(torrent, metadata))
.then(videos => this.assignKitsuOrImdbEpisodes(torrent, fileCollection.videos, metadata))
.then(videos => Promise.all(videos.map(video => video.isMovie
? this.mapSeriesMovie(video, torrent)
: this.mapSeriesEpisode(video, torrent, videos))))
.then(videos => videos
.map((video: ParsableTorrentFile) => this.isFeaturette(video) ? this.clearInfoFields(video) : video))
return {contents, videos: parsedVideos, subtitles};
.map((video: ParsedTorrent) => this.isFeaturette(video) ? this.clearInfoFields(video) : video))
return {...torrent.fileCollection, videos: parsedVideos};
}
private async getMoviesTorrentContent(torrent: TorrentInfo) {
private async getMoviesTorrentContent(torrent: ParsedTorrent) {
const files = await torrentDownloadService.getTorrentFiles(torrent)
.catch(error => {
if (!parsingService.isPackTorrent(torrent)) {
return { videos: [{name: torrent.title, path: torrent.title, size: torrent.size, fileIndex: null}], contents:[], subtitles: []}
if (!this.isPackTorrent(torrent)) {
const entries = [{name: torrent.title, path: torrent.title, size: torrent.size, fileIndex: null}];
return { videos: entries, contents:[], subtitles: [], files: entries}
}
return Promise.reject(error);
});
if (files.contents && files.contents.length && !files.videos.length && this.isDiskTorrent(files.contents)) {
files.videos = [{name: torrent.title, path: torrent.title, size: torrent.size}];
files.videos = [{name: torrent.title, path: torrent.title, size: torrent.size, fileIndex: null}];
}
return files;
}
private getDefaultFileEntries(torrent: ParsedTorrent): FileAttributes[] {
return [{title: torrent.title, path: torrent.title, size: torrent.size, fileIndex: null}];
}
private async getSeriesTorrentContent(torrent: TorrentInfo) {
private async getSeriesTorrentContent(torrent: ParsedTorrent): Promise<TorrentFileCollection> {
return torrentDownloadService.getTorrentFiles(torrent)
.catch(error => {
if (!parsingService.isPackTorrent(torrent)) {
return { videos: [{ name: torrent.title, path: torrent.title, size: torrent.size }], subtitles: [], contents: [] }
if (!this.isPackTorrent(torrent)) {
return { videos: this.getDefaultFileEntries(torrent), subtitles: [], contents: [] }
}
return Promise.reject(error);
});
}
private async mapSeriesEpisode(file: ParsableTorrentFile, torrent: TorrentInfo, files: ParsableTorrentFile[]) : Promise<ParsableTorrentFile> {
private async mapSeriesEpisode(file: FileAttributes, torrent: ParsedTorrent, files: ParsedTorrent[]) : Promise<FileAttributes[]> {
if (!file.episodes && !file.episodes) {
if (files.length === 1 || files.some(f => f.episodes || f.episodes) || parse(torrent.title).seasons) {
return Promise.resolve({
infoHash: torrent.infoHash,
fileIndex: file.fileIndex,
title: file.path || file.name,
title: file.path || file.title,
size: file.size,
imdbId: torrent.imdbId || file.imdbId,
});
@@ -134,17 +166,20 @@ class TorrentFileService {
return Promise.resolve(episodeIndexes.map((index) => ({
infoHash: torrent.infoHash,
fileIndex: file.fileIndex,
title: file.path || file.name,
title: file.path || file.title,
size: file.size,
imdbId: file.imdbId || torrent.imdbId,
imdbId: file.imdbId.toString() || torrent.imdbId.toString(),
imdbSeason: file.season,
season: file.season,
imdbEpisode: file.episodes && file.episodes[index],
episode: file.episodes && file.episodes[index],
kitsuEpisode: file.episodes && file.episodes[index],
episodes: file.episodes,
kitsuId: file.kitsuId || torrent.kitsuId,
kitsuId: parseInt(file.kitsuId.toString() || torrent.kitsuId.toString()),
})))
}
private async mapSeriesMovie(file: ParsableTorrentFile, torrent: TorrentInfo): Promise<ParsableTorrentFile> {
private async mapSeriesMovie(torrent: ParsedTorrent, file: FileAttributes): Promise<FileAttributes[]> {
const kitsuId= torrent.type === TorrentType.Anime ? await this.findMovieKitsuId(file)
.then(result => {
if (result instanceof Error) {
@@ -168,10 +203,10 @@ class TorrentFileService {
return [{
infoHash: torrent.infoHash,
fileIndex: file.fileIndex,
title: file.path || file.name,
title: file.path || file.title,
size: file.size,
imdbId: imdbId,
kitsuId: kitsuId,
kitsuId: parseInt(kitsuId),
episodes: undefined,
imdbSeason: undefined,
imdbEpisode: undefined,
@@ -185,16 +220,17 @@ class TorrentFileService {
return [{
infoHash: torrent.infoHash,
fileIndex: file.fileIndex,
title: file.path || file.name,
title: file.path || file.title,
size: file.size,
imdbId: metadata.imdbId || imdbId,
kitsuId: metadata.kitsuId || kitsuId,
season: episodeVideo && metadata.imdbId ? episodeVideo.season : undefined,
episode: episodeVideo && metadata.imdbId | metadata.kitsuId ? episodeVideo.episode || episodeVideo.episode : undefined,
imdbId: metadata.imdbId.toString() || imdbId,
kitsuId: parseInt(metadata.kitsuId.toString() || kitsuId),
imdbSeason: episodeVideo && metadata.imdbId ? episodeVideo.season : undefined,
imdbEpisode: episodeVideo && metadata.imdbId | metadata.kitsuId ? episodeVideo.episode || episodeVideo.episode : undefined,
kitsuEpisode: episodeVideo && metadata.imdbId | metadata.kitsuId ? episodeVideo.episode || episodeVideo.episode : undefined,
}];
}
private async decomposeEpisodes(torrent: TorrentInfo, files: ParsableTorrentFile[], metadata: MetadataResponse = { episodeCount: [] }) {
private async decomposeEpisodes(torrent: ParsedTorrent, files: FileAttributes[], metadata: MetadataResponse = { episodeCount: [] }) {
if (files.every(file => !file.episodes && !file.date)) {
return files;
}
@@ -239,7 +275,7 @@ class TorrentFileService {
return files;
}
private preprocessEpisodes(files: ParsableTorrentFile[]) {
private preprocessEpisodes(files: FileAttributes[]) {
// reverse special episode naming when they named with 0 episode, ie. S02E00
files
.filter(file => Number.isInteger(file.season) && file.episode === 0)
@@ -250,7 +286,7 @@ class TorrentFileService {
})
}
private isConcatSeasonAndEpisodeFiles(files: ParsableTorrentFile[], sortedEpisodes: number[], metadata: MetadataResponse) {
private isConcatSeasonAndEpisodeFiles(files: FileAttributes[], sortedEpisodes: number[], metadata: MetadataResponse) {
if (metadata.kitsuId !== undefined) {
// anime does not use this naming scheme in 99% of cases;
return false;
@@ -277,11 +313,11 @@ class TorrentFileService {
|| concatAboveTotalEpisodeCount.length >= thresholdAbove;
}
private isDateEpisodeFiles(files: ParsableTorrentFile[], metadata: MetadataResponse) {
private isDateEpisodeFiles(files: FileAttributes[], metadata: MetadataResponse) {
return files.every(file => (!file.season || !metadata.episodeCount[file.season - 1]) && file.date);
}
private isAbsoluteEpisodeFiles(torrent: TorrentInfo, files: ParsableTorrentFile[], metadata: MetadataResponse) {
private isAbsoluteEpisodeFiles(torrent: ParsedTorrent, files: FileAttributes[], metadata: MetadataResponse) {
const threshold = Math.ceil(files.length / 5);
const isAnime = torrent.type === TorrentType.Anime && torrent.kitsuId;
const nonMovieEpisodes = files
@@ -294,18 +330,18 @@ class TorrentFileService {
|| absoluteEpisodes.length >= threshold;
}
private isNewEpisodeNotInMetadata(torrent: TorrentInfo, file: ParsableTorrentFile, metadata: MetadataResponse) {
private isNewEpisodeNotInMetadata(torrent: ParsedTorrent, video: FileAttributes, metadata: MetadataResponse) {
// new episode might not yet been indexed by cinemeta.
// detect this if episode number is larger than the last episode or season is larger than the last one
// only for non anime metas
const isAnime = torrent.type === TorrentType.Anime && torrent.kitsuId;
return !isAnime && !file.isMovie && file.episodes && file.season !== 1
return !isAnime && !video.isMovie && video.episodes && video.season !== 1
&& /continuing|current/i.test(metadata.status)
&& file.season >= metadata.episodeCount.length
&& file.episodes.every(ep => ep > (metadata.episodeCount[file.season - 1] || 0));
&& video.season >= metadata.episodeCount.length
&& video.episodes.every(ep => ep > (metadata.episodeCount[video.season - 1] || 0));
}
private decomposeConcatSeasonAndEpisodeFiles(files: ParsableTorrentFile[], metadata: MetadataResponse) {
private decomposeConcatSeasonAndEpisodeFiles(files: FileAttributes[], metadata: MetadataResponse) {
files
.filter(file => file.episodes && file.season !== 0 && file.episodes.every(ep => ep > 100))
.filter(file => metadata.episodeCount[(file.season || this.div100(file.episodes[0])) - 1] < 100)
@@ -317,16 +353,16 @@ class TorrentFileService {
}
private decomposeAbsoluteEpisodeFiles(torrent: TorrentInfo, files: ParsableTorrentFile[], metadata: MetadataResponse) {
private decomposeAbsoluteEpisodeFiles(torrent: ParsedTorrent, videos: FileAttributes[], metadata: MetadataResponse) {
if (metadata.episodeCount.length === 0) {
files
videos
.filter(file => !Number.isInteger(file.season) && file.episodes && !file.isMovie)
.forEach(file => {
file.season = 1;
});
return;
}
files
videos
.filter(file => file.episodes && !file.isMovie && file.season !== 0)
.filter(file => !this.isNewEpisodeNotInMetadata(torrent, file, metadata))
.filter(file => !file.season || (metadata.episodeCount[file.season - 1] || 0) < file.episodes[0])
@@ -341,7 +377,7 @@ class TorrentFileService {
});
}
private decomposeDateEpisodeFiles(files: ParsableTorrentFile[], metadata: MetadataResponse) {
private decomposeDateEpisodeFiles(files: FileAttributes[], metadata: MetadataResponse) {
if (!metadata || !metadata.videos || !metadata.videos.length) {
return;
}
@@ -375,7 +411,7 @@ class TorrentFileService {
}
}
private assignKitsuOrImdbEpisodes(torrent: TorrentInfo, files: ParsableTorrentFile[], metadata: MetadataResponse) {
private assignKitsuOrImdbEpisodes(torrent: ParsedTorrent, files: FileAttributes[], metadata: MetadataResponse) {
if (!metadata || !metadata.videos || !metadata.videos.length) {
if (torrent.type === TorrentType.Anime) {
// assign episodes as kitsu episodes for anime when no metadata available for imdb mapping
@@ -387,7 +423,7 @@ class TorrentFileService {
})
if (metadata.type === TorrentType.Movie && files.every(file => !file.imdbId)) {
// sometimes a movie has episode naming, thus not recognized as a movie and imdbId not assigned
files.forEach(file => file.imdbId = metadata.imdbId);
files.forEach(file => file.imdbId = metadata.imdbId.toString());
}
}
return files;
@@ -402,14 +438,14 @@ class TorrentFileService {
}, {});
if (metadata.videos.some(video => Number.isInteger(video.season)) || !metadata.imdbId) {
files.filter((file => Number.isInteger(file.season) && file.episodes))
files.filter((file => Number.isInteger(torrent.season) && torrent.episodes))
.map(file => {
const seasonMapping = seriesMapping[file.season];
const episodeMapping = seasonMapping && seasonMapping[file.episodes[0]];
if (episodeMapping && Number.isInteger(episodeMapping.imdbSeason)) {
file.imdbId = metadata.imdbId;
file.season = episodeMapping.imdbSeason;
file.episodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].imdbEpisode);
if (episodeMapping && Number.isInteger(episodeMapping.season)) {
file.imdbId = metadata.imdbId.toString();
file.season = episodeMapping.season;
file.episodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].episode);
} else {
// no imdb mapping available for episode
file.season = undefined;
@@ -423,7 +459,7 @@ class TorrentFileService {
.forEach(file => {
if (seriesMapping[file.season]) {
const seasonMapping = seriesMapping[file.season];
file.imdbId = metadata.imdbId;
file.imdbId = metadata.imdbId.toString();
file.kitsuId = seasonMapping[file.episodes[0]] && seasonMapping[file.episodes[0]].kitsuId;
file.episodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].kitsuEpisode);
} else if (seriesMapping[file.season - 1]) {
@@ -434,12 +470,12 @@ class TorrentFileService {
const differentTitlesCount = new Set(episodes.map(ep => ep.id)).size
const skippedCount = episodes.filter(ep => ep.id === firstKitsuId).length;
const seasonEpisodes = files
.filter(otherFile => otherFile.season === file.season)
.filter((otherFile: FileAttributes) => otherFile.season === file.season)
.reduce((a, b) => a.episodes.concat(b.episodes), []);
const isAbsoluteOrder = seasonEpisodes.episodes.every(ep => ep > skippedCount && ep <= episodes.length)
const isNormalOrder = seasonEpisodes.episodes.every(ep => ep + skippedCount <= episodes.length)
if (differentTitlesCount >= 1 && (isAbsoluteOrder || isNormalOrder)) {
file.imdbId = metadata.imdbId;
file.imdbId = metadata.imdbId.toString();
file.season = file.season - 1;
file.episodes = file.episodes.map(ep => isAbsoluteOrder ? ep : ep + skippedCount);
file.kitsuId = seasonMapping[file.episodes[0]].kitsuId;
@@ -448,7 +484,7 @@ class TorrentFileService {
} else if (Object.values(seriesMapping).length === 1 && seriesMapping[1]) {
// sometimes series might be named with sequel season but it's not a season on imdb and a new title
const seasonMapping = seriesMapping[1];
file.imdbId = metadata.imdbId;
file.imdbId = metadata.imdbId.toString();
file.season = 1;
file.kitsuId = seasonMapping[file.episodes[0]].kitsuId;
file.episodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].kitsuEpisode);
@@ -458,7 +494,7 @@ class TorrentFileService {
return files;
}
private needsCinemetaMetadataForAnime(files: ParsableTorrentFile[], metadata: MetadataResponse) {
private needsCinemetaMetadataForAnime(files: FileAttributes[], metadata: MetadataResponse) {
if (!metadata || !metadata.imdbId || !metadata.videos || !metadata.videos.length) {
return false;
}
@@ -500,7 +536,7 @@ class TorrentFileService {
})
}
private findMovieImdbId(title: ParseTorrentTitleResult | string) {
private findMovieImdbId(title: FileAttributes | string) {
const parsedTitle = typeof title === 'string' ? parse(title) : title;
logger.debug(`Finding movie imdbId for ${title}`);
return this.imdb_limiter.schedule(async () => {
@@ -517,7 +553,7 @@ class TorrentFileService {
});
}
private async findMovieKitsuId(title: ParseTorrentTitleResult | string) {
private async findMovieKitsuId(title: FileAttributes | string) {
const parsedTitle = typeof title === 'string' ? parse(title) : title;
const kitsuQuery = {
title: parsedTitle.title,
@@ -532,26 +568,97 @@ class TorrentFileService {
}
}
private isDiskTorrent(contents: ParsableTorrentFile[]) {
private isDiskTorrent(contents: ContentAttributes[]) {
return contents.some(content => extensionService.isDisk(content.path));
}
private isSingleMovie(videos: ParsableTorrentFile[]) {
private isSingleMovie(videos: FileAttributes[]) {
return videos.length === 1 ||
(videos.length === 2 &&
videos.find(v => /\b(?:part|disc|cd)[ ._-]?0?1\b|^0?1\.\w{2,4}$/i.test(v.path)) &&
videos.find(v => /\b(?:part|disc|cd)[ ._-]?0?2\b|^0?2\.\w{2,4}$/i.test(v.path)));
}
private isFeaturette(video: ParsableTorrentFile) {
private isFeaturette(video: FileAttributes) {
return /featurettes?\/|extras-grym/i.test(video.path);
}
private clearInfoFields(video: ParsableTorrentFile) {
private parseSeriesVideo(video: FileAttributes): FileAttributes {
const videoInfo = parse(video.title);
// the episode may be in a folder containing season number
if (!Number.isInteger(videoInfo.season) && video.path.includes('/')) {
const folders = video.path.split('/');
const pathInfo = parse(folders[folders.length - 2]);
videoInfo.season = pathInfo.season;
}
if (!Number.isInteger(videoInfo.season) && video.season) {
videoInfo.season = video.season;
}
if (!Number.isInteger(videoInfo.season) && videoInfo.seasons && videoInfo.seasons.length > 1) {
// in case single file was interpreted as having multiple seasons
videoInfo.season = videoInfo.seasons[0];
}
if (!Number.isInteger(videoInfo.season) && video.path.includes('/') && video.seasons
&& video.seasons.length > 1) {
// russian season are usually named with 'series name-2` i.e. Улицы разбитых фонарей-6/22. Одиночный выстрел.mkv
const folderPathSeasonMatch = video.path.match(/[\u0400-\u04ff]-(\d{1,2})(?=.*\/)/);
videoInfo.season = folderPathSeasonMatch && parseInt(folderPathSeasonMatch[1], 10) || undefined;
}
// sometimes video file does not have correct date format as in torrent title
if (!videoInfo.episodes && !videoInfo.date && video.date) {
videoInfo.date = video.date;
}
// limit number of episodes in case of incorrect parsing
if (videoInfo.episodes && videoInfo.episodes.length > 20) {
videoInfo.episodes = [videoInfo.episodes[0]];
videoInfo.episode = videoInfo.episodes[0];
}
// force episode to any found number if it was not parsed
if (!videoInfo.episodes && !videoInfo.date) {
const epMatcher = videoInfo.title.match(
/(?<!season\W*|disk\W*|movie\W*|film\W*)(?:^|\W|_)(\d{1,4})(?:a|b|c|v\d)?(?:_|\W|$)(?!disk|movie|film)/i);
videoInfo.episodes = epMatcher && [parseInt(epMatcher[1], 10)];
videoInfo.episode = videoInfo.episodes && videoInfo.episodes[0];
}
if (!videoInfo.episodes && !videoInfo.date) {
const epMatcher = video.title.match(new RegExp(`(?:\\(${videoInfo.year}\\)|part)[._ ]?(\\d{1,3})(?:\\b|_)`, "i"));
videoInfo.episodes = epMatcher && [parseInt(epMatcher[1], 10)];
videoInfo.episode = videoInfo.episodes && videoInfo.episodes[0];
}
return { ...video, ...videoInfo };
}
private isMovieVideo(torrent: ParsedTorrent, video: FileAttributes, otherVideos: FileAttributes[], hasMovies: boolean): boolean {
if (Number.isInteger(torrent.season) && Array.isArray(torrent.episodes)) {
// not movie if video has season
return false;
}
if (torrent.title.match(/\b(?:\d+[ .]movie|movie[ .]\d+)\b/i)) {
// movie if video explicitly has numbered movie keyword in the name, ie. 1 Movie or Movie 1
return true;
}
if (!hasMovies && torrent.type !== TorrentType.Anime) {
// not movie if torrent name does not contain movies keyword or is not a pack torrent and is not anime
return false;
}
if (!torrent.episodes) {
// movie if there's no episode info it could be a movie
return true;
}
// movie if contains year info and there aren't more than 3 video with same title and year
// as some series titles might contain year in it.
return !!torrent.year
&& otherVideos.length > 3
&& otherVideos.filter(other => other.title === video.title && other.year === video.year).length < 3;
}
private clearInfoFields(video: FileAttributes) {
video.imdbId = undefined;
video.season = undefined;
video.episode = undefined;
video.imdbSeason = undefined;
video.imdbEpisode = undefined;
video.kitsuId = undefined;
video.kitsuEpisode = undefined;
return video;
}

14
src/node/consumer/src/lib/services/torrent_processing_service.ts
View File

@@ -1,23 +1,23 @@
import {TorrentInfo} from "../interfaces/torrent_info";
import {TorrentType} from "../enums/torrent_types";
import {logger} from "./logging_service";
import {checkAndUpdateTorrent, createTorrentEntry} from "../torrentEntries.js";
import {trackerService} from "./tracker_service";
import {torrentEntriesService} from "./torrent_entries_service";
import {IngestedTorrentAttributes} from "../../repository/interfaces/ingested_torrent_attributes";
import {ParsedTorrent} from "../interfaces/parsed_torrent";
class TorrentProcessingService {
public async processTorrentRecord(torrent: IngestedTorrentAttributes): Promise<void> {
const { category } = torrent;
const type = category === 'tv' ? TorrentType.Series : TorrentType.Movie;
const torrentInfo: TorrentInfo = await this.parseTorrent(torrent, type);
const torrentInfo: ParsedTorrent = await this.parseTorrent(torrent, type);
logger.info(`Processing torrent ${torrentInfo.title} with infoHash ${torrentInfo.infoHash}`);
if (await checkAndUpdateTorrent(torrentInfo)) {
if (await torrentEntriesService.checkAndUpdateTorrent(torrentInfo)) {
return;
}
return createTorrentEntry(torrentInfo);
return torrentEntriesService.createTorrentEntry(torrentInfo);
}
private async assignTorrentTrackers(): Promise<string> {
@@ -25,14 +25,14 @@ class TorrentProcessingService {
return trackers.join(',');
}
private async parseTorrent(torrent: IngestedTorrentAttributes, category: string): Promise<TorrentInfo> {
private async parseTorrent(torrent: IngestedTorrentAttributes, category: string): Promise<ParsedTorrent> {
const infoHash = torrent.info_hash?.trim().toLowerCase()
return {
title: torrent.name,
torrentId: `${torrent.name}_${infoHash}`,
infoHash: infoHash,
seeders: 100,
size: torrent.size,
size: parseInt(torrent.size),
uploadDate: torrent.createdAt,
imdbId: this.parseImdbId(torrent),
type: category,

22
src/node/consumer/src/lib/services/torrent_subtitle_service.ts
View File

@@ -1,26 +1,28 @@
import { parse } from 'parse-torrent-title';
import {TorrentFileCollection} from "../interfaces/torrent_file_collection";
import {FileAttributes} from "../../repository/interfaces/file_attributes";
class TorrentSubtitleService {
public assignSubtitles(contents: any, videos: any[], subtitles: any[]) {
if (videos && videos.length && subtitles && subtitles.length) {
if (videos.length === 1) {
videos[0].subtitles = subtitles;
return { contents, videos, subtitles: [] };
public assignSubtitles(fileCollection: TorrentFileCollection) : TorrentFileCollection {
if (fileCollection.videos && fileCollection.videos.length && fileCollection.subtitles && fileCollection.subtitles.length) {
if (fileCollection.videos.length === 1) {
fileCollection.videos[0].subtitles = fileCollection.subtitles;
return { ...fileCollection, subtitles: [] };
}
const parsedVideos = videos.map(video => this.parseVideo(video));
const assignedSubs = subtitles.map(subtitle => ({ subtitle, videos: this.mostProbableSubtitleVideos(subtitle, parsedVideos) }));
const parsedVideos = fileCollection.videos.map(video => this.parseVideo(video));
const assignedSubs = fileCollection.subtitles.map(subtitle => ({ subtitle, videos: this.mostProbableSubtitleVideos(subtitle, parsedVideos) }));
const unassignedSubs = assignedSubs.filter(assignedSub => !assignedSub.videos).map(assignedSub => assignedSub.subtitle);
assignedSubs
.filter(assignedSub => assignedSub.videos)
.forEach(assignedSub => assignedSub.videos.forEach(video => video.subtitles = (video.subtitles || []).concat(assignedSub.subtitle)));
return { contents, videos, subtitles: unassignedSubs };
return { ...fileCollection, subtitles: unassignedSubs };
}
return { contents, videos, subtitles };
return fileCollection;
}
private parseVideo(video: any) {
private parseVideo(video: FileAttributes) {
const fileName = video.title.split('/').pop().replace(/\.(\w{2,4})$/, '');
const folderName = video.title.replace(/\/?[^/]+$/, '');
return {

183
src/node/consumer/src/lib/torrentEntries.js
View File

@@ -1,183 +0,0 @@
import { parse } from 'parse-torrent-title';
import { repository } from '../repository/database_repository';
import { TorrentType } from './enums/torrent_types';
import { PromiseHelpers } from './helpers/promises_helpers';
import { logger } from './services/logging_service';
import { metadataService } from './services/metadata_service';
import { parsingService } from './services/parsing_service';
import { torrentFileService } from './services/torrent_file_service';
import { torrentSubtitleService } from './services/torrent_subtitle_service';
export async function createTorrentEntry(torrent, overwrite = false) {
const titleInfo = parse(torrent.title);
if (!torrent.imdbId && torrent.type !== TorrentType.Anime) {
const imdbQuery = {
title: titleInfo.title,
year: titleInfo.year,
type: torrent.type
};
torrent.imdbId = await metadataService.getImdbId(imdbQuery)
.catch(() => undefined);
}
if (torrent.imdbId && torrent.imdbId.length < 9) {
// pad zeros to imdbId if missing
torrent.imdbId = 'tt' + torrent.imdbId.replace('tt', '').padStart(7, '0');
}
if (torrent.imdbId && torrent.imdbId.length > 9 && torrent.imdbId.startsWith('tt0')) {
// sanitize imdbId from redundant zeros
torrent.imdbId = torrent.imdbId.replace(/tt0+([0-9]{7,})$/, 'tt$1');
}
if (!torrent.kitsuId && torrent.type === TorrentType.Anime) {
const kitsuQuery = {
title: titleInfo.title,
year: titleInfo.year,
season: titleInfo.season,
};
torrent.kitsuId = await metadataService.getKitsuId(kitsuQuery)
.catch(() => undefined);
}
if (!torrent.imdbId && !torrent.kitsuId && !parsingService.isPackTorrent(torrent)) {
logger.warn(`imdbId or kitsuId not found: ${torrent.provider} ${torrent.title}`);
return;
}
const { contents, videos, subtitles } = await torrentFileService.parseTorrentFiles(torrent)
.then(torrentContents => overwrite ? overwriteExistingFiles(torrent, torrentContents) : torrentContents)
.then(torrentContents => torrentSubtitleService.assignSubtitles(torrentContents))
.catch(error => {
logger.warn(`Failed getting files for ${torrent.title}`, error.message);
return {};
});
if (!videos || !videos.length) {
logger.warn(`no video files found for ${torrent.provider} [${torrent.infoHash}] ${torrent.title}`);
return;
}
return repository.createTorrent({ ...torrent, contents, subtitles })
.then(() => PromiseHelpers.sequence(videos.map(video => () => repository.createFile(video))))
.then(() => logger.info(`Created ${torrent.provider} entry for [${torrent.infoHash}] ${torrent.title}`));
}
async function overwriteExistingFiles(torrent, torrentContents) {
const videos = torrentContents && torrentContents.videos;
if (videos && videos.length) {
const existingFiles = await repository.getFiles({ infoHash: videos[0].infoHash })
.then((existing) => existing
.reduce((map, next) => {
const fileIndex = next.fileIndex !== undefined ? next.fileIndex : null;
map[fileIndex] = (map[fileIndex] || []).concat(next);
return map;
}, {}))
.catch(() => undefined);
if (existingFiles && Object.keys(existingFiles).length) {
const overwrittenVideos = videos
.map(file => {
const mapping = videos.length === 1 && Object.keys(existingFiles).length === 1
? Object.values(existingFiles)[0]
: existingFiles[file.fileIndex !== undefined ? file.fileIndex : null];
if (mapping) {
const originalFile = mapping.shift();
return { id: originalFile.id, ...file };
}
return file;
});
return { ...torrentContents, videos: overwrittenVideos };
}
return torrentContents;
}
return Promise.reject(`No video files found for: ${torrent.title}`);
}
export async function createSkipTorrentEntry(torrent) {
return repository.createSkipTorrent(torrent);
}
export async function getStoredTorrentEntry(torrent) {
return repository.getSkipTorrent(torrent)
.catch(() => repository.getTorrent(torrent))
.catch(() => undefined);
}
export async function checkAndUpdateTorrent(torrent) {
const storedTorrent = torrent.dataValues
? torrent
: await repository.getTorrent(torrent).catch(() => undefined);
if (!storedTorrent) {
return false;
}
if (storedTorrent.provider === 'RARBG') {
return true;
}
if (storedTorrent.provider === 'KickassTorrents' && torrent.provider) {
storedTorrent.provider = torrent.provider;
storedTorrent.torrentId = torrent.torrentId;
}
if (!storedTorrent.languages && torrent.languages && storedTorrent.provider !== 'RARBG') {
storedTorrent.languages = torrent.languages;
await storedTorrent.save();
logger.debug(`Updated [${storedTorrent.infoHash}] ${storedTorrent.title} language to ${torrent.languages}`);
}
return createTorrentContents({ ...storedTorrent.get(), torrentLink: torrent.torrentLink })
.then(() => updateTorrentSeeders(torrent));
}
export async function createTorrentContents(torrent) {
if (torrent.opened) {
return;
}
const storedVideos = await repository.getFiles(torrent).catch(() => []);
if (!storedVideos || !storedVideos.length) {
return;
}
const notOpenedVideo = storedVideos.length === 1 && !Number.isInteger(storedVideos[0].fileIndex);
const imdbId = PromiseHelpers.mostCommonValue(storedVideos.map(stored => stored.imdbId));
const kitsuId = PromiseHelpers.mostCommonValue(storedVideos.map(stored => stored.kitsuId));
const { contents, videos, subtitles } = await torrentFileService.parseTorrentFiles({ ...torrent, imdbId, kitsuId })
.then(torrentContents => notOpenedVideo ? torrentContents : { ...torrentContents, videos: storedVideos })
.then(torrentContents => torrentSubtitleService.assignSubtitles(torrentContents))
.catch(error => {
logger.warn(`Failed getting contents for [${torrent.infoHash}] ${torrent.title}`, error.message);
return {};
});
if (!contents || !contents.length) {
return;
}
if (notOpenedVideo && videos.length === 1) {
// if both have a single video and stored one was not opened, update stored one to true metadata and use that
storedVideos[0].fileIndex = videos[0].fileIndex;
storedVideos[0].title = videos[0].title;
storedVideos[0].size = videos[0].size;
storedVideos[0].subtitles = videos[0].subtitles;
videos[0] = storedVideos[0];
}
// no videos available or more than one new videos were in the torrent
const shouldDeleteOld = notOpenedVideo && videos.every(video => !video.id);
return repository.createTorrent({ ...torrent, contents, subtitles })
.then(() => {
if (shouldDeleteOld) {
logger.debug(`Deleting old video for [${torrent.infoHash}] ${torrent.title}`)
return storedVideos[0].destroy();
}
return Promise.resolve();
})
.then(() => PromiseHelpers.sequence(videos.map(video => () => repository.createFile(video))))
.then(() => logger.info(`Created contents for ${torrent.provider} [${torrent.infoHash}] ${torrent.title}`))
.catch(error => logger.error(`Failed saving contents for [${torrent.infoHash}] ${torrent.title}`, error));
}
export async function updateTorrentSeeders(torrent) {
if (!(torrent.infoHash || (torrent.provider && torrent.torrentId)) || !Number.isInteger(torrent.seeders)) {
return torrent;
}
return repository.setTorrentSeeders(torrent, torrent.seeders)
.catch(error => {
logger.warn('Failed updating seeders:', error);
return undefined;
});
}

47
src/node/consumer/src/repository/database_repository.ts
View File

@@ -50,7 +50,7 @@ class DatabaseRepository {
}
}
public async getTorrent(torrent: Torrent): Promise<Torrent | null> {
public async getTorrent(torrent: TorrentAttributes): Promise<Torrent | null> {
const where = torrent.infoHash
? { infoHash: torrent.infoHash }
: { provider: torrent.provider, torrentId: torrent.torrentId };
@@ -69,25 +69,6 @@ class DatabaseRepository {
return await File.findAll({ where });
}
public async getUnprocessedIngestedTorrents(): Promise<IngestedTorrent[]> {
return await IngestedTorrent.findAll({
where: {
processed: false,
category: {
[Op.or]: ['tv', 'movies']
}
},
});
}
public async setIngestedTorrentsProcessed(ingestedTorrents: IngestedTorrent[]): Promise<void> {
await PromiseHelpers.sequence(ingestedTorrents
.map(ingestedTorrent => async () => {
ingestedTorrent.processed = true;
await ingestedTorrent.save();
}));
}
public async getTorrentsWithoutSize(): Promise<Torrent[]> {
return await Torrent.findAll({
where: literal(
@@ -148,8 +129,8 @@ class DatabaseRepository {
);
}
public async deleteTorrent(torrent: TorrentAttributes): Promise<number> {
return await Torrent.destroy({ where: { infoHash: torrent.infoHash } });
public async deleteTorrent(infoHash: string): Promise<number> {
return await Torrent.destroy({ where: { infoHash: infoHash } });
}
public async createFile(file: File): Promise<void> {
@@ -171,16 +152,16 @@ class DatabaseRepository {
}
}
public async getFiles(torrent: Torrent): Promise<File[]> {
return File.findAll({ where: { infoHash: torrent.infoHash } });
public async getFiles(infoHash: string): Promise<File[]> {
return File.findAll({ where: { infoHash: infoHash } });
}
public async getFilesBasedOnTitle(titleQuery: string): Promise<File[]> {
return File.findAll({ where: { title: { [Op.regexp]: `${titleQuery}` } } });
}
public async deleteFile(file: File): Promise<number> {
return File.destroy({ where: { id: file.id } });
public async deleteFile(id: number): Promise<number> {
return File.destroy({ where: { id: id } });
}
public async createSubtitles(infoHash: string, subtitles: Subtitle[]): Promise<void | Model<any, any>[]> {
@@ -209,8 +190,8 @@ class DatabaseRepository {
}
}
public async getSubtitles(torrent: Torrent): Promise<Subtitle[]> {
return Subtitle.findAll({ where: { infoHash: torrent.infoHash } });
public async getSubtitles(infoHash: string): Promise<Subtitle[]> {
return Subtitle.findAll({ where: { infoHash: infoHash } });
}
public async getUnassignedSubtitles(): Promise<Subtitle[]> {
@@ -224,14 +205,14 @@ class DatabaseRepository {
}
}
public async getContents(torrent: Torrent): Promise<Content[]> {
return Content.findAll({ where: { infoHash: torrent.infoHash } });
public async getContents(infoHash: string): Promise<Content[]> {
return Content.findAll({ where: { infoHash: infoHash } });
}
public async getSkipTorrent(torrent: Torrent): Promise<SkipTorrent> {
const result = await SkipTorrent.findByPk(torrent.infoHash);
public async getSkipTorrent(infoHash: string): Promise<SkipTorrent> {
const result = await SkipTorrent.findByPk(infoHash);
if (!result) {
throw new Error(`torrent not found: ${torrent.infoHash}`);
throw new Error(`torrent not found: ${infoHash}`);
}
return result.dataValues as SkipTorrent;
}

23
src/node/consumer/src/repository/interfaces/file_attributes.ts
View File

@@ -1,18 +1,21 @@
import {Optional} from "sequelize";
import {SubtitleAttributes} from "./subtitle_attributes";
import {ParseTorrentTitleResult} from "../../lib/interfaces/parse_torrent_title_result";
export interface FileAttributes {
export interface FileAttributes extends ParseTorrentTitleResult {
id?: number;
infoHash: string;
fileIndex: number;
title: string;
size: number;
imdbId: string;
imdbSeason: number;
imdbEpisode: number;
kitsuId: number;
kitsuEpisode: number;
infoHash?: string;
fileIndex?: number;
title?: string;
size?: number;
imdbId?: string;
imdbSeason?: number;
imdbEpisode?: number;
kitsuId?: number;
kitsuEpisode?: number;
subtitles?: SubtitleAttributes[];
path?: string;
isMovie?: boolean;
}
export interface FileCreationAttributes extends Optional<FileAttributes, 'fileIndex' | 'size' | 'imdbId' | 'imdbSeason' | 'imdbEpisode' | 'kitsuId' | 'kitsuEpisode'> {

28
src/node/consumer/src/repository/interfaces/torrent_attributes.ts
View File

@@ -5,20 +5,20 @@ import {FileAttributes} from "./file_attributes";
export interface TorrentAttributes {
infoHash: string;
provider: string;
torrentId: string;
title: string;
size: number;
type: string;
uploadDate: Date;
seeders: number;
trackers: string;
languages: string;
resolution: string;
reviewed: boolean;
opened: boolean;
contents: ContentAttributes[];
files: FileAttributes[];
provider?: string;
torrentId?: string;
title?: string;
size?: number;
type?: string;
uploadDate?: Date;
seeders?: number;
trackers?: string;
languages?: string;
resolution?: string;
reviewed?: boolean;
opened?: boolean;
contents?: ContentAttributes[];
files?: FileAttributes[];
subtitles?: SubtitleAttributes[];
}