brock/torrentio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

And thats what happens when you do a crapload of work with *.ts in .eslintignore... :/

Browse Source
This commit is contained in:
iPromKnight
2024-02-07 19:58:29 +00:00
committed by iPromKnight
parent 189fdd466e
commit 9aba1c13b1
55 changed files with 1987 additions and 576 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/node/consumer/.eslintignore
View File

@@ -1 +1 @@
*.ts
dist/

30
src/node/consumer/.eslintrc
View File

@@ -59,27 +59,13 @@
}
],
"prefer-destructuring": "error",
"@typescript-eslint/consistent-type-assertions": "off",
"@typescript-eslint/explicit-function-return-type": "off"
},
"overrides": [
{
"files": [
"*.ts",
"*.mts",
"*.cts",
"*.tsx"
],
"rules": {
"@typescript-eslint/explicit-function-return-type": "error",
"@typescript-eslint/consistent-type-assertions": [
"error",
{
"assertionStyle": "as",
"objectLiteralTypeAssertions": "never"
}
]
"@typescript-eslint/explicit-function-return-type": "error",
"@typescript-eslint/consistent-type-assertions": [
"error",
{
"assertionStyle": "as",
"objectLiteralTypeAssertions": "never"
}
}
]
]
}
}

15
src/node/consumer/.swcrc Normal file
View File

@@ -0,0 +1,15 @@
{
"jsc": {
"parser": {
"syntax": "typescript",
"tsx": false,
"decorators": true,
"dynamicImport": true
},
"target": "es2020",
"baseUrl": "."
},
"module": {
"type": "commonjs"
}
}

7
src/node/consumer/Dockerfile
View File

@@ -1,8 +1,11 @@
FROM node:lts-buster-slim as builder
RUN apt-get update && \
apt-get install -y git && \
rm -rf /var/lib/apt/lists/*
apt-get upgrade -y && \
apt-get install -y python3 make g++ && \
rm -rf /var/lib/apt/lists/*
RUN npm install -g npm@^8 && npm config set python /usr/bin/python3
WORKDIR /app

54
src/node/consumer/esbuild.js
View File

@@ -1,54 +0,0 @@
import {build} from "esbuild";
import {readFileSync, rmSync} from "fs";
const {devDependencies} = JSON.parse(readFileSync("./package.json", "utf8"));
const start = Date.now();
try {
const outdir = "dist";
rmSync(outdir, {recursive: true, force: true});
build({
bundle: true,
entryPoints: [
"./src/main.ts",
],
external: [...(devDependencies && Object.keys(devDependencies))],
keepNames: true,
minify: true,
outbase: "./src",
outdir,
outExtension: {
".js": ".cjs",
},
platform: "node",
plugins: [
{
name: "populate-import-meta",
setup: ({onLoad}) => {
onLoad({filter: new RegExp(`${import.meta.dirname}/src/.*.(js|ts)$`)}, args => {
const contents = readFileSync(args.path, "utf8");
const transformedContents = contents
.replace(/import\.meta/g, `{dirname:__dirname,filename:__filename}`)
.replace(/import\.meta\.filename/g, "__filename")
.replace(/import\.meta\.dirname/g, "__dirname");
return {contents: transformedContents, loader: "default"};
});
},
}
],
}).then(() => {
// biome-ignore lint/style/useTemplate: <explanation>
// eslint-disable-next-line no-undef
console.log("⚡ " + "\x1b[32m" + `Done in ${Date.now() - start}ms`);
});
} catch (e) {
// eslint-disable-next-line no-undef
console.log(e);
// eslint-disable-next-line no-undef
process.exit(1);
}

1391
src/node/consumer/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

16
src/node/consumer/package.json
View File

@@ -3,10 +3,14 @@
"version": "0.0.1",
"type": "module",
"scripts": {
"build": "node esbuild.js",
"dev": "tsx watch --ignore node_modules src/main.ts | pino-pretty",
"start": "node dist/main.cjs",
"lint": "npx eslint ./src --ext .ts,.js"
"build": "swc src -d dist",
"watch-compile": "swc src -w --out-dir dist",
"watch-dev": "nodemon --watch \"dist/**/*\" -e js ./dist/main.js",
"dev": "concurrently \"npm run watch-compile\" \"npm run watch-dev\"",
"start": "node dist/main.js",
"clean": "rm -rf dist",
"lint": "eslint --ext .ts src",
"lint:fix": "eslint --ext .ts src --fix"
},
"license": "MIT",
"dependencies": {
@@ -27,15 +31,19 @@
"reflect-metadata": "^0.2.1",
"sequelize": "^6.36.0",
"sequelize-typescript": "^2.1.6",
"utp-native": "^2.5.3",
"webtorrent": "^2.1.35"
},
"devDependencies": {
"@swc/cli": "^0.3.9",
"@swc/core": "^1.4.0",
"@types/amqplib": "^0.10.4",
"@types/magnet-uri": "^5.1.5",
"@types/node": "^20.11.16",
"@types/validator": "^13.11.8",
"@typescript-eslint/eslint-plugin": "^6.21.0",
"@typescript-eslint/parser": "^6.21.0",
"@types/webtorrent": "^0.109.7",
"esbuild": "^0.20.0",
"eslint": "^8.56.0",
"eslint-plugin-import": "^2.29.1",

8
src/node/consumer/src/lib/helpers/extension_helpers.ts
View File

@@ -47,19 +47,19 @@ const DISK_EXTENSIONS = [
];
export const ExtensionHelpers = {
isVideo(filename: string) {
isVideo(filename: string): boolean {
return this.isExtension(filename, VIDEO_EXTENSIONS);
},
isSubtitle(filename: string) {
isSubtitle(filename: string): boolean {
return this.isExtension(filename, SUBTITLE_EXTENSIONS);
},
isDisk(filename: string) {
isDisk(filename: string): boolean {
return this.isExtension(filename, DISK_EXTENSIONS);
},
isExtension(filename: string, extensions: string[]) {
isExtension(filename: string, extensions: string[]): boolean {
const extensionMatch = filename.match(/\.(\w{2,4})$/);
return extensionMatch !== null && extensions.includes(extensionMatch[1].toLowerCase());
}

18
src/node/consumer/src/lib/helpers/promises_helpers.ts
View File

@@ -1,24 +1,25 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
export const PromiseHelpers = {
sequence: async function (promises: (() => Promise<any>)[]) {
sequence: async function (promises: (() => Promise<any>)[]): Promise<any> {
return promises.reduce((promise, func) =>
promise.then(result => func().then(res => result.concat(res))), Promise.resolve([]));
},
first: async function (promises) {
return Promise.all(promises.map(p => {
return p.then((val) => Promise.reject(val), (err) => Promise.resolve(err));
first: async function (promises: any): Promise<any> {
return Promise.all(promises.map((p: any) => {
return p.then((val: any) => Promise.reject(val), (err: any) => Promise.resolve(err));
})).then(
(errors) => Promise.reject(errors),
(val) => Promise.resolve(val)
);
},
delay: async function (duration: number) {
delay: async function (duration: number): Promise<void> {
return new Promise<void>(resolve => setTimeout(() => resolve(), duration));
},
timeout: async function (timeoutMs: number, promise, message = 'Timed out') {
timeout: async function (timeoutMs: number, promise: any, message = 'Timed out'): Promise<any> {
return Promise.race([
promise,
new Promise(function (resolve, reject) {
@@ -29,7 +30,8 @@ export const PromiseHelpers = {
]);
},
mostCommonValue: function (array) {
mostCommonValue: function (array: any[]): any {
return array.sort((a, b) => array.filter(v => v === a).length - array.filter(v => v === b).length).pop();
}
};
};
/* eslint-enable @typescript-eslint/no-explicit-any */

4
src/node/consumer/src/lib/interfaces/cache_service.ts
View File

@@ -1,8 +1,10 @@
import {CacheMethod} from "../services/cache_service";
/* eslint-disable @typescript-eslint/no-explicit-any */
export interface ICacheService {
cacheWrapImdbId: (key: string, method: CacheMethod) => Promise<any>;
cacheWrapKitsuId: (key: string, method: CacheMethod) => Promise<any>;
cacheWrapMetadata: (id: string, method: CacheMethod) => Promise<any>;
cacheTrackers: (method: CacheMethod) => Promise<any>;
}
}
/* eslint-enable @typescript-eslint/no-explicit-any */

7
src/node/consumer/src/lib/interfaces/logging_service.ts
View File

@@ -1,9 +1,8 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
export interface ILoggingService {
info(message: string, ...args: any[]): void;
error(message: string, ...args: any[]): void;
debug(message: string, ...args: any[]): void;
warn(message: string, ...args: any[]): void;
}
}
/* eslint-enable @typescript-eslint/no-explicit-any */

5
src/node/consumer/src/lib/interfaces/parse_torrent_title_result.ts
View File

@@ -1,3 +1,5 @@
import {IFileAttributes} from "../repository/interfaces/file_attributes";
export interface IParseTorrentTitleResult {
title?: string;
date?: string;
@@ -28,4 +30,7 @@ export interface IParseTorrentTitleResult {
episode?: number;
languages?: string;
dubbed?: boolean;
videoFile?: IFileAttributes;
folderName?: string;
fileName?: string;
}

4
src/node/consumer/src/lib/interfaces/parsed_torrent.ts
View File

@@ -1,5 +1,5 @@
import {IParseTorrentTitleResult} from "./parse_torrent_title_result";
import {TorrentType} from "../enums/torrent_types";
import {IParseTorrentTitleResult} from "./parse_torrent_title_result";
import {ITorrentFileCollection} from "./torrent_file_collection";
export interface IParsedTorrent extends IParseTorrentTitleResult {
@@ -9,7 +9,7 @@ export interface IParsedTorrent extends IParseTorrentTitleResult {
kitsuId?: number;
trackers?: string;
provider?: string | null;
infoHash: string | null;
infoHash: string;
type: string | TorrentType;
uploadDate?: Date;
seeders?: number;

0
src/node/consumer/src/interfaces/process_torrents_job.ts → src/node/consumer/src/lib/interfaces/process_torrents_job.ts
View File

7
src/node/consumer/src/lib/interfaces/season_episode_map.ts Normal file
View File

@@ -0,0 +1,7 @@
import {ICommonVideoMetadata} from "./common_video_metadata";
export interface ISeasonEpisodeMap {
[season: number]: {
[episode: number]: ICommonVideoMetadata;
}
}

14
src/node/consumer/src/lib/interfaces/torrent_entries_service.ts
View File

@@ -1,18 +1,18 @@
import {ITorrentAttributes} from "../repository/interfaces/torrent_attributes";
import {SkipTorrent} from "../repository/models/skipTorrent";
import {Torrent} from "../repository/models/torrent";
import {IParsedTorrent} from "./parsed_torrent";
import {Torrent} from "../../repository/models/torrent";
import {ITorrentAttributes} from "../../repository/interfaces/torrent_attributes";
import {SkipTorrent} from "../../repository/models/skipTorrent";
export interface ITorrentEntriesService {
createTorrentEntry(torrent: IParsedTorrent, overwrite): Promise<void>;
createTorrentEntry(torrent: IParsedTorrent, overwrite: boolean): Promise<void>;
createSkipTorrentEntry(torrent: Torrent): Promise<[SkipTorrent, boolean]>;
createSkipTorrentEntry(torrent: Torrent): Promise<[SkipTorrent, boolean | null]>;
getStoredTorrentEntry(torrent: Torrent): Promise<SkipTorrent | Torrent>;
getStoredTorrentEntry(torrent: Torrent): Promise<Torrent | SkipTorrent | null | undefined>;
checkAndUpdateTorrent(torrent: IParsedTorrent): Promise<boolean>;
createTorrentContents(torrent: Torrent): Promise<void>;
updateTorrentSeeders(torrent: ITorrentAttributes): Promise<[number] | ITorrentAttributes>;
updateTorrentSeeders(torrent: ITorrentAttributes): Promise<[number] | undefined>;
}

6
src/node/consumer/src/lib/interfaces/torrent_file_collection.ts
View File

@@ -1,6 +1,6 @@
import {IContentAttributes} from "../../repository/interfaces/content_attributes";
import {IFileAttributes} from "../../repository/interfaces/file_attributes";
import {ISubtitleAttributes} from "../../repository/interfaces/subtitle_attributes";
import {IContentAttributes} from "../repository/interfaces/content_attributes";
import {IFileAttributes} from "../repository/interfaces/file_attributes";
import {ISubtitleAttributes} from "../repository/interfaces/subtitle_attributes";
export interface ITorrentFileCollection {
contents?: IContentAttributes[];

2
src/node/consumer/src/lib/interfaces/torrent_processing_service.ts
View File

@@ -1,4 +1,4 @@
import {IIngestedTorrentAttributes} from "../../repository/interfaces/ingested_torrent_attributes";
import {IIngestedTorrentAttributes} from "../repository/interfaces/ingested_torrent_attributes";
export interface ITorrentProcessingService {
processTorrentRecord(torrent: IIngestedTorrentAttributes): Promise<void>;

35
src/node/consumer/src/jobs/process_torrents_job.ts → src/node/consumer/src/lib/jobs/process_torrents_job.ts
View File

@@ -1,12 +1,12 @@
import client, {Channel, Connection, ConsumeMessage, Options} from 'amqplib'
import {IIngestedRabbitMessage, IIngestedRabbitTorrent} from "../lib/interfaces/ingested_rabbit_message";
import {IIngestedTorrentAttributes} from "../repository/interfaces/ingested_torrent_attributes";
import {configurationService} from '../lib/services/configuration_service';
import {inject, injectable} from "inversify";
import {IocTypes} from "../lib/models/ioc_types";
import {ITorrentProcessingService} from "../lib/interfaces/torrent_processing_service";
import {ILoggingService} from "../lib/interfaces/logging_service";
import {IIngestedRabbitMessage, IIngestedRabbitTorrent} from "../interfaces/ingested_rabbit_message";
import {ILoggingService} from "../interfaces/logging_service";
import {IProcessTorrentsJob} from "../interfaces/process_torrents_job";
import {ITorrentProcessingService} from "../interfaces/torrent_processing_service";
import {IocTypes} from "../models/ioc_types";
import {IIngestedTorrentAttributes} from "../repository/interfaces/ingested_torrent_attributes";
import {configurationService} from '../services/configuration_service';
@injectable()
export class ProcessTorrentsJob implements IProcessTorrentsJob {
@@ -21,7 +21,7 @@ export class ProcessTorrentsJob implements IProcessTorrentsJob {
this.logger = logger;
}
public listenToQueue = async () => {
public listenToQueue = async (): Promise<void> => {
if (!configurationService.jobConfig.JOBS_ENABLED) {
return;
}
@@ -34,26 +34,27 @@ export class ProcessTorrentsJob implements IProcessTorrentsJob {
this.logger.error('Failed to connect and setup channel', error);
}
}
private processMessage = (msg: ConsumeMessage) => {
private processMessage = (msg: ConsumeMessage | null): Promise<void> => {
const ingestedTorrent: IIngestedTorrentAttributes = this.getMessageAsJson(msg);
return this.torrentProcessingService.processTorrentRecord(ingestedTorrent);
};
private getMessageAsJson = (msg: ConsumeMessage): IIngestedTorrentAttributes => {
private getMessageAsJson = (msg: ConsumeMessage | null): IIngestedTorrentAttributes => {
const content = msg?.content.toString('utf8') ?? "{}";
const receivedObject: IIngestedRabbitMessage = JSON.parse(content);
const receivedTorrent: IIngestedRabbitTorrent = receivedObject.message;
return {...receivedTorrent, info_hash: receivedTorrent.infoHash};
};
private assertAndConsumeQueue = async (channel: Channel) => {
private assertAndConsumeQueue = async (channel: Channel): Promise<void> => {
this.logger.info('Worker is running! Waiting for new torrents...');
const ackMsg = async (msg: ConsumeMessage) => {
try {
await this.processMessage(msg);
channel.ack(msg);
} catch (error) {
this.logger.error('Failed processing torrent', error);
}
const ackMsg = async (msg: ConsumeMessage | null): Promise<void> => {
await this.processMessage(msg)
.then(() => this.logger.info('Processed torrent'))
.then(() => msg && channel.ack(msg))
.catch((error) => this.logger.error('Failed to process torrent', error));
}
try {

8
src/node/consumer/src/lib/models/composition_root.ts
View File

@@ -1,8 +1,8 @@
import {inject, injectable} from "inversify";
import {IDatabaseRepository} from "../../repository/interfaces/database_repository";
import {ITrackerService} from "../interfaces/tracker_service";
import {IProcessTorrentsJob} from "../../interfaces/process_torrents_job";
import {ICompositionalRoot} from "../interfaces/composition_root";
import {IProcessTorrentsJob} from "../interfaces/process_torrents_job";
import {ITrackerService} from "../interfaces/tracker_service";
import {IDatabaseRepository} from "../repository/interfaces/database_repository";
import {IocTypes} from "./ioc_types";
@injectable()
@@ -19,7 +19,7 @@ export class CompositionalRoot implements ICompositionalRoot {
this.processTorrentsJob = processTorrentsJob;
}
start = async () => {
start = async (): Promise<void> => {
await this.trackerService.getTrackers();
await this.databaseRepository.connect();
await this.processTorrentsJob.listenToQueue();

2
src/node/consumer/src/lib/models/configuration/cache_config.ts
View File

@@ -9,7 +9,7 @@ export const cacheConfig = {
NO_CACHE: BooleanHelpers.parseBool(process.env.NO_CACHE, false),
COLLECTION_NAME: process.env.MONGODB_COLLECTION || 'knightcrawler_consumer_collection',
get MONGO_URI() {
get MONGO_URI(): string {
return `mongodb://${this.MONGO_INITDB_ROOT_USERNAME}:${this.MONGO_INITDB_ROOT_PASSWORD}@${this.MONGODB_HOST}:${this.MONGODB_PORT}/${this.MONGODB_DB}?authSource=admin`;
}
};

2
src/node/consumer/src/lib/models/configuration/database_config.ts
View File

@@ -8,7 +8,7 @@ export const databaseConfig = {
POSTGRES_PASSWORD: process.env.POSTGRES_PASSWORD || 'postgres',
AUTO_CREATE_AND_APPLY_MIGRATIONS: BooleanHelpers.parseBool(process.env.AUTO_CREATE_AND_APPLY_MIGRATIONS, false),
get POSTGRES_URI() {
get POSTGRES_URI(): string {
return `postgres://${this.POSTGRES_USER}:${this.POSTGRES_PASSWORD}@${this.POSTGRES_HOST}:${this.POSTGRES_PORT}/${this.POSTGRES_DB}`;
}
};

2
src/node/consumer/src/lib/models/configuration/torrent_config.ts
View File

@@ -1,5 +1,5 @@
export const torrentConfig = {
MAX_CONNECTIONS_PER_TORRENT: parseInt(process.env.MAX_SINGLE_TORRENT_CONNECTIONS || "20", 10),
MAX_CONNECTIONS_PER_TORRENT: parseInt(process.env.MAX_CONNECTIONS_PER_TORRENT || "20", 10),
MAX_CONNECTIONS_OVERALL: parseInt(process.env.MAX_CONNECTIONS_OVERALL || "100", 10),
TIMEOUT: parseInt(process.env.TORRENT_TIMEOUT || "30000", 10)
};

21
src/node/consumer/src/lib/models/inversify_config.ts
View File

@@ -1,30 +1,29 @@
import "reflect-metadata"; // required
import {Container} from "inversify";
import {IocTypes} from "./ioc_types";
import {ICacheService} from "../interfaces/cache_service";
import {ICompositionalRoot} from "../interfaces/composition_root";
import {ILoggingService} from "../interfaces/logging_service";
import {IMetadataService} from "../interfaces/metadata_service";
import {IProcessTorrentsJob} from "../interfaces/process_torrents_job";
import {ITorrentDownloadService} from "../interfaces/torrent_download_service";
import {ITorrentEntriesService} from "../interfaces/torrent_entries_service";
import {ITorrentFileService} from "../interfaces/torrent_file_service";
import {ITorrentProcessingService} from "../interfaces/torrent_processing_service";
import {ITorrentSubtitleService} from "../interfaces/torrent_subtitle_service";
import {ITorrentEntriesService} from "../interfaces/torrent_entries_service";
import {ITorrentDownloadService} from "../interfaces/torrent_download_service";
import {ITrackerService} from "../interfaces/tracker_service";
import {IProcessTorrentsJob} from "../../interfaces/process_torrents_job";
import {ICompositionalRoot} from "../interfaces/composition_root";
import {IDatabaseRepository} from "../../repository/interfaces/database_repository";
import {CompositionalRoot} from "./composition_root";
import {ProcessTorrentsJob} from "../jobs/process_torrents_job";
import {DatabaseRepository} from "../repository/database_repository";
import {IDatabaseRepository} from "../repository/interfaces/database_repository";
import {CacheService} from "../services/cache_service";
import {LoggingService} from "../services/logging_service";
import {MetadataService} from "../services/metadata_service";
import {TorrentDownloadService} from "../services/torrent_download_service";
import {TorrentEntriesService} from "../services/torrent_entries_service";
import {TorrentProcessingService} from "../services/torrent_processing_service";
import {TorrentFileService} from "../services/torrent_file_service";
import {TorrentProcessingService} from "../services/torrent_processing_service";
import {TorrentSubtitleService} from "../services/torrent_subtitle_service";
import {TrackerService} from "../services/tracker_service";
import {DatabaseRepository} from "../../repository/database_repository";
import {ProcessTorrentsJob} from "../../jobs/process_torrents_job";
import {CompositionalRoot} from "./composition_root";
import {IocTypes} from "./ioc_types";
const serviceContainer = new Container();

68
src/node/consumer/src/repository/database_repository.ts → src/node/consumer/src/lib/repository/database_repository.ts
View File

@@ -1,24 +1,24 @@
import {inject, injectable} from "inversify";
import moment from 'moment';
import {literal, Op, WhereOptions} from "sequelize";
import {Model, Sequelize} from 'sequelize-typescript';
import {configurationService} from '../lib/services/configuration_service';
import {PromiseHelpers} from '../lib/helpers/promises_helpers';
import {Provider} from "./models/provider";
import {File} from "./models/file";
import {Torrent} from "./models/torrent";
import {IngestedTorrent} from "./models/ingestedTorrent";
import {Subtitle} from "./models/subtitle";
import {Content} from "./models/content";
import {SkipTorrent} from "./models/skipTorrent";
import {IFileAttributes, IFileCreationAttributes} from "./interfaces/file_attributes";
import {ITorrentAttributes, ITorrentCreationAttributes} from "./interfaces/torrent_attributes";
import {IngestedPage} from "./models/ingestedPage";
import {ILoggingService} from "../lib/interfaces/logging_service";
import {IocTypes} from "../lib/models/ioc_types";
import {inject, injectable} from "inversify";
import {IDatabaseRepository} from "./interfaces/database_repository";
import {PromiseHelpers} from '../helpers/promises_helpers';
import {ILoggingService} from "../interfaces/logging_service";
import {IocTypes} from "../models/ioc_types";
import {configurationService} from '../services/configuration_service';
import {IContentCreationAttributes} from "./interfaces/content_attributes";
import {ISubtitleCreationAttributes} from "./interfaces/subtitle_attributes";
import {IDatabaseRepository} from "./interfaces/database_repository";
import {IFileAttributes, IFileCreationAttributes} from "./interfaces/file_attributes";
import {ISubtitleAttributes, ISubtitleCreationAttributes} from "./interfaces/subtitle_attributes";
import {ITorrentAttributes, ITorrentCreationAttributes} from "./interfaces/torrent_attributes";
import {Content} from "./models/content";
import {File} from "./models/file";
import {IngestedPage} from "./models/ingestedPage";
import {IngestedTorrent} from "./models/ingestedTorrent";
import {Provider} from "./models/provider";
import {SkipTorrent} from "./models/skipTorrent";
import {Subtitle} from "./models/subtitle";
import {Torrent} from "./models/torrent";
@injectable()
export class DatabaseRepository implements IDatabaseRepository {
@@ -33,6 +33,7 @@ export class DatabaseRepository implements IDatabaseRepository {
SkipTorrent,
IngestedTorrent,
IngestedPage];
private logger: ILoggingService;
constructor(@inject(IocTypes.ILoggingService) logger: ILoggingService) {
@@ -40,7 +41,7 @@ export class DatabaseRepository implements IDatabaseRepository {
this.database = this.createDatabase();
}
public connect = async () => {
public connect = async (): Promise<void> => {
try {
await this.database.sync({alter: configurationService.databaseConfig.AUTO_CREATE_AND_APPLY_MIGRATIONS});
} catch (error) {
@@ -50,7 +51,7 @@ export class DatabaseRepository implements IDatabaseRepository {
}
};
public getProvider = async (provider: Provider) => {
public getProvider = async (provider: Provider): Promise<Provider> => {
try {
const [result] = await Provider.findOrCreate({where: {name: {[Op.eq]: provider.name}}, defaults: provider});
return result;
@@ -121,7 +122,7 @@ export class DatabaseRepository implements IDatabaseRepository {
await this.createSubtitles(torrent.infoHash, torrent.subtitles);
} catch (error) {
this.logger.error(`Failed to create torrent: ${torrent.infoHash}`);
this.logger.debug(error);
this.logger.debug("Error: ", error);
}
};
@@ -145,13 +146,13 @@ export class DatabaseRepository implements IDatabaseRepository {
if (operatingFile.dataValues) {
await operatingFile.save();
} else {
await File.upsert(operatingFile);
await File.upsert(file);
}
await this.upsertSubtitles(operatingFile, operatingFile.subtitles);
await this.upsertSubtitles(operatingFile, file.subtitles);
} else {
if (operatingFile.subtitles && operatingFile.subtitles.length) {
operatingFile.subtitles = operatingFile.subtitles.map(subtitle => {
subtitle.title = subtitle.path;
subtitle.title = subtitle.path || '';
return subtitle;
});
}
@@ -159,7 +160,7 @@ export class DatabaseRepository implements IDatabaseRepository {
}
} catch (error) {
this.logger.error(`Failed to create file: ${file.infoHash}`);
this.logger.debug(error);
this.logger.debug("Error: ", error);
}
};
@@ -169,25 +170,26 @@ export class DatabaseRepository implements IDatabaseRepository {
public deleteFile = async (id: number): Promise<number> => File.destroy({where: {id: id}});
public createSubtitles = async (infoHash: string, subtitles: ISubtitleCreationAttributes[]): Promise<void | Model<any, any>[]> => {
public createSubtitles = async (infoHash: string, subtitles: ISubtitleCreationAttributes[] | undefined): Promise<void | Model<ISubtitleAttributes, ISubtitleCreationAttributes>[]> => {
if (subtitles && subtitles.length) {
return Subtitle.bulkCreate(subtitles.map(subtitle => ({infoHash, title: subtitle.path, ...subtitle})));
return Subtitle.bulkCreate(subtitles.map(subtitle => ({...subtitle, infoHash: infoHash, title: subtitle.path})));
}
return Promise.resolve();
};
public upsertSubtitles = async (file: File, subtitles: Subtitle[]): Promise<void> => {
public upsertSubtitles = async (file: File, subtitles: ISubtitleCreationAttributes[] | undefined): Promise<void> => {
if (file.id && subtitles && subtitles.length) {
await PromiseHelpers.sequence(subtitles
.map(subtitle => {
subtitle.fileId = file.id;
subtitle.infoHash = subtitle.infoHash || file.infoHash;
subtitle.title = subtitle.title || subtitle.path;
subtitle.title = subtitle.title || subtitle.path || '';
return subtitle;
})
.map(subtitle => async () => {
if (subtitle.dataValues) {
await subtitle.save();
const operatingInstance = Subtitle.build(subtitle);
if (operatingInstance.dataValues) {
await operatingInstance.save();
} else {
await Subtitle.create(subtitle);
}
@@ -199,9 +201,9 @@ export class DatabaseRepository implements IDatabaseRepository {
public getUnassignedSubtitles = async (): Promise<Subtitle[]> => Subtitle.findAll({where: {fileId: null}});
public createContents = async (infoHash: string, contents: IContentCreationAttributes[]): Promise<void> => {
public createContents = async (infoHash: string, contents: IContentCreationAttributes[] | undefined): Promise<void> => {
if (contents && contents.length) {
await Content.bulkCreate(contents.map(content => ({infoHash, ...content})), {ignoreDuplicates: true});
await Content.bulkCreate(contents.map(content => ({...content, infoHash})), {ignoreDuplicates: true});
await Torrent.update({opened: true}, {where: {infoHash: infoHash}, silent: true});
}
};
@@ -216,7 +218,7 @@ export class DatabaseRepository implements IDatabaseRepository {
return result.dataValues as SkipTorrent;
};
public createSkipTorrent = async (torrent: ITorrentCreationAttributes): Promise<[SkipTorrent, boolean]> => SkipTorrent.upsert({infoHash: torrent.infoHash});
public createSkipTorrent = async (torrent: ITorrentCreationAttributes): Promise<[SkipTorrent, boolean | null]> => SkipTorrent.upsert({infoHash: torrent.infoHash});
private createDatabase = (): Sequelize => {
const newDatabase = new Sequelize(

0
src/node/consumer/src/repository/interfaces/content_attributes.ts → src/node/consumer/src/lib/repository/interfaces/content_attributes.ts
View File

24
src/node/consumer/src/repository/interfaces/database_repository.ts → src/node/consumer/src/lib/repository/interfaces/database_repository.ts
View File

@@ -1,15 +1,15 @@
import {Provider} from "../models/provider";
import {WhereOptions} from "sequelize";
import {ITorrentAttributes, ITorrentCreationAttributes} from "./torrent_attributes";
import {Torrent} from "../models/torrent";
import {IFileAttributes, IFileCreationAttributes} from "./file_attributes";
import {File} from "../models/file";
import {Subtitle} from "../models/subtitle";
import {Model} from "sequelize-typescript";
import {Content} from "../models/content";
import {File} from "../models/file";
import {Provider} from "../models/provider";
import {SkipTorrent} from "../models/skipTorrent";
import {ISubtitleCreationAttributes} from "./subtitle_attributes";
import {Subtitle} from "../models/subtitle";
import {Torrent} from "../models/torrent";
import {IContentCreationAttributes} from "./content_attributes";
import {IFileAttributes, IFileCreationAttributes} from "./file_attributes";
import {ISubtitleAttributes, ISubtitleCreationAttributes} from "./subtitle_attributes";
import {ITorrentAttributes, ITorrentCreationAttributes} from "./torrent_attributes";
export interface IDatabaseRepository {
connect(): Promise<void>;
@@ -26,9 +26,9 @@ export interface IDatabaseRepository {
getTorrentsWithoutSize(): Promise<Torrent[]>;
getUpdateSeedersTorrents(limit): Promise<Torrent[]>;
getUpdateSeedersTorrents(limit: number): Promise<Torrent[]>;
getUpdateSeedersNewTorrents(limit): Promise<Torrent[]>;
getUpdateSeedersNewTorrents(limit: number): Promise<Torrent[]>;
getNoContentsTorrents(): Promise<Torrent[]>;
@@ -46,9 +46,9 @@ export interface IDatabaseRepository {
deleteFile(id: number): Promise<number>;
createSubtitles(infoHash: string, subtitles: ISubtitleCreationAttributes[]): Promise<void | Model<any, any>[]>;
createSubtitles(infoHash: string, subtitles: ISubtitleCreationAttributes[]): Promise<void | Model<ISubtitleAttributes, ISubtitleCreationAttributes>[]>;
upsertSubtitles(file: File, subtitles: Subtitle[]): Promise<void>;
upsertSubtitles(file: File, subtitles: ISubtitleCreationAttributes[] | undefined): Promise<void>;
getSubtitles(infoHash: string): Promise<Subtitle[]>;
@@ -60,5 +60,5 @@ export interface IDatabaseRepository {
getSkipTorrent(infoHash: string): Promise<SkipTorrent>;
createSkipTorrent(torrent: ITorrentCreationAttributes): Promise<[SkipTorrent, boolean]>;
createSkipTorrent(torrent: ITorrentCreationAttributes): Promise<[SkipTorrent, boolean | null]>;
}

4
src/node/consumer/src/repository/interfaces/file_attributes.ts → src/node/consumer/src/lib/repository/interfaces/file_attributes.ts
View File

@@ -1,12 +1,12 @@
import {Optional} from "sequelize";
import {IParseTorrentTitleResult} from "../../interfaces/parse_torrent_title_result";
import {ISubtitleAttributes} from "./subtitle_attributes";
import {IParseTorrentTitleResult} from "../../lib/interfaces/parse_torrent_title_result";
export interface IFileAttributes extends IParseTorrentTitleResult {
id?: number;
infoHash?: string;
fileIndex?: number;
title?: string;
title: string;
size?: number;
imdbId?: string;
imdbSeason?: number;

0
src/node/consumer/src/repository/interfaces/ingested_page_attributes.ts → src/node/consumer/src/lib/repository/interfaces/ingested_page_attributes.ts
View File

0
src/node/consumer/src/repository/interfaces/ingested_torrent_attributes.ts → src/node/consumer/src/lib/repository/interfaces/ingested_torrent_attributes.ts
View File

0
src/node/consumer/src/repository/interfaces/provider_attributes.ts → src/node/consumer/src/lib/repository/interfaces/provider_attributes.ts
View File

0
src/node/consumer/src/repository/interfaces/skip_torrent_attributes.ts → src/node/consumer/src/lib/repository/interfaces/skip_torrent_attributes.ts
View File

2
src/node/consumer/src/repository/interfaces/subtitle_attributes.ts → src/node/consumer/src/lib/repository/interfaces/subtitle_attributes.ts
View File

@@ -3,7 +3,7 @@ import {Optional} from "sequelize";
export interface ISubtitleAttributes {
infoHash: string;
fileIndex: number;
fileId?: number;
fileId?: number | null;
title: string;
path: string;
}

4
src/node/consumer/src/repository/interfaces/torrent_attributes.ts → src/node/consumer/src/lib/repository/interfaces/torrent_attributes.ts
View File

@@ -1,11 +1,11 @@
import {Optional} from "sequelize";
import {IContentAttributes} from "./content_attributes";
import {ISubtitleAttributes} from "./subtitle_attributes";
import {IFileAttributes} from "./file_attributes";
import {ISubtitleAttributes} from "./subtitle_attributes";
export interface ITorrentAttributes {
infoHash: string;
provider?: string;
provider?: string | null;
torrentId?: string;
title?: string;
size?: number;

2
src/node/consumer/src/repository/models/content.ts → src/node/consumer/src/lib/repository/models/content.ts
View File

@@ -18,5 +18,5 @@ export class Content extends Model<IContentAttributes, IContentCreationAttribute
declare size: number;
@BelongsTo(() => Torrent, {constraints: false, foreignKey: 'infoHash'})
torrent: Torrent;
torrent?: Torrent;
}

4
src/node/consumer/src/repository/models/file.ts → src/node/consumer/src/lib/repository/models/file.ts
View File

@@ -1,7 +1,7 @@
import {BelongsTo, Column, DataType, ForeignKey, HasMany, Model, Table} from 'sequelize-typescript';
import {IFileAttributes, IFileCreationAttributes} from "../interfaces/file_attributes";
import {Torrent} from "./torrent";
import {Subtitle} from "./subtitle";
import {Torrent} from "./torrent";
const indexes = [
{
@@ -55,5 +55,5 @@ export class File extends Model<IFileAttributes, IFileCreationAttributes> {
declare subtitles?: Subtitle[];
@BelongsTo(() => Torrent, {constraints: false, foreignKey: 'infoHash'})
torrent: Torrent;
torrent?: Torrent;
}

0
src/node/consumer/src/repository/models/ingestedPage.ts → src/node/consumer/src/lib/repository/models/ingestedPage.ts
View File

0
src/node/consumer/src/repository/models/ingestedTorrent.ts → src/node/consumer/src/lib/repository/models/ingestedTorrent.ts
View File

0
src/node/consumer/src/repository/models/provider.ts → src/node/consumer/src/lib/repository/models/provider.ts
View File

0
src/node/consumer/src/repository/models/skipTorrent.ts → src/node/consumer/src/lib/repository/models/skipTorrent.ts
View File

4
src/node/consumer/src/repository/models/subtitle.ts → src/node/consumer/src/lib/repository/models/subtitle.ts
View File

@@ -32,7 +32,7 @@ export class Subtitle extends Model<ISubtitleAttributes, ISubtitleCreationAttrib
declare title: string;
@BelongsTo(() => File, {constraints: false, foreignKey: 'fileId'})
file: File;
file?: File;
path: string;
path?: string;
}

0
src/node/consumer/src/repository/models/torrent.ts → src/node/consumer/src/lib/repository/models/torrent.ts
View File

35
src/node/consumer/src/lib/services/cache_service.ts
View File

@@ -1,12 +1,12 @@
import {Cache, createCache, MemoryCache, memoryStore, Store} from 'cache-manager';
import {mongoDbStore} from '@tirke/node-cache-manager-mongodb'
import {configurationService} from './configuration_service';
import {Cache, createCache, MemoryCache, memoryStore} from 'cache-manager';
import {inject, injectable} from "inversify";
import {CacheType} from "../enums/cache_types";
import {ICacheOptions} from "../interfaces/cache_options";
import {ICacheService} from "../interfaces/cache_service";
import {inject, injectable} from "inversify";
import {IocTypes} from "../models/ioc_types";
import {ILoggingService} from "../interfaces/logging_service";
import {IocTypes} from "../models/ioc_types";
import {configurationService} from './configuration_service';
const GLOBAL_KEY_PREFIX = 'knightcrawler-consumer';
const IMDB_ID_PREFIX = `${GLOBAL_KEY_PREFIX}|imdb_id`;
@@ -18,13 +18,14 @@ const GLOBAL_TTL: number = Number(process.env.METADATA_TTL) || 7 * 24 * 60 * 60;
const MEMORY_TTL: number = Number(process.env.METADATA_TTL) || 2 * 60 * 60; // 2 hours
const TRACKERS_TTL: number = 2 * 24 * 60 * 60; // 2 days
/* eslint-disable-next-line @typescript-eslint/no-explicit-any */
export type CacheMethod = () => any;
@injectable()
export class CacheService implements ICacheService {
private logger: ILoggingService;
private readonly memoryCache: MemoryCache;
private readonly remoteCache: Cache<Store> | MemoryCache;
private readonly memoryCache: MemoryCache | undefined;
private readonly remoteCache: Cache | MemoryCache | undefined;
constructor(@inject(IocTypes.ILoggingService) logger: ILoggingService) {
this.logger = logger;
@@ -37,24 +38,24 @@ export class CacheService implements ICacheService {
this.remoteCache = this.initiateRemoteCache();
}
public cacheWrapImdbId = (key: string, method: CacheMethod): Promise<any> =>
public cacheWrapImdbId = (key: string, method: CacheMethod): Promise<CacheMethod> =>
this.cacheWrap(CacheType.MongoDb, `${IMDB_ID_PREFIX}:${key}`, method, {ttl: GLOBAL_TTL});
public cacheWrapKitsuId = (key: string, method: CacheMethod): Promise<any> =>
public cacheWrapKitsuId = (key: string, method: CacheMethod): Promise<CacheMethod> =>
this.cacheWrap(CacheType.MongoDb, `${KITSU_ID_PREFIX}:${key}`, method, {ttl: GLOBAL_TTL});
public cacheWrapMetadata = (id: string, method: CacheMethod): Promise<any> =>
public cacheWrapMetadata = (id: string, method: CacheMethod): Promise<CacheMethod> =>
this.cacheWrap(CacheType.Memory, `${METADATA_PREFIX}:${id}`, method, {ttl: MEMORY_TTL});
public cacheTrackers = (method: CacheMethod): Promise<any> =>
public cacheTrackers = (method: CacheMethod): Promise<CacheMethod> =>
this.cacheWrap(CacheType.Memory, `${TRACKERS_KEY_PREFIX}`, method, {ttl: TRACKERS_TTL});
private initiateMemoryCache = () =>
private initiateMemoryCache = (): MemoryCache =>
createCache(memoryStore(), {
ttl: MEMORY_TTL
});
private initiateMongoCache = () => {
private initiateMongoCache = (): Cache => {
const store = mongoDbStore({
collectionName: configurationService.cacheConfig.COLLECTION_NAME,
ttl: GLOBAL_TTL,
@@ -70,28 +71,28 @@ export class CacheService implements ICacheService {
});
}
private initiateRemoteCache = (): Cache => {
private initiateRemoteCache = (): Cache | undefined => {
if (configurationService.cacheConfig.NO_CACHE) {
this.logger.debug('Cache is disabled');
return null;
return undefined;
}
return configurationService.cacheConfig.MONGO_URI ? this.initiateMongoCache() : this.initiateMemoryCache();
}
private getCacheType = (cacheType: CacheType): MemoryCache | Cache<Store> => {
private getCacheType = (cacheType: CacheType): MemoryCache | Cache | undefined => {
switch (cacheType) {
case CacheType.Memory:
return this.memoryCache;
case CacheType.MongoDb:
return this.remoteCache;
default:
return null;
return undefined;
}
}
private cacheWrap = async (
cacheType: CacheType, key: string, method: CacheMethod, options: ICacheOptions): Promise<any> => {
cacheType: CacheType, key: string, method: CacheMethod, options: ICacheOptions): Promise<CacheMethod> => {
const cache = this.getCacheType(cacheType);
if (configurationService.cacheConfig.NO_CACHE || !cache) {

6
src/node/consumer/src/lib/services/configuration_service.ts
View File

@@ -1,10 +1,10 @@
import {rabbitConfig} from "../models/configuration/rabbit_config";
import {cacheConfig} from "../models/configuration/cache_config";
import {cacheConfig} from "../models/configuration/cache_config";
import {databaseConfig} from "../models/configuration/database_config";
import {jobConfig} from "../models/configuration/job_config";
import {metadataConfig} from "../models/configuration/metadata_config";
import {trackerConfig} from "../models/configuration/tracker_config";
import {rabbitConfig} from "../models/configuration/rabbit_config";
import {torrentConfig} from "../models/configuration/torrent_config";
import {trackerConfig} from "../models/configuration/tracker_config";
export const configurationService = {
rabbitConfig: rabbitConfig,

6
src/node/consumer/src/lib/services/logging_service.ts
View File

@@ -1,7 +1,8 @@
import {injectable} from "inversify";
import {Logger, pino} from "pino";
import {ILoggingService} from "../interfaces/logging_service";
import {injectable} from "inversify";
/* eslint-disable @typescript-eslint/no-explicit-any */
@injectable()
export class LoggingService implements ILoggingService {
private readonly logger: Logger;
@@ -27,4 +28,5 @@ export class LoggingService implements ILoggingService {
public warn = (message: string, ...args: any[]): void => {
this.logger.warn(message, args);
};
}
}
/* eslint-enable @typescript-eslint/no-explicit-any */

88
src/node/consumer/src/lib/services/metadata_service.ts
View File

@@ -1,17 +1,20 @@
import axios, {AxiosResponse} from 'axios';
import {ResultTypes, search} from 'google-sr';
import {inject, injectable} from "inversify";
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-expect-error
import nameToImdb from 'name-to-imdb';
import {TorrentType} from '../enums/torrent_types';
import {IMetadataResponse} from "../interfaces/metadata_response";
import {ICacheService} from "../interfaces/cache_service";
import {ICinemetaJsonResponse} from "../interfaces/cinemeta_metadata";
import {ICommonVideoMetadata} from "../interfaces/common_video_metadata";
import {IKitsuCatalogJsonResponse} from "../interfaces/kitsu_catalog_metadata";
import {IKitsuJsonResponse} from "../interfaces/kitsu_metadata";
import {IMetaDataQuery} from "../interfaces/metadata_query";
import {IKitsuCatalogJsonResponse} from "../interfaces/kitsu_catalog_metadata";
import {IMetadataResponse} from "../interfaces/metadata_response";
import {IMetadataService} from "../interfaces/metadata_service";
import {inject, injectable} from "inversify";
import {IocTypes} from "../models/ioc_types";
import {ICacheService} from "../interfaces/cache_service";
const CINEMETA_URL = 'https://v3-cinemeta.strem.io';
const KITSU_URL = 'https://anime-kitsu.strem.fun';
@@ -25,10 +28,10 @@ export class MetadataService implements IMetadataService {
this.cacheService = cacheService;
}
public getKitsuId = async (info: IMetaDataQuery): Promise<string | Error> => {
const title = this.escapeTitle(info.title.replace(/\s\|\s.*/, ''));
public getKitsuId = async (info: IMetaDataQuery): Promise<number | Error> => {
const title = this.escapeTitle(info.title!.replace(/\s\|\s.*/, ''));
const year = info.year ? ` ${info.year}` : '';
const season = info.season > 1 ? ` S${info.season}` : '';
const season = info.season || 0 > 1 ? ` S${info.season}` : '';
const key = `${title}${year}${season}`;
const query = encodeURIComponent(key);
@@ -45,7 +48,7 @@ export class MetadataService implements IMetadataService {
};
public getImdbId = async (info: IMetaDataQuery): Promise<string | undefined> => {
const name = this.escapeTitle(info.title);
const name = this.escapeTitle(info.title!);
const year = info.year || (info.date && info.date.slice(0, 4));
const key = `${name}_${year || 'NA'}_${info.type}`;
const query = `${name} ${year || ''} ${info.type} imdb`;
@@ -102,7 +105,7 @@ export class MetadataService implements IMetadataService {
.trim();
private requestMetadata = async (url: string): Promise<IMetadataResponse> => {
let response: AxiosResponse<any, any> = await axios.get(url, {timeout: TIMEOUT});
const response: AxiosResponse = await axios.get(url, {timeout: TIMEOUT});
let result: IMetadataResponse;
const body = response.data;
if ('kitsu_id' in body.meta) {
@@ -117,14 +120,14 @@ export class MetadataService implements IMetadataService {
};
private handleCinemetaResponse = (body: ICinemetaJsonResponse): IMetadataResponse => ({
imdbId: parseInt(body.meta.imdb_id),
type: body.meta.type,
title: body.meta.name,
year: parseInt(body.meta.year),
country: body.meta.country,
genres: body.meta.genres,
status: body.meta.status,
videos: body.meta.videos
imdbId: parseInt(body.meta?.imdb_id || '0'),
type: body.meta?.type,
title: body.meta?.name,
year: parseInt(body.meta?.year || '0'),
country: body.meta?.country,
genres: body.meta?.genres,
status: body.meta?.status,
videos: body.meta?.videos
? body.meta.videos.map(video => ({
name: video.name,
season: video.season,
@@ -133,10 +136,10 @@ export class MetadataService implements IMetadataService {
imdbEpisode: video.episode,
}))
: [],
episodeCount: body.meta.videos
episodeCount: body.meta?.videos
? this.getEpisodeCount(body.meta.videos)
: [],
totalCount: body.meta.videos
totalCount: body.meta?.videos
? body.meta.videos.filter(
entry => entry.season !== 0 && entry.episode !== 0
).length
@@ -144,15 +147,15 @@ export class MetadataService implements IMetadataService {
});
private handleKitsuResponse = (body: IKitsuJsonResponse): IMetadataResponse => ({
kitsuId: parseInt(body.meta.kitsu_id),
type: body.meta.type,
title: body.meta.name,
year: parseInt(body.meta.year),
country: body.meta.country,
genres: body.meta.genres,
status: body.meta.status,
videos: body.meta.videos
? body.meta.videos.map(video => ({
kitsuId: parseInt(body.meta?.kitsu_id || '0'),
type: body.meta?.type,
title: body.meta?.name,
year: parseInt(body.meta?.year || '0'),
country: body.meta?.country,
genres: body.meta?.genres,
status: body.meta?.status,
videos: body.meta?.videos
? body.meta?.videos.map(video => ({
name: video.title,
season: video.season,
episode: video.episode,
@@ -161,29 +164,32 @@ export class MetadataService implements IMetadataService {
released: video.released,
}))
: [],
episodeCount: body.meta.videos
episodeCount: body.meta?.videos
? this.getEpisodeCount(body.meta.videos)
: [],
totalCount: body.meta.videos
totalCount: body.meta?.videos
? body.meta.videos.filter(
entry => entry.season !== 0 && entry.episode !== 0
).length
: 0,
});
private getEpisodeCount = (videos: ICommonVideoMetadata[]) => Object.values(
videos
.filter(entry => entry.season !== 0 && entry.episode !== 0)
.sort((a, b) => a.season - b.season)
.reduce((map, next) => {
map[next.season] = map[next.season] + 1 || 1;
return map;
}, {})
);
private getEpisodeCount = (videos: ICommonVideoMetadata[]): number[] =>
Object.values(
videos
.filter(entry => entry.season !== null && entry.season !== 0 && entry.episode !== 0)
.sort((a, b) => (a.season || 0) - (b.season || 0))
.reduce((map: Record<number, number>, next) => {
if(next.season || next.season === 0) {
map[next.season] = (map[next.season] || 0) + 1;
}
return map;
}, {})
);
private getIMDbIdFromNameToImdb = (name: string, info: IMetaDataQuery): Promise<string | Error> => {
const year = info.year;
const type = info.type;
const {year} = info;
const {type} = info;
return new Promise((resolve, reject) => {
nameToImdb({name, year, type}, function (err: Error, res: string) {
if (res) {

98
src/node/consumer/src/lib/services/torrent_download_service.ts
View File

@@ -1,17 +1,17 @@
import {encode} from 'magnet-uri';
import {configurationService} from './configuration_service';
import {ExtensionHelpers} from '../helpers/extension_helpers';
import {ITorrentFileCollection} from "../interfaces/torrent_file_collection";
import {IParsedTorrent} from "../interfaces/parsed_torrent";
import {IFileAttributes} from "../../repository/interfaces/file_attributes";
import {ISubtitleAttributes} from "../../repository/interfaces/subtitle_attributes";
import {IContentAttributes} from "../../repository/interfaces/content_attributes";
import {parse} from "parse-torrent-title";
import {ITorrentDownloadService} from "../interfaces/torrent_download_service";
import {inject, injectable} from "inversify";
import {ILoggingService} from "../interfaces/logging_service";
import {IocTypes} from "../models/ioc_types";
import {encode} from 'magnet-uri';
import {parse} from "parse-torrent-title";
import WebTorrent from "webtorrent";
import {ExtensionHelpers} from '../helpers/extension_helpers';
import {ILoggingService} from "../interfaces/logging_service";
import {IParsedTorrent} from "../interfaces/parsed_torrent";
import {ITorrentDownloadService} from "../interfaces/torrent_download_service";
import {ITorrentFileCollection} from "../interfaces/torrent_file_collection";
import {IocTypes} from "../models/ioc_types";
import {IContentAttributes} from "../repository/interfaces/content_attributes";
import {IFileAttributes} from "../repository/interfaces/file_attributes";
import {ISubtitleAttributes} from "../repository/interfaces/subtitle_attributes";
import {configurationService} from './configuration_service';
interface ITorrentFile {
name: string;
@@ -20,13 +20,13 @@ interface ITorrentFile {
fileIndex: number;
}
const clientOptions = {
const clientOptions : WebTorrent.Options = {
maxConns: configurationService.torrentConfig.MAX_CONNECTIONS_OVERALL,
utp: false,
}
const torrentOptions = {
const torrentOptions: WebTorrent.TorrentOptions = {
skipVerify: true,
addUID: true,
destroyStoreOnDestroy: true,
private: true,
maxWebConns: configurationService.torrentConfig.MAX_CONNECTIONS_PER_TORRENT,
@@ -61,23 +61,20 @@ export class TorrentDownloadService implements ITorrentDownloadService {
if (!torrent.infoHash) {
return Promise.reject(new Error("No infoHash..."));
}
const magnet = encode({infoHash: torrent.infoHash, announce: torrent.trackers.split(',')});
this.logger.debug(`Constructing torrent stream for ${torrent.title} with magnet ${magnet}`);
const magnet = encode({infoHash: torrent.infoHash, announce: torrent.trackers!.split(',')});
return new Promise((resolve, reject) => {
this.logger.debug(`Adding torrent with infoHash ${torrent.infoHash} to webtorrent client...`);
const currentTorrent = this.torrentClient.add(magnet, torrentOptions);
const timeoutId = setTimeout(() => {
this.torrentClient.remove(magnet, {destroyStore: true});
this.removeTorrent(currentTorrent, torrent);
reject(new Error('No available connections for torrent!'));
}, timeout);
this.logger.debug(`Adding torrent with infoHash ${torrent.infoHash}`);
this.torrentClient.add(magnet, torrentOptions, (torrent) => {
this.logger.debug(`torrent with infoHash ${torrent.infoHash} added to client.`);
const files: ITorrentFile[] = torrent.files.map((file, fileId) => ({
currentTorrent.on('ready', () => {
const files: ITorrentFile[] = currentTorrent.files.map((file, fileId) => ({
fileIndex: fileId,
length: file.length,
name: file.name,
@@ -87,13 +84,22 @@ export class TorrentDownloadService implements ITorrentDownloadService {
this.logger.debug(`Found ${files.length} files in torrent ${torrent.infoHash}`);
resolve(files);
this.torrentClient.remove(magnet, {destroyStore: true});
clearTimeout(timeoutId);
this.removeTorrent(currentTorrent, torrent);
});
});
};
private removeTorrent = (currentTorrent: WebTorrent.Torrent, torrent: IParsedTorrent): void => {
try {
this.torrentClient.remove(currentTorrent, {destroyStore: true}, () => {
this.logger.debug(`Removed torrent ${torrent.infoHash} from webtorrent client...`);
});
} catch (error) {
this.logClientErrors(error);
}
};
private filterVideos = (torrent: IParsedTorrent, torrentFiles: ITorrentFile[]): IFileAttributes[] => {
if (torrentFiles.length === 1 && !Number.isInteger(torrentFiles[0].fileIndex)) {
return [this.mapTorrentFileToFileAttributes(torrent, torrentFiles[0])];
@@ -104,13 +110,25 @@ export class TorrentDownloadService implements ITorrentDownloadService {
const minAnimeExtraRatio = 5;
const minRedundantRatio = videos.length <= 3 ? 30 : Number.MAX_VALUE;
const isSample = (video: ITorrentFile) => video.path.toString()?.match(/sample|bonus|promo/i) && maxSize / video.length > minSampleRatio;
const isRedundant = (video: ITorrentFile) => maxSize / video.length > minRedundantRatio;
const isExtra = (video: ITorrentFile) => video.path.toString()?.match(/extras?\//i);
const isAnimeExtra = (video: ITorrentFile) => video.path.toString()?.match(/(?:\b|_)(?:NC)?(?:ED|OP|PV)(?:v?\d\d?)?(?:\b|_)/i)
&& maxSize / parseInt(video.length.toString()) > minAnimeExtraRatio;
const isWatermark = (video: ITorrentFile) => video.path.toString()?.match(/^[A-Z-]+(?:\.[A-Z]+)?\.\w{3,4}$/)
&& maxSize / parseInt(video.length.toString()) > minAnimeExtraRatio
const isSample = (video: ITorrentFile): boolean => video.path?.toString()?.match(/sample|bonus|promo/i) && maxSize / video.length > minSampleRatio || false;
const isRedundant = (video: ITorrentFile):boolean => maxSize / video.length > minRedundantRatio;
const isExtra = (video: ITorrentFile): boolean => /extras?\//i.test(video.path?.toString() || "");
const isAnimeExtra = (video: ITorrentFile): boolean => {
if (!video.path || !video.length) {
return false;
}
return video.path.toString()?.match(/(?:\b|_)(?:NC)?(?:ED|OP|PV)(?:v?\d\d?)?(?:\b|_)/i)
&& maxSize / parseInt(video.length.toString()) > minAnimeExtraRatio || false;
};
const isWatermark = (video: ITorrentFile): boolean => {
if (!video.path || !video.length) {
return false;
}
return video.path.toString()?.match(/^[A-Z-]+(?:\.[A-Z]+)?\.\w{3,4}$/)
&& maxSize / parseInt(video.length.toString()) > minAnimeExtraRatio || false;
}
return videos
.filter(video => !isSample(video))
@@ -133,11 +151,11 @@ export class TorrentDownloadService implements ITorrentDownloadService {
size: file.length,
fileIndex: file.fileIndex || 0,
path: file.path,
infoHash: torrent.infoHash,
imdbId: torrent.imdbId.toString(),
infoHash: torrent.infoHash?.toString(),
imdbId: torrent.imdbId?.toString() || '',
imdbSeason: torrent.season || 0,
imdbEpisode: torrent.episode || 0,
kitsuId: parseInt(torrent.kitsuId?.toString()) || 0,
kitsuId: parseInt(torrent.kitsuId?.toString() || '0') || 0,
kitsuEpisode: torrent.episode || 0,
};
@@ -162,8 +180,8 @@ export class TorrentDownloadService implements ITorrentDownloadService {
size: file.length,
});
private logClientErrors(errors: Error | string) {
this.logger.error(`Error in torrent client: ${errors}`);
private logClientErrors(errors: Error | string | unknown): void {
this.logger.error(`Error in webtorrent client: ${errors}`);
}
}

119
src/node/consumer/src/lib/services/torrent_entries_service.ts
View File

@@ -1,22 +1,24 @@
import {parse} from 'parse-torrent-title';
import {IParsedTorrent} from "../interfaces/parsed_torrent";
import {TorrentType} from '../enums/torrent_types';
import {ITorrentFileCollection} from "../interfaces/torrent_file_collection";
import {Torrent} from "../../repository/models/torrent";
import {PromiseHelpers} from '../helpers/promises_helpers';
import {ITorrentAttributes, ITorrentCreationAttributes} from "../../repository/interfaces/torrent_attributes";
import {File} from "../../repository/models/file";
import {Subtitle} from "../../repository/models/subtitle";
import {ITorrentEntriesService} from "../interfaces/torrent_entries_service";
import {inject, injectable} from "inversify";
import {IocTypes} from "../models/ioc_types";
import {IMetadataService} from "../interfaces/metadata_service";
import {parse} from 'parse-torrent-title';
import {TorrentType} from '../enums/torrent_types';
import {PromiseHelpers} from '../helpers/promises_helpers';
import {ILoggingService} from "../interfaces/logging_service";
import {IMetaDataQuery} from "../interfaces/metadata_query";
import {IMetadataService} from "../interfaces/metadata_service";
import {IParsedTorrent} from "../interfaces/parsed_torrent";
import {ITorrentEntriesService} from "../interfaces/torrent_entries_service";
import {ITorrentFileCollection} from "../interfaces/torrent_file_collection";
import {ITorrentFileService} from "../interfaces/torrent_file_service";
import {ITorrentSubtitleService} from "../interfaces/torrent_subtitle_service";
import {IDatabaseRepository} from "../../repository/interfaces/database_repository";
import {IIngestedTorrentCreationAttributes} from "../../repository/interfaces/ingested_torrent_attributes";
import {IFileCreationAttributes} from "../../repository/interfaces/file_attributes";
import {IocTypes} from "../models/ioc_types";
import {IDatabaseRepository} from "../repository/interfaces/database_repository";
import {IFileCreationAttributes} from "../repository/interfaces/file_attributes";
import {ISubtitleAttributes} from "../repository/interfaces/subtitle_attributes";
import {ITorrentAttributes, ITorrentCreationAttributes} from "../repository/interfaces/torrent_attributes";
import {File} from "../repository/models/file";
import {SkipTorrent} from "../repository/models/skipTorrent";
import {Subtitle} from "../repository/models/subtitle";
import {Torrent} from "../repository/models/torrent";
@injectable()
export class TorrentEntriesService implements ITorrentEntriesService {
@@ -39,6 +41,11 @@ export class TorrentEntriesService implements ITorrentEntriesService {
}
public createTorrentEntry = async (torrent: IParsedTorrent, overwrite = false): Promise<void> => {
if (!torrent.title) {
this.logger.warn(`No title found for ${torrent.provider} [${torrent.infoHash}]`);
return;
}
const titleInfo = parse(torrent.title);
if (!torrent.imdbId && torrent.type !== TorrentType.Anime) {
@@ -93,7 +100,7 @@ export class TorrentEntriesService implements ITorrentEntriesService {
});
return this.repository.createTorrent(newTorrent)
.then(() => PromiseHelpers.sequence(fileCollection.videos.map(video => () => {
.then(() => PromiseHelpers.sequence(fileCollection.videos!.map(video => () => {
const newVideo: IFileCreationAttributes = {...video, infoHash: video.infoHash, title: video.title};
if (!newVideo.kitsuId) {
newVideo.kitsuId = 0;
@@ -103,7 +110,7 @@ export class TorrentEntriesService implements ITorrentEntriesService {
.then(() => this.logger.info(`Created ${torrent.provider} entry for [${torrent.infoHash}] ${torrent.title}`));
};
private assignKitsuId = async (kitsuQuery: { year: number | string; season: number; title: string }, torrent: IParsedTorrent) => {
private assignKitsuId = async (kitsuQuery: IMetaDataQuery, torrent: IParsedTorrent): Promise<void> => {
await this.metadataService.getKitsuId(kitsuQuery)
.then((result: number | Error) => {
if (typeof result === 'number') {
@@ -118,10 +125,10 @@ export class TorrentEntriesService implements ITorrentEntriesService {
});
};
public createSkipTorrentEntry = async (torrent: Torrent) => this.repository.createSkipTorrent(torrent);
public createSkipTorrentEntry: (torrent: Torrent) => Promise<[SkipTorrent, boolean | null]> = async (torrent: Torrent)=> this.repository.createSkipTorrent(torrent.dataValues);
public getStoredTorrentEntry = async (torrent: Torrent) => this.repository.getSkipTorrent(torrent.infoHash)
.catch(() => this.repository.getTorrent(torrent))
public getStoredTorrentEntry = async (torrent: Torrent): Promise<Torrent | SkipTorrent | null | undefined> => this.repository.getSkipTorrent(torrent.infoHash)
.catch(() => this.repository.getTorrent(torrent.dataValues))
.catch(() => undefined);
public checkAndUpdateTorrent = async (torrent: IParsedTorrent): Promise<boolean> => {
@@ -141,7 +148,7 @@ export class TorrentEntriesService implements ITorrentEntriesService {
}
if (existingTorrent.provider === 'KickassTorrents' && torrent.provider) {
existingTorrent.provider = torrent.provider;
existingTorrent.torrentId = torrent.torrentId;
existingTorrent.torrentId = torrent.torrentId!;
}
if (!existingTorrent.languages && torrent.languages && existingTorrent.provider !== 'RARBG') {
@@ -149,11 +156,14 @@ export class TorrentEntriesService implements ITorrentEntriesService {
await existingTorrent.save();
this.logger.debug(`Updated [${existingTorrent.infoHash}] ${existingTorrent.title} language to ${torrent.languages}`);
}
return this.createTorrentContents(existingTorrent)
.then(() => this.updateTorrentSeeders(existingTorrent));
.then(() => this.updateTorrentSeeders(existingTorrent.dataValues))
.then(() => Promise.resolve(true))
.catch(() => Promise.reject(false));
};
public createTorrentContents = async (torrent: Torrent) => {
public createTorrentContents = async (torrent: Torrent): Promise<void> => {
if (torrent.opened) {
return;
}
@@ -167,7 +177,7 @@ export class TorrentEntriesService implements ITorrentEntriesService {
const kitsuId: number = PromiseHelpers.mostCommonValue(storedVideos.map(stored => stored.kitsuId || 0));
const fileCollection: ITorrentFileCollection = await this.fileService.parseTorrentFiles(torrent)
.then(torrentContents => notOpenedVideo ? torrentContents : {...torrentContents, videos: storedVideos})
.then(torrentContents => notOpenedVideo ? torrentContents : {...torrentContents, videos: storedVideos.map(video => video.dataValues)})
.then(torrentContents => this.subtitleService.assignSubtitles(torrentContents))
.then(torrentContents => this.assignMetaIds(torrentContents, imdbId, kitsuId))
.catch(error => {
@@ -181,22 +191,24 @@ export class TorrentEntriesService implements ITorrentEntriesService {
return;
}
if (notOpenedVideo && fileCollection.videos.length === 1) {
if (notOpenedVideo && fileCollection.videos?.length === 1) {
// if both have a single video and stored one was not opened, update stored one to true metadata and use that
storedVideos[0].fileIndex = fileCollection.videos[0].fileIndex;
storedVideos[0].fileIndex = fileCollection?.videos[0]?.fileIndex || 0;
storedVideos[0].title = fileCollection.videos[0].title;
storedVideos[0].size = fileCollection.videos[0].size;
storedVideos[0].subtitles = fileCollection.videos[0].subtitles.map(subtitle => Subtitle.build(subtitle));
fileCollection.videos[0] = storedVideos[0];
storedVideos[0].size = fileCollection.videos[0].size || 0;
const subtitles: ISubtitleAttributes[] = fileCollection.videos[0]?.subtitles || [];
storedVideos[0].subtitles = subtitles.map(subtitle => Subtitle.build(subtitle));
fileCollection.videos[0] = {...storedVideos[0], subtitles: subtitles};
}
// no videos available or more than one new videos were in the torrent
const shouldDeleteOld = notOpenedVideo && fileCollection.videos.every(video => !video.id);
const shouldDeleteOld = notOpenedVideo && fileCollection.videos?.every(video => !video.id) || false;
const newTorrent: Torrent = Torrent.build({
const newTorrent: ITorrentCreationAttributes = {
...torrent,
files: fileCollection.videos,
contents: fileCollection.contents,
subtitles: fileCollection.subtitles
});
};
return this.repository.createTorrent(newTorrent)
.then(() => {
@@ -206,7 +218,7 @@ export class TorrentEntriesService implements ITorrentEntriesService {
}
return Promise.resolve();
})
.then(() => PromiseHelpers.sequence(fileCollection.videos.map(video => () => {
.then(() => PromiseHelpers.sequence(fileCollection.videos!.map(video => (): Promise<void> => {
const newVideo: IFileCreationAttributes = {...video, infoHash: video.infoHash, title: video.title};
return this.repository.createFile(newVideo)
})))
@@ -214,19 +226,25 @@ export class TorrentEntriesService implements ITorrentEntriesService {
.catch(error => this.logger.error(`Failed saving contents for [${torrent.infoHash}] ${torrent.title}`, error));
};
public updateTorrentSeeders = async (torrent: ITorrentAttributes) => {
public updateTorrentSeeders = async (torrent: ITorrentAttributes): Promise<[number]> => {
if (!(torrent.infoHash || (torrent.provider && torrent.torrentId)) || !Number.isInteger(torrent.seeders)) {
return torrent;
return [0];
}
if (torrent.seeders === undefined) {
this.logger.warn(`Seeders not found for ${torrent.provider} [${torrent.infoHash}] ${torrent.title}`);
return [0];
}
return this.repository.setTorrentSeeders(torrent, torrent.seeders)
.catch(error => {
this.logger.warn('Failed updating seeders:', error);
return undefined;
return [0];
});
};
private assignMetaIds = (fileCollection: ITorrentFileCollection, imdbId: string, kitsuId: number): ITorrentFileCollection => {
private assignMetaIds = (fileCollection: ITorrentFileCollection, imdbId: string | undefined, kitsuId: number): ITorrentFileCollection => {
if (fileCollection.videos && fileCollection.videos.length) {
fileCollection.videos.forEach(video => {
video.imdbId = imdbId || '';
@@ -237,26 +255,31 @@ export class TorrentEntriesService implements ITorrentEntriesService {
return fileCollection;
};
private overwriteExistingFiles = async (torrent: IParsedTorrent, torrentContents: ITorrentFileCollection) => {
private overwriteExistingFiles = async (torrent: IParsedTorrent, torrentContents: ITorrentFileCollection): Promise<ITorrentFileCollection> => {
const videos = torrentContents && torrentContents.videos;
if (videos && videos.length) {
const existingFiles = await this.repository.getFiles(torrent.infoHash)
.then((existing) => existing
.reduce((map, next) => {
const fileIndex = next.fileIndex !== undefined ? next.fileIndex : null;
.then((existing) => existing.reduce<{ [key: number]: File[] }>((map, next) => {
const fileIndex = next.fileIndex !== undefined ? next.fileIndex : null;
if (fileIndex !== null) {
map[fileIndex] = (map[fileIndex] || []).concat(next);
return map;
}, {}))
}
return map;
}, {}))
.catch(() => undefined);
if (existingFiles && Object.keys(existingFiles).length) {
const overwrittenVideos = videos
.map(file => {
const mapping = videos.length === 1 && Object.keys(existingFiles).length === 1
? Object.values(existingFiles)[0]
: existingFiles[file.fileIndex !== undefined ? file.fileIndex : null];
const index = file.fileIndex !== undefined ? file.fileIndex : null;
let mapping;
if (index !== null) {
mapping = videos.length === 1 && Object.keys(existingFiles).length === 1
? Object.values(existingFiles)[0]
: existingFiles[index];
}
if (mapping) {
const originalFile = mapping.shift();
return {id: originalFile.id, ...file};
return {id: originalFile!.id, ...file};
}
return file;
});

329
src/node/consumer/src/lib/services/torrent_file_service.ts
View File

@@ -1,23 +1,24 @@
import Bottleneck from 'bottleneck';
import {inject, injectable} from "inversify";
import moment from 'moment';
import {parse} from 'parse-torrent-title';
import {PromiseHelpers} from '../helpers/promises_helpers';
import {TorrentType} from '../enums/torrent_types';
import {configurationService} from './configuration_service';
import {ExtensionHelpers} from '../helpers/extension_helpers';
import {IMetadataResponse} from "../interfaces/metadata_response";
import {IMetaDataQuery} from "../interfaces/metadata_query";
import {PromiseHelpers} from '../helpers/promises_helpers';
import {ICommonVideoMetadata} from "../interfaces/common_video_metadata";
import {ITorrentFileCollection} from "../interfaces/torrent_file_collection";
import {IParsedTorrent} from "../interfaces/parsed_torrent";
import {IFileAttributes} from "../../repository/interfaces/file_attributes";
import {IContentAttributes} from "../../repository/interfaces/content_attributes";
import {ITorrentFileService} from "../interfaces/torrent_file_service";
import {inject, injectable} from "inversify";
import {IocTypes} from "../models/ioc_types";
import {IMetadataService} from "../interfaces/metadata_service";
import {ITorrentDownloadService} from "../interfaces/torrent_download_service";
import {ILoggingService} from "../interfaces/logging_service";
import {IMetaDataQuery} from "../interfaces/metadata_query";
import {IMetadataResponse} from "../interfaces/metadata_response";
import {IMetadataService} from "../interfaces/metadata_service";
import {IParsedTorrent} from "../interfaces/parsed_torrent";
import {ISeasonEpisodeMap} from "../interfaces/season_episode_map";
import {ITorrentDownloadService} from "../interfaces/torrent_download_service";
import {ITorrentFileCollection} from "../interfaces/torrent_file_collection";
import {ITorrentFileService} from "../interfaces/torrent_file_service";
import {IocTypes} from "../models/ioc_types";
import {IContentAttributes} from "../repository/interfaces/content_attributes";
import {IFileAttributes} from "../repository/interfaces/file_attributes";
import {configurationService} from './configuration_service';
const MIN_SIZE: number = 5 * 1024 * 1024; // 5 MB
const MULTIPLE_FILES_SIZE = 4 * 1024 * 1024 * 1024; // 4 GB
@@ -41,6 +42,10 @@ export class TorrentFileService implements ITorrentFileService {
}
public parseTorrentFiles = async (torrent: IParsedTorrent): Promise<ITorrentFileCollection> => {
if (!torrent.title) {
return Promise.reject(new Error('Torrent title is missing'));
}
const parsedTorrentName = parse(torrent.title);
const query: IMetaDataQuery = {
id: torrent.kitsuId || torrent.imdbId,
@@ -49,6 +54,12 @@ export class TorrentFileService implements ITorrentFileService {
const metadata = await this.metadataService.getMetadata(query)
.then(meta => Object.assign({}, meta))
.catch(() => undefined);
if (metadata === undefined || metadata instanceof Error) {
this.logger.warn(`Failed to retrieve metadata for torrent ${torrent.title}`);
this.logger.debug(`Metadata Error: ${torrent.title}`, metadata);
return Promise.reject(new Error('Failed to retrieve metadata'));
}
if (torrent.type !== TorrentType.Anime && metadata && metadata.type && metadata.type !== torrent.type) {
// it's actually a movie/series
@@ -56,7 +67,7 @@ export class TorrentFileService implements ITorrentFileService {
}
if (torrent.type === TorrentType.Movie && (!parsedTorrentName.seasons ||
parsedTorrentName.season === 5 && [1, 5].includes(parsedTorrentName.episode))) {
parsedTorrentName.season === 5 && [1, 5].includes(parsedTorrentName.episode || 0))) {
return this.parseMovieFiles(torrent, metadata);
}
@@ -67,22 +78,27 @@ export class TorrentFileService implements ITorrentFileService {
if (torrent.isPack) {
return true;
}
if (!torrent.title) {
return false;
}
const parsedInfo = parse(torrent.title);
if (torrent.type === TorrentType.Movie) {
return parsedInfo.complete || typeof parsedInfo.year === 'string' || /movies/i.test(torrent.title);
}
const hasMultipleEpisodes = parsedInfo.complete ||
torrent.size > MULTIPLE_FILES_SIZE ||
const hasMultipleEpisodes = Boolean(parsedInfo.complete || torrent.size || 0 > MULTIPLE_FILES_SIZE ||
(parsedInfo.seasons && parsedInfo.seasons.length > 1) ||
(parsedInfo.episodes && parsedInfo.episodes.length > 1) ||
(parsedInfo.seasons && !parsedInfo.episodes);
const hasSingleEpisode = Number.isInteger(parsedInfo.episode) || (!parsedInfo.episodes && parsedInfo.date);
(parsedInfo.seasons && !parsedInfo.episodes));
const hasSingleEpisode: boolean = Boolean(Number.isInteger(parsedInfo.episode) || (!parsedInfo.episodes && parsedInfo.date));
return hasMultipleEpisodes && !hasSingleEpisode;
};
private parseSeriesVideos = (torrent: IParsedTorrent, videos: IFileAttributes[]): IFileAttributes[] => {
const parsedTorrentName = parse(torrent.title);
const hasMovies = parsedTorrentName.complete || !!torrent.title.match(/movies?(?:\W|$)/i);
const parsedTorrentName = parse(torrent.title!);
const hasMovies = parsedTorrentName.complete || !!torrent.title!.match(/movies?(?:\W|$)/i);
const parsedVideos = videos.map(video => this.parseSeriesVideo(video));
return parsedVideos.map(video => ({...video, isMovie: this.isMovieVideo(torrent, video, parsedVideos, hasMovies)}));
@@ -90,28 +106,32 @@ export class TorrentFileService implements ITorrentFileService {
private parseMovieFiles = async (torrent: IParsedTorrent, metadata: IMetadataResponse): Promise<ITorrentFileCollection> => {
const fileCollection: ITorrentFileCollection = await this.getMoviesTorrentContent(torrent);
if (fileCollection.videos === undefined || fileCollection.videos.length === 0) {
return {...fileCollection, videos: this.getDefaultFileEntries(torrent)};
}
const filteredVideos = fileCollection.videos
.filter(video => video.size > MIN_SIZE)
.filter(video => video.size! > MIN_SIZE)
.filter(video => !this.isFeaturette(video));
if (this.isSingleMovie(filteredVideos)) {
const parsedVideos = filteredVideos.map(video => ({
infoHash: torrent.infoHash,
fileIndex: video.fileIndex,
title: video.path || torrent.title,
title: video.path || video.title || video.fileName || '',
size: video.size || torrent.size,
imdbId: torrent.imdbId?.toString() || metadata && metadata.imdbId?.toString(),
kitsuId: parseInt(torrent.kitsuId?.toString() || metadata && metadata.kitsuId?.toString())
kitsuId: parseInt(torrent.kitsuId?.toString() || metadata && metadata.kitsuId?.toString() || '0')
}));
return {...fileCollection, videos: parsedVideos};
}
const parsedVideos = await PromiseHelpers.sequence(filteredVideos.map(video => () => this.isFeaturette(video)
? Promise.resolve(video)
: this.findMovieImdbId(video.title).then(imdbId => ({...video, imdbId}))))
.then(videos => videos.map(video => ({
: this.findMovieImdbId(video.title).then(imdbId => ({...video, imdbId: imdbId?.toString() || ''}))))
.then(videos => videos.map((video: IFileAttributes) => ({
infoHash: torrent.infoHash,
fileIndex: video.fileIndex,
title: video.path || video.name,
title: video.path || video.title,
size: video.size,
imdbId: video.imdbId,
})));
@@ -120,8 +140,12 @@ export class TorrentFileService implements ITorrentFileService {
private parseSeriesFiles = async (torrent: IParsedTorrent, metadata: IMetadataResponse): Promise<ITorrentFileCollection> => {
const fileCollection: ITorrentFileCollection = await this.getSeriesTorrentContent(torrent);
if (fileCollection.videos === undefined || fileCollection.videos.length === 0) {
return {...fileCollection, videos: this.getDefaultFileEntries(torrent)};
}
const parsedVideos: IFileAttributes[] = await Promise.resolve(fileCollection.videos)
.then(videos => videos.filter(video => videos.length === 1 || video.size > MIN_SIZE))
.then(videos => videos.filter(video => videos?.length === 1 || video.size! > MIN_SIZE))
.then(videos => this.parseSeriesVideos(torrent, videos))
.then(videos => this.decomposeEpisodes(torrent, videos, metadata))
.then(videos => this.assignKitsuOrImdbEpisodes(torrent, videos, metadata))
@@ -138,23 +162,24 @@ export class TorrentFileService implements ITorrentFileService {
const files = await this.torrentDownloadService.getTorrentFiles(torrent, configurationService.torrentConfig.TIMEOUT)
.catch(error => {
if (!this.isPackTorrent(torrent)) {
const entries = [{name: torrent.title, path: torrent.title, size: torrent.size, fileIndex: null}];
const entries = this.getDefaultFileEntries(torrent);
return {videos: entries, contents: [], subtitles: [], files: entries}
}
return Promise.reject(error);
});
if (files.contents && files.contents.length && !files.videos.length && this.isDiskTorrent(files.contents)) {
files.videos = [{name: torrent.title, path: torrent.title, size: torrent.size, fileIndex: null}];
if (files.contents && files.contents.length && !files.videos?.length && this.isDiskTorrent(files.contents)) {
files.videos = this.getDefaultFileEntries(torrent);
}
return files;
};
private getDefaultFileEntries = (torrent: IParsedTorrent): IFileAttributes[] => [{
title: torrent.title,
title: torrent.title!,
path: torrent.title,
size: torrent.size,
fileIndex: null
fileIndex: 0,
}];
private getSeriesTorrentContent = async (torrent: IParsedTorrent): Promise<ITorrentFileCollection> => this.torrentDownloadService.getTorrentFiles(torrent, configurationService.torrentConfig.TIMEOUT)
@@ -167,13 +192,13 @@ export class TorrentFileService implements ITorrentFileService {
private mapSeriesEpisode = async (torrent: IParsedTorrent, file: IFileAttributes, files: IFileAttributes[]): Promise<IFileAttributes[]> => {
if (!file.episodes && !file.episodes) {
if (files.length === 1 || files.some(f => f.episodes || f.episodes) || parse(torrent.title).seasons) {
if (files.length === 1 || files.some(f => f.episodes || f.episodes) || parse(torrent.title!).seasons) {
return Promise.resolve([{
infoHash: torrent.infoHash,
fileIndex: file.fileIndex,
title: file.path || file.title,
size: file.size,
imdbId: torrent.imdbId.toString() || file.imdbId.toString(),
imdbId: torrent?.imdbId?.toString() || file?.imdbId?.toString() || '',
}]);
}
return Promise.resolve([]);
@@ -184,14 +209,14 @@ export class TorrentFileService implements ITorrentFileService {
fileIndex: file.fileIndex,
title: file.path || file.title,
size: file.size,
imdbId: file.imdbId.toString() || torrent.imdbId.toString(),
imdbId: file?.imdbId?.toString() || torrent?.imdbId?.toString() || '',
imdbSeason: file.season,
season: file.season,
imdbEpisode: file.episodes && file.episodes[index],
episode: file.episodes && file.episodes[index],
kitsuEpisode: file.episodes && file.episodes[index],
episodes: file.episodes,
kitsuId: parseInt(file.kitsuId.toString() || torrent.kitsuId.toString()) || 0,
kitsuId: parseInt(file.kitsuId?.toString() || torrent.kitsuId?.toString() || '0') || 0,
})))
};
@@ -222,7 +247,7 @@ export class TorrentFileService implements ITorrentFileService {
title: file.path || file.title,
size: file.size,
imdbId: imdbId,
kitsuId: parseInt(kitsuId) || 0,
kitsuId: parseInt(kitsuId?.toString() || '0') || 0,
episodes: undefined,
imdbSeason: undefined,
imdbEpisode: undefined,
@@ -232,21 +257,21 @@ export class TorrentFileService implements ITorrentFileService {
// at this point, TypeScript infers that metadataOrError is actually MetadataResponse
const metadata = metadataOrError;
const hasEpisode = metadata.videos && metadata.videos.length && (file.episode || metadata.videos.length === 1);
const episodeVideo = hasEpisode && metadata.videos[(file.episode || 1) - 1];
const episodeVideo = hasEpisode && metadata.videos && metadata.videos[(file.episode || 1) - 1];
return [{
infoHash: torrent.infoHash,
fileIndex: file.fileIndex,
title: file.path || file.title,
size: file.size,
imdbId: metadata.imdbId.toString() || imdbId,
kitsuId: parseInt(metadata.kitsuId.toString() || kitsuId) || 0,
imdbId: metadata.imdbId?.toString() || imdbId || '',
kitsuId: parseInt(metadata.kitsuId?.toString() || kitsuId?.toString() || '0') || 0,
imdbSeason: episodeVideo && metadata.imdbId ? episodeVideo.season : undefined,
imdbEpisode: episodeVideo && metadata.imdbId | metadata.kitsuId ? episodeVideo.episode || episodeVideo.episode : undefined,
kitsuEpisode: episodeVideo && metadata.imdbId | metadata.kitsuId ? episodeVideo.episode || episodeVideo.episode : undefined,
imdbEpisode: episodeVideo && metadata.imdbId || episodeVideo && metadata.kitsuId ? episodeVideo.episode || episodeVideo.episode : undefined,
kitsuEpisode: episodeVideo && metadata.imdbId || episodeVideo && metadata.kitsuId ? episodeVideo.episode || episodeVideo.episode : undefined,
}];
};
private decomposeEpisodes = async (torrent: IParsedTorrent, files: IFileAttributes[], metadata: IMetadataResponse = {episodeCount: []}) => {
private decomposeEpisodes = async (torrent: IParsedTorrent, files: IFileAttributes[], metadata: IMetadataResponse = {episodeCount: []}):Promise<IFileAttributes[]> => {
if (files.every(file => !file.episodes && !file.date)) {
return files;
}
@@ -291,18 +316,18 @@ export class TorrentFileService implements ITorrentFileService {
return files;
};
private preprocessEpisodes = (files: IFileAttributes[]) => {
private preprocessEpisodes = (files: IFileAttributes[]): void => {
// reverse special episode naming when they named with 0 episode, ie. S02E00
files
.filter(file => Number.isInteger(file.season) && file.episode === 0)
.forEach(file => {
file.episode = file.season
file.episodes = [file.season]
file.episodes = [file.season || 0];
file.season = 0;
})
};
private isConcatSeasonAndEpisodeFiles = (files: IFileAttributes[], sortedEpisodes: number[], metadata: IMetadataResponse) => {
private isConcatSeasonAndEpisodeFiles = (files: IFileAttributes[], sortedEpisodes: number[], metadata: IMetadataResponse): boolean => {
if (metadata.kitsuId !== undefined) {
// anime does not use this naming scheme in 99% of cases;
return false;
@@ -317,58 +342,59 @@ export class TorrentFileService implements ITorrentFileService {
const threshold = Math.max(Math.ceil(files.length * 0.8), 5);
const sortedConcatEpisodes = sortedEpisodes
.filter(ep => ep > 100)
.filter(ep => metadata.episodeCount[this.div100(ep) - 1] < ep)
.filter(ep => metadata.episodeCount[this.div100(ep) - 1] >= this.mod100(ep));
.filter(ep => metadata.episodeCount && metadata.episodeCount[this.div100(ep) - 1] < ep)
.filter(ep => metadata.episodeCount && metadata.episodeCount[this.div100(ep) - 1] >= this.mod100(ep));
const concatFileEpisodes = files
.filter(file => !file.isMovie && file.episodes)
.filter(file => !file.season || file.episodes.every(ep => this.div100(ep) === file.season));
.filter(file => !file.season || file.episodes?.every(ep => this.div100(ep) === file.season));
const concatAboveTotalEpisodeCount = files
.filter(file => !file.isMovie && file.episodes && file.episodes.every(ep => ep > 100))
.filter(file => file.episodes.every(ep => ep > metadata.totalCount));
.filter(file => file.episodes?.every(ep => ep > metadata.totalCount!));
return sortedConcatEpisodes.length >= thresholdSorted && concatFileEpisodes.length >= threshold
|| concatAboveTotalEpisodeCount.length >= thresholdAbove;
};
private isDateEpisodeFiles = (files: IFileAttributes[], metadata: IMetadataResponse) => files.every(file => (!file.season || !metadata.episodeCount[file.season - 1]) && file.date);
private isDateEpisodeFiles = (files: IFileAttributes[], metadata: IMetadataResponse): boolean => files.every(file => (!file.season || metadata.episodeCount && !metadata.episodeCount[file.season - 1]) && file.date);
private isAbsoluteEpisodeFiles = (torrent: IParsedTorrent, files: IFileAttributes[], metadata: IMetadataResponse) => {
private isAbsoluteEpisodeFiles = (torrent: IParsedTorrent, files: IFileAttributes[], metadata: IMetadataResponse): boolean => {
const threshold = Math.ceil(files.length / 5);
const isAnime = torrent.type === TorrentType.Anime && torrent.kitsuId;
const nonMovieEpisodes = files
.filter(file => !file.isMovie && file.episodes);
const nonMovieEpisodes = files.filter(file => !file.isMovie && file.episodes);
const absoluteEpisodes = files
.filter(file => file.season && file.episodes)
.filter(file => file.episodes.every(ep => metadata.episodeCount[file.season - 1] < ep))
.filter(file => file.episodes?.every(ep =>
metadata.episodeCount && file.season && metadata.episodeCount[file.season - 1] < ep));
return nonMovieEpisodes.every(file => !file.season)
|| (isAnime && nonMovieEpisodes.every(file => file.season > metadata.episodeCount.length))
|| (isAnime && nonMovieEpisodes.every(file =>
metadata.episodeCount && file.season && file.season > metadata.episodeCount.length))
|| absoluteEpisodes.length >= threshold;
};
private isNewEpisodeNotInMetadata = (torrent: IParsedTorrent, video: IFileAttributes, metadata: IMetadataResponse) => {
// new episode might not yet been indexed by cinemeta.
// detect this if episode number is larger than the last episode or season is larger than the last one
// only for non anime metas
private isNewEpisodeNotInMetadata = (torrent: IParsedTorrent, video: IFileAttributes, metadata: IMetadataResponse): boolean => {
const isAnime = torrent.type === TorrentType.Anime && torrent.kitsuId;
return !isAnime && !video.isMovie && video.episodes && video.season !== 1
&& /continuing|current/i.test(metadata.status)
&& video.season >= metadata.episodeCount.length
&& video.episodes.every(ep => ep > (metadata.episodeCount[video.season - 1] || 0));
return !!( !isAnime && !video.isMovie && video.episodes && video.season !== 1
&& metadata.status && /continuing|current/i.test(metadata.status)
&& metadata.episodeCount && video.season && video.season >= metadata.episodeCount.length
&& video.episodes.every(ep => metadata.episodeCount && video.season && ep > (metadata.episodeCount[video.season - 1] || 0)));
};
private decomposeConcatSeasonAndEpisodeFiles = (files: IFileAttributes[], metadata: IMetadataResponse) => {
private decomposeConcatSeasonAndEpisodeFiles = (files: IFileAttributes[], metadata: IMetadataResponse): void => {
files
.filter(file => file.episodes && file.season !== 0 && file.episodes.every(ep => ep > 100))
.filter(file => metadata.episodeCount[(file.season || this.div100(file.episodes[0])) - 1] < 100)
.filter(file => file.season && file.episodes.every(ep => this.div100(ep) === file.season) || !file.season)
.filter(file => file.episodes && metadata?.episodeCount &&
((file.season || this.div100(file.episodes[0])) - 1) >= 0 &&
metadata.episodeCount[(file.season || this.div100(file.episodes[0])) - 1] < 100)
.filter(file => (file.season && file.episodes && file.episodes.every(ep => this.div100(ep) === file.season)) || !file.season)
.forEach(file => {
file.season = this.div100(file.episodes[0]);
file.episodes = file.episodes.map(ep => this.mod100(ep))
if (file.episodes) {
file.season = this.div100(file.episodes[0]);
file.episodes = file.episodes.map(ep => this.mod100(ep));
}
});
};
private decomposeAbsoluteEpisodeFiles = (torrent: IParsedTorrent, videos: IFileAttributes[], metadata: IMetadataResponse) => {
if (metadata.episodeCount.length === 0) {
private decomposeAbsoluteEpisodeFiles = (torrent: IParsedTorrent, videos: IFileAttributes[], metadata: IMetadataResponse): void => {
if (metadata.episodeCount?.length === 0) {
videos
.filter(file => !Number.isInteger(file.season) && file.episodes && !file.isMovie)
.forEach(file => {
@@ -376,29 +402,38 @@ export class TorrentFileService implements ITorrentFileService {
});
return;
}
if (!metadata.episodeCount) return;
videos
.filter(file => file.episodes && !file.isMovie && file.season !== 0)
.filter(file => !this.isNewEpisodeNotInMetadata(torrent, file, metadata))
.filter(file => !file.season || (metadata.episodeCount[file.season - 1] || 0) < file.episodes[0])
.filter(file => {
if (!file.episodes || !metadata.episodeCount) return false;
return !file.season || (metadata.episodeCount[file.season - 1] || 0) < file.episodes[0];
})
.forEach(file => {
const seasonIdx = ([...metadata.episodeCount.keys()]
.find((i) => metadata.episodeCount.slice(0, i + 1).reduce((a, b) => a + b) >= file.episodes[0])
+ 1 || metadata.episodeCount.length) - 1;
if(!file.episodes || !metadata.episodeCount) return;
let seasonIdx = metadata.episodeCount
.map((_, i) => i)
.find(i => metadata.episodeCount && file.episodes && metadata.episodeCount.slice(0, i + 1).reduce((a, b) => a + b) >= file.episodes[0]);
seasonIdx = (seasonIdx || 1 || metadata.episodeCount.length) - 1;
file.season = seasonIdx + 1;
file.episodes = file.episodes
.map(ep => ep - metadata.episodeCount.slice(0, seasonIdx).reduce((a, b) => a + b, 0))
.map(ep => ep - (metadata.episodeCount?.slice(0, seasonIdx).reduce((a, b) => a + b, 0) || 0));
});
};
private decomposeDateEpisodeFiles = (files: IFileAttributes[], metadata: IMetadataResponse) => {
private decomposeDateEpisodeFiles = (files: IFileAttributes[], metadata: IMetadataResponse): void => {
if (!metadata || !metadata.videos || !metadata.videos.length) {
return;
}
const timeZoneOffset = this.getTimeZoneOffset(metadata.country);
const offsetVideos = metadata.videos
.reduce((map, video) => {
const offsetVideos: { [key: string]: ICommonVideoMetadata } = metadata.videos
.reduce((map: { [key: string]: ICommonVideoMetadata }, video: ICommonVideoMetadata) => {
const releaseDate = moment(video.released).utcOffset(timeZoneOffset).format('YYYY-MM-DD');
map[releaseDate] = video;
return map;
@@ -407,15 +442,15 @@ export class TorrentFileService implements ITorrentFileService {
files
.filter(file => file.date)
.forEach(file => {
const video = offsetVideos[file.date];
const video = offsetVideos[file.date!];
if (video) {
file.season = video.season;
file.episodes = [video.episode];
file.episodes = [video.episode || 0];
}
});
};
private getTimeZoneOffset = (country: string | undefined) => {
private getTimeZoneOffset = (country: string | undefined): string => {
switch (country) {
case 'United States':
case 'USA':
@@ -425,7 +460,7 @@ export class TorrentFileService implements ITorrentFileService {
}
};
private assignKitsuOrImdbEpisodes = (torrent: IParsedTorrent, files: IFileAttributes[], metadata: IMetadataResponse) => {
private assignKitsuOrImdbEpisodes = (torrent: IParsedTorrent, files: IFileAttributes[], metadata: IMetadataResponse): IFileAttributes[] => {
if (!metadata || !metadata.videos || !metadata.videos.length) {
if (torrent.type === TorrentType.Anime) {
// assign episodes as kitsu episodes for anime when no metadata available for imdb mapping
@@ -437,31 +472,35 @@ export class TorrentFileService implements ITorrentFileService {
})
if (metadata.type === TorrentType.Movie && files.every(file => !file.imdbId)) {
// sometimes a movie has episode naming, thus not recognized as a movie and imdbId not assigned
files.forEach(file => file.imdbId = metadata.imdbId.toString());
files.forEach(file => file.imdbId = metadata.imdbId?.toString());
}
}
return files;
}
const seriesMapping: ICommonVideoMetadata = metadata.videos
.reduce((map, video) => {
const episodeMap = map[video.season] || {};
episodeMap[video.episode] = video;
map[video.season] = episodeMap;
const seriesMapping = metadata.videos
.filter(video => video.season !== undefined && Number.isInteger(video.season) && video.episode !== undefined && Number.isInteger(video.episode))
.reduce<ISeasonEpisodeMap>((map, video) => {
if (video.season !== undefined && video.episode !== undefined) {
const episodeMap = map[video.season] || {};
episodeMap[video.episode] = video;
map[video.season] = episodeMap;
}
return map;
}, {});
if (metadata.videos.some(video => Number.isInteger(video.season)) || !metadata.imdbId) {
files.filter((file => Number.isInteger(file.season) && file.episodes))
files.filter(file => file && Number.isInteger(file.season) && file.episodes)
.map(file => {
const seasonMapping = seriesMapping[file.season];
const episodeMapping = seasonMapping && seasonMapping[file.episodes[0]];
const seasonMapping = file && file.season && seriesMapping[file.season] || null;
const episodeMapping = seasonMapping && file && file.episodes && file.episodes[0] && seasonMapping[file.episodes[0]] || null;
if (episodeMapping && Number.isInteger(episodeMapping.season)) {
file.imdbId = metadata.imdbId.toString();
file.imdbId = metadata.imdbId?.toString();
file.season = episodeMapping.season;
file.episodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].episode);
file.episodes = file.episodes && file.episodes.map(ep => (seasonMapping && seasonMapping[ep]) ? Number(seasonMapping[ep].episode) : 0);
} else {
// no imdb mapping available for episode
file.season = undefined;
file.episodes = undefined;
}
@@ -471,11 +510,16 @@ export class TorrentFileService implements ITorrentFileService {
files
.filter(file => Number.isInteger(file.season) && file.episodes)
.forEach(file => {
if (!file.season || !file.episodes) {
return;
}
if (seriesMapping[file.season]) {
const seasonMapping = seriesMapping[file.season];
file.imdbId = metadata.imdbId.toString();
file.kitsuId = seasonMapping[file.episodes[0]] && seasonMapping[file.episodes[0]].kitsuId || 0;
file.episodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].kitsuEpisode);
file.imdbId = metadata.imdbId?.toString();
file.kitsuId = seasonMapping[file.episodes[0]] && parseInt(seasonMapping[file.episodes[0]].id || '0') || 0;
file.episodes = file.episodes.map(ep => seasonMapping[ep]?.episode)
.filter((ep): ep is number => ep !== undefined);
} else if (seriesMapping[file.season - 1]) {
// sometimes a second season might be a continuation of the previous season
const seasonMapping = seriesMapping[file.season - 1] as ICommonVideoMetadata;
@@ -483,48 +527,65 @@ export class TorrentFileService implements ITorrentFileService {
const firstKitsuId = episodes.length && episodes[0];
const differentTitlesCount = new Set(episodes.map(ep => ep.id)).size
const skippedCount = episodes.filter(ep => ep.id === firstKitsuId).length;
const emptyArray: number[] = [];
const seasonEpisodes = files
.filter((otherFile: IFileAttributes) => otherFile.season === file.season)
.reduce((a, b) => a.concat(b.episodes), []);
.filter((otherFile: IFileAttributes) => otherFile.season === file.season && otherFile.episodes)
.reduce((a, b) => a.concat(b.episodes || []), emptyArray);
const isAbsoluteOrder = seasonEpisodes.every(ep => ep > skippedCount && ep <= episodes.length)
const isNormalOrder = seasonEpisodes.every(ep => ep + skippedCount <= episodes.length)
if (differentTitlesCount >= 1 && (isAbsoluteOrder || isNormalOrder)) {
file.imdbId = metadata.imdbId.toString();
const {season} = file;
const [episode] = file.episodes;
file.imdbId = metadata.imdbId?.toString();
file.season = file.season - 1;
file.episodes = file.episodes.map(ep => isAbsoluteOrder ? ep : ep + skippedCount);
file.kitsuId = seasonMapping[file.episodes[0]].kitsuId || 0;
file.episodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].kitsuEpisode);
const currentEpisode = seriesMapping[season][episode];
file.kitsuId = currentEpisode ? parseInt(currentEpisode.id || '0') : 0;
if (typeof season === 'number' && Array.isArray(file.episodes)) {
file.episodes = file.episodes.map(ep =>
seriesMapping[season]
&& seriesMapping[season][ep]
&& seriesMapping[season][ep].episode
|| ep);
}
}
} else if (Object.values(seriesMapping).length === 1 && seriesMapping[1]) {
// sometimes series might be named with sequel season but it's not a season on imdb and a new title
// eslint-disable-next-line prefer-destructuring
const seasonMapping = seriesMapping[1];
file.imdbId = metadata.imdbId.toString();
file.imdbId = metadata.imdbId?.toString();
file.season = 1;
file.kitsuId = seasonMapping[file.episodes[0]].kitsuId || 0;
file.episodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].kitsuEpisode);
file.kitsuId = parseInt(seasonMapping[file.episodes[0]].id || '0') || 0;
file.episodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].episode)
.filter((ep): ep is number => ep !== undefined);
}
});
}
return files;
};
private needsCinemetaMetadataForAnime = (files: IFileAttributes[], metadata: IMetadataResponse) => {
private needsCinemetaMetadataForAnime = (files: IFileAttributes[], metadata: IMetadataResponse): boolean => {
if (!metadata || !metadata.imdbId || !metadata.videos || !metadata.videos.length) {
return false;
}
const minSeason = Math.min(...metadata.videos.map(video => video.season)) || Number.MAX_VALUE;
const maxSeason = Math.max(...metadata.videos.map(video => video.season)) || Number.MAX_VALUE;
const differentSeasons = new Set(metadata.videos
const seasons = metadata.videos
.map(video => video.season)
.filter(season => Number.isInteger(season))).size;
.filter((season): season is number => season !== null && season !== undefined);
// Using || 0 instead of || Number.MAX_VALUE to match previous logic
const minSeason = Math.min(...seasons) || 0;
const maxSeason = Math.max(...seasons) || 0;
const differentSeasons = new Set(seasons.filter(season => Number.isInteger(season))).size;
const total = metadata.totalCount || Number.MAX_VALUE;
return differentSeasons > 1 || files
.filter(file => !file.isMovie && file.episodes)
.some(file => file.season < minSeason || file.season > maxSeason || file.episodes.every(ep => ep > total));
.some(file => file.season || 0 < minSeason || file.season || 0 > maxSeason || file.episodes?.every(ep => ep > total));
};
private updateToCinemetaMetadata = async (metadata: IMetadataResponse) => {
private updateToCinemetaMetadata = async (metadata: IMetadataResponse): Promise<IMetadataResponse> => {
const query: IMetaDataQuery = {
id: metadata.imdbId,
type: metadata.type
@@ -538,7 +599,7 @@ export class TorrentFileService implements ITorrentFileService {
return metadata; // or throw newMetadataOrError to propagate error up the call stack
}
// At this point TypeScript infers newMetadataOrError to be of type MetadataResponse
let newMetadata = newMetadataOrError;
const newMetadata = newMetadataOrError;
if (!newMetadata.videos || !newMetadata.videos.length) {
return metadata;
} else {
@@ -550,7 +611,7 @@ export class TorrentFileService implements ITorrentFileService {
})
};
private findMovieImdbId = (title: IFileAttributes | string) => {
private findMovieImdbId = (title: IFileAttributes | string):Promise<string | undefined> => {
const parsedTitle = typeof title === 'string' ? parse(title) : title;
this.logger.debug(`Finding movie imdbId for ${title}`);
return this.imdb_limiter.schedule(async () => {
@@ -567,7 +628,7 @@ export class TorrentFileService implements ITorrentFileService {
});
};
private findMovieKitsuId = async (title: IFileAttributes | string) => {
private findMovieKitsuId = async (title: IFileAttributes | string):Promise<number | Error | undefined> => {
const parsedTitle = typeof title === 'string' ? parse(title) : title;
const kitsuQuery = {
title: parsedTitle.title,
@@ -582,20 +643,20 @@ export class TorrentFileService implements ITorrentFileService {
}
};
private isDiskTorrent = (contents: IContentAttributes[]) => contents.some(content => ExtensionHelpers.isDisk(content.path));
private isDiskTorrent = (contents: IContentAttributes[]): boolean => contents.some(content => ExtensionHelpers.isDisk(content.path));
private isSingleMovie = (videos: IFileAttributes[]) => videos.length === 1 ||
private isSingleMovie = (videos: IFileAttributes[]): boolean => videos.length === 1 ||
(videos.length === 2 &&
videos.find(v => /\b(?:part|disc|cd)[ ._-]?0?1\b|^0?1\.\w{2,4}$/i.test(v.path)) &&
videos.find(v => /\b(?:part|disc|cd)[ ._-]?0?2\b|^0?2\.\w{2,4}$/i.test(v.path)));
videos.find(v => /\b(?:part|disc|cd)[ ._-]?0?1\b|^0?1\.\w{2,4}$/i.test(v.path!)) &&
videos.find(v => /\b(?:part|disc|cd)[ ._-]?0?2\b|^0?2\.\w{2,4}$/i.test(v.path!))) !== undefined;
private isFeaturette = (video: IFileAttributes) => /featurettes?\/|extras-grym/i.test(video.path);
private isFeaturette = (video: IFileAttributes):boolean => /featurettes?\/|extras-grym/i.test(video.path!);
private parseSeriesVideo = (video: IFileAttributes): IFileAttributes => {
const videoInfo = parse(video.title);
// the episode may be in a folder containing season number
if (!Number.isInteger(videoInfo.season) && video.path.includes('/')) {
const folders = video.path.split('/');
if (!Number.isInteger(videoInfo.season) && video.path?.includes('/')) {
const folders = video.path?.split('/');
const pathInfo = parse(folders[folders.length - 2]);
videoInfo.season = pathInfo.season;
}
@@ -604,12 +665,12 @@ export class TorrentFileService implements ITorrentFileService {
}
if (!Number.isInteger(videoInfo.season) && videoInfo.seasons && videoInfo.seasons.length > 1) {
// in case single file was interpreted as having multiple seasons
videoInfo.season = videoInfo.seasons[0];
[videoInfo.season] = videoInfo.seasons;
}
if (!Number.isInteger(videoInfo.season) && video.path.includes('/') && video.seasons
if (!Number.isInteger(videoInfo.season) && video.path?.includes('/') && video.seasons
&& video.seasons.length > 1) {
// russian season are usually named with 'series name-2` i.e. Улицы разбитых фонарей-6/22. Одиночный выстрел.mkv
const folderPathSeasonMatch = video.path.match(/[\u0400-\u04ff]-(\d{1,2})(?=.*\/)/);
const folderPathSeasonMatch = video.path?.match(/[\u0400-\u04ff]-(\d{1,2})(?=.*\/)/);
videoInfo.season = folderPathSeasonMatch && parseInt(folderPathSeasonMatch[1], 10) || undefined;
}
// sometimes video file does not have correct date format as in torrent title
@@ -619,18 +680,18 @@ export class TorrentFileService implements ITorrentFileService {
// limit number of episodes in case of incorrect parsing
if (videoInfo.episodes && videoInfo.episodes.length > 20) {
videoInfo.episodes = [videoInfo.episodes[0]];
videoInfo.episode = videoInfo.episodes[0];
[videoInfo.episode] = videoInfo.episodes;
}
// force episode to any found number if it was not parsed
if (!videoInfo.episodes && !videoInfo.date) {
const epMatcher = videoInfo.title.match(
/(?<!season\W*|disk\W*|movie\W*|film\W*)(?:^|\W|_)(\d{1,4})(?:a|b|c|v\d)?(?:_|\W|$)(?!disk|movie|film)/i);
videoInfo.episodes = epMatcher && [parseInt(epMatcher[1], 10)];
videoInfo.episodes = epMatcher && [parseInt(epMatcher[1], 10)] || undefined;
videoInfo.episode = videoInfo.episodes && videoInfo.episodes[0];
}
if (!videoInfo.episodes && !videoInfo.date) {
const epMatcher = video.title.match(new RegExp(`(?:\\(${videoInfo.year}\\)|part)[._ ]?(\\d{1,3})(?:\\b|_)`, "i"));
videoInfo.episodes = epMatcher && [parseInt(epMatcher[1], 10)];
videoInfo.episodes = epMatcher && [parseInt(epMatcher[1], 10)] || undefined;
videoInfo.episode = videoInfo.episodes && videoInfo.episodes[0];
}
@@ -642,7 +703,7 @@ export class TorrentFileService implements ITorrentFileService {
// not movie if video has season
return false;
}
if (torrent.title.match(/\b(?:\d+[ .]movie|movie[ .]\d+)\b/i)) {
if (torrent.title?.match(/\b(?:\d+[ .]movie|movie[ .]\d+)\b/i)) {
// movie if video explicitly has numbered movie keyword in the name, ie. 1 Movie or Movie 1
return true;
}
@@ -661,7 +722,7 @@ export class TorrentFileService implements ITorrentFileService {
&& otherVideos.filter(other => other.title === video.title && other.year === video.year).length < 3;
};
private clearInfoFields = (video: IFileAttributes) => {
private clearInfoFields = (video: IFileAttributes): IFileAttributes => {
video.imdbId = undefined;
video.imdbSeason = undefined;
video.imdbEpisode = undefined;
@@ -670,9 +731,9 @@ export class TorrentFileService implements ITorrentFileService {
return video;
};
private div100 = (episode: number) => (episode / 100 >> 0);
private div100 = (episode: number): number => (episode / 100 >> 0);
private mod100 = (episode: number) => episode % 100;
private mod100 = (episode: number): number => episode % 100;
}

12
src/node/consumer/src/lib/services/torrent_processing_service.ts
View File

@@ -1,12 +1,12 @@
import {TorrentType} from "../enums/torrent_types";
import {IIngestedTorrentAttributes} from "../../repository/interfaces/ingested_torrent_attributes";
import {IParsedTorrent} from "../interfaces/parsed_torrent";
import {ITorrentProcessingService} from "../interfaces/torrent_processing_service";
import {inject, injectable} from "inversify";
import {IocTypes} from "../models/ioc_types";
import {ITorrentEntriesService} from "../interfaces/torrent_entries_service";
import {TorrentType} from "../enums/torrent_types";
import {ILoggingService} from "../interfaces/logging_service";
import {IParsedTorrent} from "../interfaces/parsed_torrent";
import {ITorrentEntriesService} from "../interfaces/torrent_entries_service";
import {ITorrentProcessingService} from "../interfaces/torrent_processing_service";
import {ITrackerService} from "../interfaces/tracker_service";
import {IocTypes} from "../models/ioc_types";
import {IIngestedTorrentAttributes} from "../repository/interfaces/ingested_torrent_attributes";
@injectable()
export class TorrentProcessingService implements ITorrentProcessingService {

51
src/node/consumer/src/lib/services/torrent_subtitle_service.ts
View File

@@ -1,8 +1,9 @@
import {injectable} from "inversify";
import {parse} from 'parse-torrent-title';
import {ITorrentFileCollection} from "../interfaces/torrent_file_collection";
import {IFileAttributes} from "../../repository/interfaces/file_attributes";
import {ITorrentSubtitleService} from "../interfaces/torrent_subtitle_service";
import {injectable} from "inversify";
import {IFileAttributes} from "../repository/interfaces/file_attributes";
import {ISubtitleAttributes} from "../repository/interfaces/subtitle_attributes";
@injectable()
export class TorrentSubtitleService implements ITorrentSubtitleService {
@@ -28,57 +29,57 @@ export class TorrentSubtitleService implements ITorrentSubtitleService {
return fileCollection;
};
private parseVideo = (video: IFileAttributes) => {
const fileName = video.title.split('/').pop().replace(/\.(\w{2,4})$/, '');
const folderName = video.title.replace(/\/?[^/]+$/, '');
private parseVideo = (video: IFileAttributes): IFileAttributes => {
const fileName = video.title?.split('/')?.pop()?.replace(/\.(\w{2,4})$/, '') || '';
const folderName = video.title?.replace(/\/?[^/]+$/, '') || '';
return {
videoFile: video,
fileName: fileName,
folderName: folderName,
...this.parseFilename(video.title)
...this.parseFilename(video.title.toString() || '')
};
}
private mostProbableSubtitleVideos = (subtitle: any, parsedVideos: any[]) => {
const subTitle = (subtitle.title || subtitle.path).split('/').pop().replace(/\.(\w{2,4})$/, '');
private mostProbableSubtitleVideos = (subtitle: ISubtitleAttributes, parsedVideos: IFileAttributes[]) : IFileAttributes[] => {
const subTitle = (subtitle.title || subtitle.path)?.split('/')?.pop()?.replace(/\.(\w{2,4})$/, '') || '';
const parsedSub = this.parsePath(subtitle.title || subtitle.path);
const byFileName = parsedVideos.filter(video => subTitle.includes(video.fileName));
const byFileName = parsedVideos.filter(video => subTitle.includes(video.title!));
if (byFileName.length === 1) {
return byFileName.map(v => v.videoFile);
return byFileName.map(v => v);
}
const byTitleSeasonEpisode = parsedVideos.filter(video => video.title === parsedSub.title
&& this.arrayEquals(video.seasons, parsedSub.seasons)
&& this.arrayEquals(video.episodes, parsedSub.episodes));
&& this.arrayEquals(video.seasons || [], parsedSub.seasons || [])
&& this.arrayEquals(video.episodes || [], parsedSub.episodes || []));
if (this.singleVideoFile(byTitleSeasonEpisode)) {
return byTitleSeasonEpisode.map(v => v.videoFile);
return byTitleSeasonEpisode.map(v => v);
}
const bySeasonEpisode = parsedVideos.filter(video => this.arrayEquals(video.seasons, parsedSub.seasons)
&& this.arrayEquals(video.episodes, parsedSub.episodes));
const bySeasonEpisode = parsedVideos.filter(video => this.arrayEquals(video.seasons || [], parsedSub.seasons || [])
&& this.arrayEquals(video.episodes || [], parsedSub.episodes || []));
if (this.singleVideoFile(bySeasonEpisode)) {
return bySeasonEpisode.map(v => v.videoFile);
return bySeasonEpisode.map(v => v);
}
const byTitle = parsedVideos.filter(video => video.title && video.title === parsedSub.title);
if (this.singleVideoFile(byTitle)) {
return byTitle.map(v => v.videoFile);
return byTitle.map(v => v);
}
const byEpisode = parsedVideos.filter(video => this.arrayEquals(video.episodes, parsedSub.episodes));
const byEpisode = parsedVideos.filter(video => this.arrayEquals(video.episodes || [], parsedSub.episodes || []));
if (this.singleVideoFile(byEpisode)) {
return byEpisode.map(v => v.videoFile);
return byEpisode.map(v => v);
}
return undefined;
return [];
}
private singleVideoFile = (videos: any[]) => {
return new Set(videos.map(v => v.videoFile.fileIndex)).size === 1;
private singleVideoFile = (videos: IFileAttributes[]): boolean => {
return new Set(videos.map(v => v.fileIndex)).size === 1;
}
private parsePath = (path: string) => {
private parsePath = (path: string): IFileAttributes => {
const pathParts = path.split('/').map(part => this.parseFilename(part));
const parsedWithEpisode = pathParts.find(parsed => parsed.season && parsed.episodes);
return parsedWithEpisode || pathParts[pathParts.length - 1];
}
private parseFilename = (filename: string) => {
private parseFilename = (filename: string) : IFileAttributes => {
const parsedInfo = parse(filename)
const titleEpisode = parsedInfo.title.match(/(\d+)$/);
if (!parsedInfo.episodes && titleEpisode) {
@@ -87,7 +88,7 @@ export class TorrentSubtitleService implements ITorrentSubtitleService {
return parsedInfo;
}
private arrayEquals = (array1: any[], array2: any[]) => {
private arrayEquals = <T>(array1: T[], array2: T[]): boolean => {
if (!array1 || !array2) return array1 === array2;
return array1.length === array2.length && array1.every((value, index) => value === array2[index])
}

6
src/node/consumer/src/lib/services/tracker_service.ts
View File

@@ -1,10 +1,10 @@
import axios, {AxiosResponse} from 'axios';
import {configurationService} from './configuration_service';
import {ITrackerService} from "../interfaces/tracker_service";
import {inject, injectable} from "inversify";
import {IocTypes} from "../models/ioc_types";
import {ICacheService} from "../interfaces/cache_service";
import {ILoggingService} from "../interfaces/logging_service";
import {ITrackerService} from "../interfaces/tracker_service";
import {IocTypes} from "../models/ioc_types";
import {configurationService} from './configuration_service';
@injectable()
export class TrackerService implements ITrackerService {

5
src/node/consumer/src/main.ts
View File

@@ -1,8 +1,9 @@
import "reflect-metadata"; // required
import {ICompositionalRoot} from "./lib/interfaces/composition_root";
import {serviceContainer} from "./lib/models/inversify_config";
import {IocTypes} from "./lib/models/ioc_types";
import {ICompositionalRoot} from "./lib/interfaces/composition_root";
(async () => {
(async (): Promise<void> => {
const compositionalRoot = serviceContainer.get<ICompositionalRoot>(IocTypes.ICompositionalRoot);
await compositionalRoot.start();
})();

32
src/node/consumer/tsconfig.json
View File

@@ -1,23 +1,29 @@
{
"compilerOptions": {
"module": "CommonJS",
"moduleResolution": "node",
"outDir": "dist",
"pretty": true,
"target": "es2020",
"module": "es2020",
"allowJs": true,
"removeComments": true,
"rootDir": "./src",
"resolveJsonModule": true,
"typeRoots": [
"./node_modules/@types"
],
"sourceMap": true,
"target": "ES6",
"outDir": "dist",
"strict": true,
"lib": [
"es6"
],
"types": [
"node",
"reflect-metadata"
"es2020"
],
"baseUrl": ".",
"forceConsistentCasingInFileNames": true,
"esModuleInterop": true,
"experimentalDecorators": true,
"emitDecoratorMetadata": true,
"allowSyntheticDefaultImports": true
}
"moduleResolution": "Node",
"skipLibCheck": true,
},
"include": [
"src/**/*"
],
"exclude": ["node_modules"],
}