tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

replace torrent-stream (high vunerabilities) with webtorrent, gives us clean package audit

Browse Source
This commit is contained in:
iPromKnight
2024-02-07 14:03:57 +00:00
committed by iPromKnight
parent 6e2b776211
commit 7fe9b64f66
32 changed files with 1846 additions and 758 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/node/consumer/.eslintrc
View File

@@ -64,7 +64,12 @@
},
"overrides": [
{
"files": ["*.ts", "*.mts", "*.cts", "*.tsx"],
"files": [
"*.ts",
"*.mts",
"*.cts",
"*.tsx"
],
"rules": {
"@typescript-eslint/explicit-function-return-type": "error",
"@typescript-eslint/consistent-type-assertions": [
@@ -73,7 +78,7 @@
"assertionStyle": "as",
"objectLiteralTypeAssertions": "never"
}
]
]
}
}
]

14
src/node/consumer/esbuild.js
View File

@@ -1,14 +1,14 @@
import { build } from "esbuild";
import { readFileSync, rmSync } from "fs";
import {build} from "esbuild";
import {readFileSync, rmSync} from "fs";
const { devDependencies } = JSON.parse(readFileSync("./package.json", "utf8"));
const {devDependencies} = JSON.parse(readFileSync("./package.json", "utf8"));
const start = Date.now();
try {
const outdir = "dist";
rmSync(outdir, { recursive: true, force: true });
rmSync(outdir, {recursive: true, force: true});
build({
bundle: true,
@@ -27,8 +27,8 @@ try {
plugins: [
{
name: "populate-import-meta",
setup: ({ onLoad }) => {
onLoad({ filter: new RegExp(`${import.meta.dirname}/src/.*.(js|ts)$`) }, args => {
setup: ({onLoad}) => {
onLoad({filter: new RegExp(`${import.meta.dirname}/src/.*.(js|ts)$`)}, args => {
const contents = readFileSync(args.path, "utf8");
const transformedContents = contents
@@ -36,7 +36,7 @@ try {
.replace(/import\.meta\.filename/g, "__filename")
.replace(/import\.meta\.dirname/g, "__dirname");
return { contents: transformedContents, loader: "default" };
return {contents: transformedContents, loader: "default"};
});
},
}

2160
src/node/consumer/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

5
src/node/consumer/package.json
View File

@@ -27,15 +27,12 @@
"reflect-metadata": "^0.2.1",
"sequelize": "^6.36.0",
"sequelize-typescript": "^2.1.6",
"torrent-stream": "^1.2.1",
"user-agents": "^1.0.1444"
"webtorrent": "^2.1.35"
},
"devDependencies": {
"@types/amqplib": "^0.10.4",
"@types/magnet-uri": "^5.1.5",
"@types/node": "^20.11.16",
"@types/stremio-addon-sdk": "^1.6.10",
"@types/torrent-stream": "^0.0.9",
"@types/validator": "^13.11.8",
"@typescript-eslint/eslint-plugin": "^6.21.0",
"@typescript-eslint/parser": "^6.21.0",

6
src/node/consumer/src/jobs/process_torrents_job.ts
View File

@@ -14,9 +14,9 @@ export class ProcessTorrentsJob implements IProcessTorrentsJob {
private readonly consumeQueueOptions: Options.Consume = {noAck: false};
private torrentProcessingService: ITorrentProcessingService;
private logger: ILoggingService;
constructor(@inject(IocTypes.ITorrentProcessingService) torrentProcessingService: ITorrentProcessingService,
@inject(IocTypes.ILoggingService) logger: ILoggingService){
@inject(IocTypes.ILoggingService) logger: ILoggingService) {
this.torrentProcessingService = torrentProcessingService;
this.logger = logger;
}
@@ -55,7 +55,7 @@ export class ProcessTorrentsJob implements IProcessTorrentsJob {
this.logger.error('Failed processing torrent', error);
}
}
try {
await channel.assertQueue(configurationService.rabbitConfig.QUEUE_NAME, this.assertQueueOptions);
await channel.prefetch(configurationService.jobConfig.JOB_CONCURRENCY);

2
src/node/consumer/src/lib/helpers/boolean_helpers.ts
View File

@@ -1,5 +1,5 @@
export const BooleanHelpers = {
parseBool: function(value: string | number | undefined, defaultValue: boolean): boolean {
parseBool: function (value: string | number | undefined, defaultValue: boolean): boolean {
switch (typeof value) {
case 'string':
return parseStringToBool(value, defaultValue);

10
src/node/consumer/src/lib/helpers/promises_helpers.ts
View File

@@ -1,11 +1,11 @@
export const PromiseHelpers = {
sequence: async function(promises: (() => Promise<any>)[]) {
sequence: async function (promises: (() => Promise<any>)[]) {
return promises.reduce((promise, func) =>
promise.then(result => func().then(res => result.concat(res))), Promise.resolve([]));
},
first: async function(promises) {
first: async function (promises) {
return Promise.all(promises.map(p => {
return p.then((val) => Promise.reject(val), (err) => Promise.resolve(err));
})).then(
@@ -14,11 +14,11 @@ export const PromiseHelpers = {
);
},
delay: async function(duration: number) {
delay: async function (duration: number) {
return new Promise<void>(resolve => setTimeout(() => resolve(), duration));
},
timeout: async function(timeoutMs: number, promise, message = 'Timed out') {
timeout: async function (timeoutMs: number, promise, message = 'Timed out') {
return Promise.race([
promise,
new Promise(function (resolve, reject) {
@@ -29,7 +29,7 @@ export const PromiseHelpers = {
]);
},
mostCommonValue: function(array) {
mostCommonValue: function (array) {
return array.sort((a, b) => array.filter(v => v === a).length - array.filter(v => v === b).length).pop();
}
};

7
src/node/consumer/src/lib/interfaces/cinemeta_metadata.ts
View File

@@ -6,6 +6,7 @@ export interface ICinemetaJsonResponse {
links?: ICinemetaLink[];
behaviorHints?: ICinemetaBehaviorHints;
}
export interface ICinemetaMetaData {
awards?: string;
cast?: string[];
@@ -37,6 +38,7 @@ export interface ICinemetaMetaData {
releaseInfo?: string;
videos?: ICinemetaVideo[];
}
export interface ICinemetaPopularities {
PXS_TEST?: number;
PXS?: number;
@@ -49,10 +51,12 @@ export interface ICinemetaPopularities {
stremio?: number;
stremio_lib?: number;
}
export interface ICinemetaTrailer {
source?: string;
type?: string;
}
export interface ICinemetaVideo extends ICommonVideoMetadata {
name?: string;
number?: number;
@@ -63,15 +67,18 @@ export interface ICinemetaVideo extends ICommonVideoMetadata {
thumbnail?: string;
description?: string;
}
export interface ICinemetaTrailerStream {
title?: string;
ytId?: string;
}
export interface ICinemetaLink {
name?: string;
category?: string;
url?: string;
}
export interface ICinemetaBehaviorHints {
defaultVideoId?: null;
hasScheduledVideos?: boolean;

4
src/node/consumer/src/lib/interfaces/kitsu_metadata.ts
View File

@@ -4,6 +4,7 @@ export interface IKitsuJsonResponse {
cacheMaxAge?: number;
meta?: IKitsuMeta;
}
export interface IKitsuMeta {
aliases?: string[];
animeType?: string;
@@ -29,16 +30,19 @@ export interface IKitsuMeta {
videos?: IKitsuVideo[];
year?: string;
}
export interface IKitsuVideo extends ICommonVideoMetadata {
imdbEpisode?: number;
imdbSeason?: number;
imdb_id?: string;
thumbnail?: string;
}
export interface IKitsuTrailer {
source?: string;
type?: string;
}
export interface IKitsuLink {
name?: string;
category?: string;

3
src/node/consumer/src/lib/interfaces/logging_service.ts
View File

@@ -1,6 +1,9 @@
export interface ILoggingService {
info(message: string, ...args: any[]): void;
error(message: string, ...args: any[]): void;
debug(message: string, ...args: any[]): void;
warn(message: string, ...args: any[]): void;
}

2
src/node/consumer/src/lib/interfaces/metadata_query.ts
View File

@@ -1,4 +1,4 @@
export interface IMetaDataQuery {
export interface IMetaDataQuery {
title?: string
type?: string
year?: number | string

1
src/node/consumer/src/lib/models/composition_root.ts
View File

@@ -10,6 +10,7 @@ export class CompositionalRoot implements ICompositionalRoot {
private trackerService: ITrackerService;
private databaseRepository: IDatabaseRepository;
private processTorrentsJob: IProcessTorrentsJob;
constructor(@inject(IocTypes.ITrackerService) trackerService: ITrackerService,
@inject(IocTypes.IDatabaseRepository) databaseRepository: IDatabaseRepository,
@inject(IocTypes.IProcessTorrentsJob) processTorrentsJob: IProcessTorrentsJob) {

1
src/node/consumer/src/lib/models/configuration/torrent_config.ts
View File

@@ -1,4 +1,5 @@
export const torrentConfig = {
MAX_CONNECTIONS_PER_TORRENT: parseInt(process.env.MAX_SINGLE_TORRENT_CONNECTIONS || "20", 10),
MAX_CONNECTIONS_OVERALL: parseInt(process.env.MAX_CONNECTIONS_OVERALL || "100", 10),
TIMEOUT: parseInt(process.env.TORRENT_TIMEOUT || "30000", 10)
};

6
src/node/consumer/src/lib/models/inversify_config.ts
View File

@@ -1,6 +1,6 @@
import "reflect-metadata"; // required
import {Container} from "inversify";
import { IocTypes } from "./ioc_types";
import {IocTypes} from "./ioc_types";
import {ICacheService} from "../interfaces/cache_service";
import {ILoggingService} from "../interfaces/logging_service";
import {IMetadataService} from "../interfaces/metadata_service";
@@ -32,13 +32,13 @@ serviceContainer.bind<ICompositionalRoot>(IocTypes.ICompositionalRoot).to(Compos
serviceContainer.bind<ICacheService>(IocTypes.ICacheService).to(CacheService).inSingletonScope();
serviceContainer.bind<ILoggingService>(IocTypes.ILoggingService).to(LoggingService).inSingletonScope();
serviceContainer.bind<ITrackerService>(IocTypes.ITrackerService).to(TrackerService).inSingletonScope();
serviceContainer.bind<ITorrentDownloadService>(IocTypes.ITorrentDownloadService).to(TorrentDownloadService).inSingletonScope();
serviceContainer.bind<ITorrentFileService>(IocTypes.ITorrentFileService).to(TorrentFileService);
serviceContainer.bind<ITorrentProcessingService>(IocTypes.ITorrentProcessingService).to(TorrentProcessingService);
serviceContainer.bind<ITorrentSubtitleService>(IocTypes.ITorrentSubtitleService).to(TorrentSubtitleService);
serviceContainer.bind<ITorrentEntriesService>(IocTypes.ITorrentEntriesService).to(TorrentEntriesService);
serviceContainer.bind<ITorrentDownloadService>(IocTypes.ITorrentDownloadService).to(TorrentDownloadService);
serviceContainer.bind<IMetadataService>(IocTypes.IMetadataService).to(MetadataService);
serviceContainer.bind<IDatabaseRepository>(IocTypes.IDatabaseRepository).to(DatabaseRepository);
serviceContainer.bind<IProcessTorrentsJob>(IocTypes.IProcessTorrentsJob).to(ProcessTorrentsJob);
export { serviceContainer };
export {serviceContainer};

4
src/node/consumer/src/lib/services/cache_service.ts
View File

@@ -25,7 +25,7 @@ export class CacheService implements ICacheService {
private logger: ILoggingService;
private readonly memoryCache: MemoryCache;
private readonly remoteCache: Cache<Store> | MemoryCache;
constructor(@inject(IocTypes.ILoggingService) logger: ILoggingService) {
this.logger = logger;
if (configurationService.cacheConfig.NO_CACHE) {
@@ -59,7 +59,7 @@ export class CacheService implements ICacheService {
collectionName: configurationService.cacheConfig.COLLECTION_NAME,
ttl: GLOBAL_TTL,
url: configurationService.cacheConfig.MONGO_URI,
mongoConfig:{
mongoConfig: {
socketTimeoutMS: 120000,
appName: 'knightcrawler-consumer',
}

1
src/node/consumer/src/lib/services/metadata_service.ts
View File

@@ -20,6 +20,7 @@ const TIMEOUT = 20000;
@injectable()
export class MetadataService implements IMetadataService {
private cacheService: ICacheService;
constructor(@inject(IocTypes.ICacheService) cacheService: ICacheService) {
this.cacheService = cacheService;
}

104
src/node/consumer/src/lib/services/torrent_download_service.ts
View File

@@ -1,5 +1,4 @@
import {encode} from 'magnet-uri';
import torrentStream from 'torrent-stream';
import {configurationService} from './configuration_service';
import {ExtensionHelpers} from '../helpers/extension_helpers';
import {ITorrentFileCollection} from "../interfaces/torrent_file_collection";
@@ -9,7 +8,10 @@ import {ISubtitleAttributes} from "../../repository/interfaces/subtitle_attribut
import {IContentAttributes} from "../../repository/interfaces/content_attributes";
import {parse} from "parse-torrent-title";
import {ITorrentDownloadService} from "../interfaces/torrent_download_service";
import {injectable} from "inversify";
import {inject, injectable} from "inversify";
import {ILoggingService} from "../interfaces/logging_service";
import {IocTypes} from "../models/ioc_types";
import WebTorrent from "webtorrent";
interface ITorrentFile {
name: string;
@@ -18,15 +20,28 @@ interface ITorrentFile {
fileIndex: number;
}
const clientOptions = {
maxConns: configurationService.torrentConfig.MAX_CONNECTIONS_OVERALL,
}
const torrentOptions = {
skipVerify: true,
addUID: true,
destroyStoreOnDestroy: true,
private: true,
maxWebConns: configurationService.torrentConfig.MAX_CONNECTIONS_PER_TORRENT,
}
@injectable()
export class TorrentDownloadService implements ITorrentDownloadService {
private engineOptions: TorrentStream.TorrentEngineOptions = {
connections: configurationService.torrentConfig.MAX_CONNECTIONS_PER_TORRENT,
uploads: 0,
verify: false,
dht: false,
tracker: true,
};
private torrentClient: WebTorrent.Instance;
private logger: ILoggingService;
constructor(@inject(IocTypes.ILoggingService) logger: ILoggingService) {
this.logger = logger;
this.torrentClient = new WebTorrent(clientOptions);
this.torrentClient.on('error', errors => this.logClientErrors(errors));
}
public getTorrentFiles = async (torrent: IParsedTorrent, timeout: number = 30000): Promise<ITorrentFileCollection> => {
const torrentFiles: ITorrentFile[] = await this.filesFromTorrentStream(torrent, timeout);
@@ -48,29 +63,32 @@ export class TorrentDownloadService implements ITorrentDownloadService {
}
const magnet = encode({infoHash: torrent.infoHash, announce: torrent.trackers.split(',')});
return new Promise((resolve, reject) => {
let engine: TorrentStream.TorrentEngine;
this.logger.debug(`Constructing torrent stream for ${torrent.title} with magnet ${magnet}`);
return new Promise((resolve, reject) => {
const timeoutId = setTimeout(() => {
engine.destroy(() => {
});
this.torrentClient.remove(magnet, {destroyStore: true});
reject(new Error('No available connections for torrent!'));
}, timeout);
engine = torrentStream(magnet, this.engineOptions);
this.logger.debug(`Adding torrent with infoHash ${torrent.infoHash}`);
this.torrentClient.add(magnet, torrentOptions, (torrent) => {
engine.on("ready", () => {
const files: ITorrentFile[] = engine.files.map((file, fileId) => ({
...file,
this.logger.debug(`torrent with infoHash ${torrent.infoHash} added to client.`);
const files: ITorrentFile[] = torrent.files.map((file, fileId) => ({
fileIndex: fileId,
size: file.length,
title: file.name
length: file.length,
name: file.name,
path: file.path,
}));
this.logger.debug(`Found ${files.length} files in torrent ${torrent.infoHash}`);
resolve(files);
engine.destroy(() => {
});
this.torrentClient.remove(magnet, {destroyStore: true});
clearTimeout(timeoutId);
});
});
@@ -86,12 +104,12 @@ export class TorrentDownloadService implements ITorrentDownloadService {
const minAnimeExtraRatio = 5;
const minRedundantRatio = videos.length <= 3 ? 30 : Number.MAX_VALUE;
const isSample = (video: ITorrentFile) => video.path?.match(/sample|bonus|promo/i) && maxSize / parseInt(video.path.toString()) > minSampleRatio;
const isRedundant = (video: ITorrentFile) => maxSize / parseInt(video.path.toString()) > minRedundantRatio;
const isExtra = (video: ITorrentFile) => video.path?.match(/extras?\//i);
const isAnimeExtra = (video: ITorrentFile) => video.path?.match(/(?:\b|_)(?:NC)?(?:ED|OP|PV)(?:v?\d\d?)?(?:\b|_)/i)
const isSample = (video: ITorrentFile) => video.path.toString()?.match(/sample|bonus|promo/i) && maxSize / video.length > minSampleRatio;
const isRedundant = (video: ITorrentFile) => maxSize / video.length > minRedundantRatio;
const isExtra = (video: ITorrentFile) => video.path.toString()?.match(/extras?\//i);
const isAnimeExtra = (video: ITorrentFile) => video.path.toString()?.match(/(?:\b|_)(?:NC)?(?:ED|OP|PV)(?:v?\d\d?)?(?:\b|_)/i)
&& maxSize / parseInt(video.length.toString()) > minAnimeExtraRatio;
const isWatermark = (video: ITorrentFile) => video.path?.match(/^[A-Z-]+(?:\.[A-Z]+)?\.\w{3,4}$/)
const isWatermark = (video: ITorrentFile) => video.path.toString()?.match(/^[A-Z-]+(?:\.[A-Z]+)?\.\w{3,4}$/)
&& maxSize / parseInt(video.length.toString()) > minAnimeExtraRatio
return videos
@@ -109,19 +127,23 @@ export class TorrentDownloadService implements ITorrentDownloadService {
private createContent = (torrent: IParsedTorrent, torrentFiles: ITorrentFile[]): IContentAttributes[] => torrentFiles.map(file => this.mapTorrentFileToContentAttributes(torrent, file));
private mapTorrentFileToFileAttributes = (torrent: IParsedTorrent, file: ITorrentFile): IFileAttributes => {
const videoFile: IFileAttributes = {
title: file.name,
size: file.length,
fileIndex: file.fileIndex || 0,
infoHash: torrent.infoHash,
imdbId: torrent.imdbId.toString(),
imdbSeason: torrent.season || 0,
imdbEpisode: torrent.episode || 0,
kitsuId: parseInt(torrent.kitsuId.toString()) || 0,
kitsuEpisode: torrent.episode || 0
};
return {...videoFile, ...parse(file.name)};
try {
const videoFile: IFileAttributes = {
title: file.name,
size: file.length,
fileIndex: file.fileIndex || 0,
infoHash: torrent.infoHash,
imdbId: torrent.imdbId.toString(),
imdbSeason: torrent.season || 0,
imdbEpisode: torrent.episode || 0,
kitsuId: parseInt(torrent.kitsuId?.toString()) || 0,
kitsuEpisode: torrent.episode || 0
};
return {...videoFile, ...parse(file.name)};
} catch (error) {
throw new Error(`Error parsing file ${file.name} from torrent ${torrent.infoHash}: ${error}`);
}
};
private mapTorrentFileToSubtitleAttributes = (torrent: IParsedTorrent, file: ITorrentFile): ISubtitleAttributes => ({
@@ -138,5 +160,9 @@ export class TorrentDownloadService implements ITorrentDownloadService {
path: file.path,
size: file.length,
});
private logClientErrors(errors: Error | string) {
this.logger.error(`Error in torrent client: ${errors}`);
}
}

4
src/node/consumer/src/lib/services/torrent_entries_service.ts
View File

@@ -23,7 +23,7 @@ export class TorrentEntriesService implements ITorrentEntriesService {
private fileService: ITorrentFileService;
private subtitleService: ITorrentSubtitleService;
private repository: IDatabaseRepository;
constructor(@inject(IocTypes.IMetadataService) metadataService: IMetadataService,
@inject(IocTypes.ILoggingService) logger: ILoggingService,
@inject(IocTypes.ITorrentFileService) fileService: ITorrentFileService,
@@ -73,7 +73,7 @@ export class TorrentEntriesService implements ITorrentEntriesService {
const fileCollection: ITorrentFileCollection = await this.fileService.parseTorrentFiles(torrent)
.then((torrentContents: ITorrentFileCollection) => overwrite ? this.overwriteExistingFiles(torrent, torrentContents) : torrentContents)
.then((torrentContents: ITorrentFileCollection) =>this.subtitleService.assignSubtitles(torrentContents))
.then((torrentContents: ITorrentFileCollection) => this.subtitleService.assignSubtitles(torrentContents))
.catch(error => {
this.logger.warn(`Failed getting files for ${torrent.title}`, error.message);
return {};

35
src/node/consumer/src/lib/services/torrent_file_service.ts
View File

@@ -27,7 +27,11 @@ export class TorrentFileService implements ITorrentFileService {
private metadataService: IMetadataService;
private torrentDownloadService: ITorrentDownloadService;
private logger: ILoggingService;
private readonly imdb_limiter: Bottleneck = new Bottleneck({
maxConcurrent: configurationService.metadataConfig.IMDB_CONCURRENT,
minTime: configurationService.metadataConfig.IMDB_INTERVAL_MS
});
constructor(@inject(IocTypes.IMetadataService) metadataService: IMetadataService,
@inject(IocTypes.ITorrentDownloadService) torrentDownloadService: ITorrentDownloadService,
@inject(IocTypes.ILoggingService) logger: ILoggingService) {
@@ -35,11 +39,6 @@ export class TorrentFileService implements ITorrentFileService {
this.torrentDownloadService = torrentDownloadService;
this.logger = logger;
}
private readonly imdb_limiter: Bottleneck = new Bottleneck({
maxConcurrent: configurationService.metadataConfig.IMDB_CONCURRENT,
minTime: configurationService.metadataConfig.IMDB_INTERVAL_MS
});
public parseTorrentFiles = async (torrent: IParsedTorrent): Promise<ITorrentFileCollection> => {
const parsedTorrentName = parse(torrent.title);
@@ -85,8 +84,8 @@ export class TorrentFileService implements ITorrentFileService {
const parsedTorrentName = parse(torrent.title);
const hasMovies = parsedTorrentName.complete || !!torrent.title.match(/movies?(?:\W|$)/i);
const parsedVideos = videos.map(video => this.parseSeriesVideo(video));
return parsedVideos.map(video => ({ ...video, isMovie: this.isMovieVideo(torrent, video, parsedVideos, hasMovies) }));
return parsedVideos.map(video => ({...video, isMovie: this.isMovieVideo(torrent, video, parsedVideos, hasMovies)}));
};
private parseMovieFiles = async (torrent: IParsedTorrent, metadata: IMetadataResponse): Promise<ITorrentFileCollection> => {
@@ -100,8 +99,8 @@ export class TorrentFileService implements ITorrentFileService {
fileIndex: video.fileIndex,
title: video.path || torrent.title,
size: video.size || torrent.size,
imdbId: torrent.imdbId.toString() || metadata && metadata.imdbId.toString(),
kitsuId: parseInt(torrent.kitsuId.toString() || metadata && metadata.kitsuId.toString())
imdbId: torrent.imdbId?.toString() || metadata && metadata.imdbId?.toString(),
kitsuId: parseInt(torrent.kitsuId?.toString() || metadata && metadata.kitsuId?.toString())
}));
return {...fileCollection, videos: parsedVideos};
}
@@ -140,11 +139,11 @@ export class TorrentFileService implements ITorrentFileService {
.catch(error => {
if (!this.isPackTorrent(torrent)) {
const entries = [{name: torrent.title, path: torrent.title, size: torrent.size, fileIndex: null}];
return { videos: entries, contents:[], subtitles: [], files: entries}
return {videos: entries, contents: [], subtitles: [], files: entries}
}
return Promise.reject(error);
});
if (files.contents && files.contents.length && !files.videos.length && this.isDiskTorrent(files.contents)) {
files.videos = [{name: torrent.title, path: torrent.title, size: torrent.size, fileIndex: null}];
}
@@ -197,22 +196,22 @@ export class TorrentFileService implements ITorrentFileService {
};
private mapSeriesMovie = async (torrent: IParsedTorrent, file: IFileAttributes): Promise<IFileAttributes[]> => {
const kitsuId= torrent.type === TorrentType.Anime ? await this.findMovieKitsuId(file)
const kitsuId = torrent.type === TorrentType.Anime ? await this.findMovieKitsuId(file)
.then(result => {
if (result instanceof Error) {
this.logger.warn(`Failed to retrieve kitsuId due to error: ${result.message}`);
return undefined;
}
return result;
}): undefined;
}) : undefined;
const imdbId = !kitsuId ? await this.findMovieImdbId(file) : undefined;
const query: IMetaDataQuery = {
id: kitsuId || imdbId,
type: TorrentType.Movie
};
const metadataOrError = await this.metadataService.getMetadata(query);
if (metadataOrError instanceof Error) {
this.logger.warn(`Failed to retrieve metadata due to error: ${metadataOrError.message}`);
@@ -247,7 +246,7 @@ export class TorrentFileService implements ITorrentFileService {
}];
};
private decomposeEpisodes = async (torrent: IParsedTorrent, files: IFileAttributes[], metadata: IMetadataResponse = { episodeCount: [] }) => {
private decomposeEpisodes = async (torrent: IParsedTorrent, files: IFileAttributes[], metadata: IMetadataResponse = {episodeCount: []}) => {
if (files.every(file => !file.episodes && !file.date)) {
return files;
}
@@ -635,7 +634,7 @@ export class TorrentFileService implements ITorrentFileService {
videoInfo.episode = videoInfo.episodes && videoInfo.episodes[0];
}
return { ...video, ...videoInfo };
return {...video, ...videoInfo};
};
private isMovieVideo = (torrent: IParsedTorrent, video: IFileAttributes, otherVideos: IFileAttributes[], hasMovies: boolean): boolean => {

5
src/node/consumer/src/lib/services/torrent_processing_service.ts
View File

@@ -13,9 +13,10 @@ export class TorrentProcessingService implements ITorrentProcessingService {
private torrentEntriesService: ITorrentEntriesService;
private logger: ILoggingService;
private trackerService: ITrackerService;
constructor(@inject(IocTypes.ITorrentEntriesService) torrentEntriesService: ITorrentEntriesService,
@inject(IocTypes.ILoggingService) logger: ILoggingService,
@inject(IocTypes.ITrackerService) trackerService: ITrackerService){
@inject(IocTypes.ITrackerService) trackerService: ITrackerService) {
this.torrentEntriesService = torrentEntriesService;
this.logger = logger;
this.trackerService = trackerService;
@@ -62,6 +63,6 @@ export class TorrentProcessingService implements ITorrentProcessingService {
}
return torrent.imdb;
};
};
}

4
src/node/consumer/src/lib/services/torrent_subtitle_service.ts
View File

@@ -28,7 +28,7 @@ export class TorrentSubtitleService implements ITorrentSubtitleService {
return fileCollection;
};
private parseVideo = (video: IFileAttributes)=> {
private parseVideo = (video: IFileAttributes) => {
const fileName = video.title.split('/').pop().replace(/\.(\w{2,4})$/, '');
const folderName = video.title.replace(/\/?[^/]+$/, '');
return {
@@ -68,7 +68,7 @@ export class TorrentSubtitleService implements ITorrentSubtitleService {
return undefined;
}
private singleVideoFile = (videos: any[])=> {
private singleVideoFile = (videos: any[]) => {
return new Set(videos.map(v => v.videoFile.fileIndex)).size === 1;
}

4
src/node/consumer/src/lib/services/tracker_service.ts
View File

@@ -10,7 +10,7 @@ import {ILoggingService} from "../interfaces/logging_service";
export class TrackerService implements ITrackerService {
private cacheService: ICacheService;
private logger: ILoggingService;
constructor(@inject(IocTypes.ICacheService) cacheService: ICacheService,
@inject(IocTypes.ILoggingService) logger: ILoggingService) {
this.cacheService = cacheService;
@@ -19,7 +19,7 @@ export class TrackerService implements ITrackerService {
public getTrackers = async (): Promise<string[]> => this.cacheService.cacheTrackers(this.downloadTrackers);
private downloadTrackers = async(): Promise<string[]> => {
private downloadTrackers = async (): Promise<string[]> => {
const response: AxiosResponse<string> = await axios.get(configurationService.trackerConfig.TRACKERS_URL);
const trackersListText: string = response.data;
// Trackers are separated by a newline character

22
src/node/consumer/src/repository/database_repository.ts
View File

@@ -21,7 +21,7 @@ import {IDatabaseRepository} from "./interfaces/database_repository";
@injectable()
export class DatabaseRepository implements IDatabaseRepository {
private readonly database: Sequelize;
private models = [
Torrent,
Provider,
@@ -32,7 +32,7 @@ export class DatabaseRepository implements IDatabaseRepository {
IngestedTorrent,
IngestedPage];
private logger: ILoggingService;
constructor(@inject(IocTypes.ILoggingService) logger: ILoggingService) {
this.logger = logger;
this.database = this.createDatabase();
@@ -41,7 +41,8 @@ export class DatabaseRepository implements IDatabaseRepository {
public connect = async () => {
try {
await this.database.sync({alter: configurationService.databaseConfig.AUTO_CREATE_AND_APPLY_MIGRATIONS});
} catch {
} catch (error) {
this.logger.debug('Failed to sync database', error);
this.logger.error('Failed syncing database');
process.exit(1);
}
@@ -112,9 +113,14 @@ export class DatabaseRepository implements IDatabaseRepository {
});
public createTorrent = async (torrent: Torrent): Promise<void> => {
await Torrent.upsert(torrent);
await this.createContents(torrent.infoHash, torrent.contents);
await this.createSubtitles(torrent.infoHash, torrent.subtitles);
try {
await Torrent.upsert(torrent);
await this.createContents(torrent.infoHash, torrent.contents);
await this.createSubtitles(torrent.infoHash, torrent.subtitles);
} catch (error) {
this.logger.error(`Failed to create torrent: ${torrent.infoHash}`);
this.logger.debug(error);
}
};
public setTorrentSeeders = async (torrent: ITorrentAttributes, seeders: number): Promise<[number]> => {
@@ -211,9 +217,9 @@ export class DatabaseRepository implements IDatabaseRepository {
logging: false
}
);
newDatabase.addModels(this.models);
return newDatabase;
};
}

18
src/node/consumer/src/repository/models/content.ts
View File

@@ -1,22 +1,22 @@
import {Table, Column, Model, HasMany, DataType, BelongsTo, ForeignKey} from 'sequelize-typescript';
import {BelongsTo, Column, DataType, ForeignKey, Model, Table} from 'sequelize-typescript';
import {IContentAttributes, IContentCreationAttributes} from "../interfaces/content_attributes";
import {Torrent} from "./torrent";
@Table({modelName: 'content', timestamps: false})
export class Content extends Model<IContentAttributes, IContentCreationAttributes> {
@Column({ type: DataType.STRING(64), primaryKey: true, allowNull: false, onDelete: 'CASCADE' })
@Column({type: DataType.STRING(64), primaryKey: true, allowNull: false, onDelete: 'CASCADE'})
@ForeignKey(() => Torrent)
declare infoHash: string;
@Column({ type: DataType.INTEGER, primaryKey: true, allowNull: false })
@Column({type: DataType.INTEGER, primaryKey: true, allowNull: false})
declare fileIndex: number;
@Column({ type: DataType.STRING(512), allowNull: false })
@Column({type: DataType.STRING(512), allowNull: false})
declare path: string;
@Column({ type: DataType.BIGINT })
@Column({type: DataType.BIGINT})
declare size: number;
@BelongsTo(() => Torrent, { constraints: false, foreignKey: 'infoHash' })
@BelongsTo(() => Torrent, {constraints: false, foreignKey: 'infoHash'})
torrent: Torrent;
}

47
src/node/consumer/src/repository/models/file.ts
View File

@@ -1,8 +1,7 @@
import {Table, Column, Model, HasMany, DataType, BelongsTo, ForeignKey} from 'sequelize-typescript';
import {BelongsTo, Column, DataType, ForeignKey, HasMany, Model, Table} from 'sequelize-typescript';
import {IFileAttributes, IFileCreationAttributes} from "../interfaces/file_attributes";
import {Torrent} from "./torrent";
import {Subtitle} from "./subtitle";
import {ISubtitleAttributes} from "../interfaces/subtitle_attributes";
const indexes = [
{
@@ -18,43 +17,43 @@ const indexes = [
'kitsuEpisode'
]
},
{ unique: false, fields: ['imdbId', 'imdbSeason', 'imdbEpisode'] },
{ unique: false, fields: ['kitsuId', 'kitsuEpisode'] }
{unique: false, fields: ['imdbId', 'imdbSeason', 'imdbEpisode']},
{unique: false, fields: ['kitsuId', 'kitsuEpisode']}
];
@Table({modelName: 'file', timestamps: true, indexes: indexes })
@Table({modelName: 'file', timestamps: true, indexes: indexes})
export class File extends Model<IFileAttributes, IFileCreationAttributes> {
@Column({ type: DataType.STRING(64), allowNull: false, onDelete: 'CASCADE' })
@Column({type: DataType.STRING(64), allowNull: false, onDelete: 'CASCADE'})
@ForeignKey(() => Torrent)
declare infoHash: string;
@Column({ type: DataType.INTEGER})
@Column({type: DataType.INTEGER})
declare fileIndex: number;
@Column({ type: DataType.STRING(512), allowNull: false })
@Column({type: DataType.STRING(512), allowNull: false})
declare title: string;
@Column({ type: DataType.BIGINT })
@Column({type: DataType.BIGINT})
declare size: number;
@Column({ type: DataType.STRING(32) })
@Column({type: DataType.STRING(32)})
declare imdbId: string;
@Column({ type: DataType.INTEGER })
@Column({type: DataType.INTEGER})
declare imdbSeason: number;
@Column({ type: DataType.INTEGER })
@Column({type: DataType.INTEGER})
declare imdbEpisode: number;
@Column({ type: DataType.INTEGER })
@Column({type: DataType.INTEGER})
declare kitsuId: number;
@Column({ type: DataType.INTEGER })
@Column({type: DataType.INTEGER})
declare kitsuEpisode: number;
@HasMany(() => Subtitle, { constraints: false, foreignKey: 'fileId'})
@HasMany(() => Subtitle, {constraints: false, foreignKey: 'fileId'})
declare subtitles?: Subtitle[];
@BelongsTo(() => Torrent, { constraints: false, foreignKey: 'infoHash' })
@BelongsTo(() => Torrent, {constraints: false, foreignKey: 'infoHash'})
torrent: Torrent;
}

4
src/node/consumer/src/repository/models/ingestedPage.ts
View File

@@ -1,4 +1,4 @@
import { Table, Column, Model, HasMany, DataType } from 'sequelize-typescript';
import {Column, DataType, Model, Table} from 'sequelize-typescript';
import {IIngestedPageAttributes, IIngestedPageCreationAttributes} from "../interfaces/ingested_page_attributes";
const indexes = [
@@ -11,6 +11,6 @@ const indexes = [
@Table({modelName: 'ingested_page', timestamps: true, indexes: indexes})
export class IngestedPage extends Model<IIngestedPageAttributes, IIngestedPageCreationAttributes> {
@Column({ type: DataType.STRING(512), allowNull: false })
@Column({type: DataType.STRING(512), allowNull: false})
declare url: string;
}

36
src/node/consumer/src/repository/models/ingestedTorrent.ts
View File

@@ -1,4 +1,4 @@
import { Table, Column, Model, HasMany, DataType } from 'sequelize-typescript';
import {Column, DataType, Model, Table} from 'sequelize-typescript';
import {IIngestedTorrentAttributes, IIngestedTorrentCreationAttributes} from "../interfaces/ingested_torrent_attributes";
const indexes = [
@@ -11,30 +11,30 @@ const indexes = [
@Table({modelName: 'ingested_torrent', timestamps: true, indexes: indexes})
export class IngestedTorrent extends Model<IIngestedTorrentAttributes, IIngestedTorrentCreationAttributes> {
@Column({ type: DataType.STRING(512) })
@Column({type: DataType.STRING(512)})
declare name: string;
@Column({ type: DataType.STRING(512) })
@Column({type: DataType.STRING(512)})
declare source: string;
@Column({ type: DataType.STRING(32) })
@Column({type: DataType.STRING(32)})
declare category: string;
@Column({ type: DataType.STRING(64) })
@Column({type: DataType.STRING(64)})
declare info_hash: string;
@Column({ type: DataType.STRING(32) })
@Column({type: DataType.STRING(32)})
declare size: string;
@Column({ type: DataType.INTEGER })
@Column({type: DataType.INTEGER})
declare seeders: number;
@Column({ type: DataType.INTEGER })
@Column({type: DataType.INTEGER})
declare leechers: number;
@Column({ type: DataType.STRING(32) })
@Column({type: DataType.STRING(32)})
declare imdb: string;
@Column({ type: DataType.BOOLEAN, defaultValue: false })
@Column({type: DataType.BOOLEAN, defaultValue: false})
declare processed: boolean;
}

14
src/node/consumer/src/repository/models/provider.ts
View File

@@ -1,15 +1,15 @@
import { Table, Column, Model, HasMany, DataType } from 'sequelize-typescript';
import {Column, DataType, Model, Table} from 'sequelize-typescript';
import {IProviderAttributes, IProviderCreationAttributes} from "../interfaces/provider_attributes";
@Table({modelName: 'provider', timestamps: false})
export class Provider extends Model<IProviderAttributes, IProviderCreationAttributes> {
@Column({ type: DataType.STRING(32), primaryKey: true })
@Column({type: DataType.STRING(32), primaryKey: true})
declare name: string;
@Column({ type: DataType.DATE })
@Column({type: DataType.DATE})
declare lastScraped: Date;
@Column({ type: DataType.STRING(128) })
@Column({type: DataType.STRING(128)})
declare lastScrapedId: string;
}

6
src/node/consumer/src/repository/models/skipTorrent.ts
View File

@@ -1,10 +1,10 @@
import { Table, Column, Model, HasMany, DataType } from 'sequelize-typescript';
import {Column, DataType, Model, Table} from 'sequelize-typescript';
import {ISkipTorrentAttributes, ISkipTorrentCreationAttributes} from "../interfaces/skip_torrent_attributes";
@Table({modelName: 'skip_torrent', timestamps: false})
export class SkipTorrent extends Model<ISkipTorrentAttributes, ISkipTorrentCreationAttributes> {
@Column({ type: DataType.STRING(64), primaryKey: true })
@Column({type: DataType.STRING(64), primaryKey: true})
declare infoHash: string;
}

25
src/node/consumer/src/repository/models/subtitle.ts
View File

@@ -1,7 +1,6 @@
import {Table, Column, Model, HasMany, DataType, BelongsTo, ForeignKey} from 'sequelize-typescript';
import {BelongsTo, Column, DataType, ForeignKey, Model, Table} from 'sequelize-typescript';
import {ISubtitleAttributes, ISubtitleCreationAttributes} from "../interfaces/subtitle_attributes";
import {File} from "./file";
import {Torrent} from "./torrent";
const indexes = [
{
@@ -13,27 +12,27 @@ const indexes = [
'fileId'
]
},
{ unique: false, fields: ['fileId'] }
{unique: false, fields: ['fileId']}
];
@Table({modelName: 'subtitle', timestamps: false, indexes: indexes})
export class Subtitle extends Model<ISubtitleAttributes, ISubtitleCreationAttributes> {
@Column({ type: DataType.STRING(64), allowNull: false, onDelete: 'CASCADE' })
@Column({type: DataType.STRING(64), allowNull: false, onDelete: 'CASCADE'})
declare infoHash: string;
@Column({ type: DataType.INTEGER, allowNull: false })
@Column({type: DataType.INTEGER, allowNull: false})
declare fileIndex: number;
@Column({ type: DataType.BIGINT, allowNull: true, onDelete: 'SET NULL' })
@Column({type: DataType.BIGINT, allowNull: true, onDelete: 'SET NULL'})
@ForeignKey(() => File)
declare fileId?: number | null;
@Column({ type: DataType.STRING(512), allowNull: false })
@Column({type: DataType.STRING(512), allowNull: false})
declare title: string;
@BelongsTo(() => File, { constraints: false, foreignKey: 'fileId' })
@BelongsTo(() => File, {constraints: false, foreignKey: 'fileId'})
file: File;
path: string;
}

32
src/node/consumer/src/repository/models/torrent.ts
View File

@@ -1,4 +1,4 @@
import { Table, Column, Model, HasMany, DataType } from 'sequelize-typescript';
import {Column, DataType, HasMany, Model, Table} from 'sequelize-typescript';
import {ITorrentAttributes, ITorrentCreationAttributes} from "../interfaces/torrent_attributes";
import {Content} from "./content";
import {File} from "./file";
@@ -9,48 +9,48 @@ import {Subtitle} from "./subtitle";
export class Torrent extends Model<ITorrentAttributes, ITorrentCreationAttributes> {
@Column({type: DataType.STRING(64), primaryKey: true})
declare infoHash: string;
@Column({type: DataType.STRING(32), allowNull: false})
declare provider: string;
@Column({type: DataType.STRING(512)})
declare torrentId: string;
@Column({type: DataType.STRING(512), allowNull: false})
declare title: string;
@Column({type: DataType.BIGINT})
declare size: number;
@Column({type: DataType.STRING(16), allowNull: false})
declare type: string;
@Column({type: DataType.DATE, allowNull: false})
declare uploadDate: Date;
@Column({type: DataType.SMALLINT})
declare seeders: number;
@Column({type: DataType.STRING(8000)})
declare trackers: string;
@Column({type: DataType.STRING(4096)})
declare languages: string;
@Column({type: DataType.STRING(16)})
declare resolution: string;
@Column({type: DataType.BOOLEAN, allowNull: false, defaultValue: false})
declare reviewed: boolean;
@Column({type: DataType.BOOLEAN, allowNull: false, defaultValue: false})
declare opened: boolean;
@HasMany(() => Content, { foreignKey: 'infoHash', constraints: false })
@HasMany(() => Content, {foreignKey: 'infoHash', constraints: false})
contents?: Content[];
@HasMany(() => File, { foreignKey: 'infoHash', constraints: false })
@HasMany(() => File, {foreignKey: 'infoHash', constraints: false})
files?: File[];
subtitles?: Subtitle[];
}

9
src/node/consumer/tsconfig.json
View File

@@ -8,8 +8,13 @@
"rootDir": "./src",
"sourceMap": true,
"target": "ES6",
"lib": ["es6"],
"types": ["node", "reflect-metadata"],
"lib": [
"es6"
],
"types": [
"node",
"reflect-metadata"
],
"esModuleInterop": true,
"experimentalDecorators": true,
"emitDecoratorMetadata": true,