tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

replace torrent-stream (high vunerabilities) with webtorrent, gives us clean package audit

Browse Source
This commit is contained in:
iPromKnight
2024-02-07 14:03:57 +00:00
committed by iPromKnight
parent 6e2b776211
commit 7fe9b64f66
32 changed files with 1846 additions and 758 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/node/consumer/.eslintrc
View File

@@ -64,7 +64,12 @@
},
"overrides": [
{
"files": ["*.ts", "*.mts", "*.cts", "*.tsx"],
"files": [
"*.ts",
"*.mts",
"*.cts",
"*.tsx"
],
"rules": {
"@typescript-eslint/explicit-function-return-type": "error",
"@typescript-eslint/consistent-type-assertions": [

2156
src/node/consumer/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

5
src/node/consumer/package.json
View File

@@ -27,15 +27,12 @@
"reflect-metadata": "^0.2.1",
"sequelize": "^6.36.0",
"sequelize-typescript": "^2.1.6",
"torrent-stream": "^1.2.1",
"user-agents": "^1.0.1444"
"webtorrent": "^2.1.35"
},
"devDependencies": {
"@types/amqplib": "^0.10.4",
"@types/magnet-uri": "^5.1.5",
"@types/node": "^20.11.16",
"@types/stremio-addon-sdk": "^1.6.10",
"@types/torrent-stream": "^0.0.9",
"@types/validator": "^13.11.8",
"@typescript-eslint/eslint-plugin": "^6.21.0",
"@typescript-eslint/parser": "^6.21.0",

7
src/node/consumer/src/lib/interfaces/cinemeta_metadata.ts
View File

@@ -6,6 +6,7 @@ export interface ICinemetaJsonResponse {
links?: ICinemetaLink[];
behaviorHints?: ICinemetaBehaviorHints;
}
export interface ICinemetaMetaData {
awards?: string;
cast?: string[];
@@ -37,6 +38,7 @@ export interface ICinemetaMetaData {
releaseInfo?: string;
videos?: ICinemetaVideo[];
}
export interface ICinemetaPopularities {
PXS_TEST?: number;
PXS?: number;
@@ -49,10 +51,12 @@ export interface ICinemetaPopularities {
stremio?: number;
stremio_lib?: number;
}
export interface ICinemetaTrailer {
source?: string;
type?: string;
}
export interface ICinemetaVideo extends ICommonVideoMetadata {
name?: string;
number?: number;
@@ -63,15 +67,18 @@ export interface ICinemetaVideo extends ICommonVideoMetadata {
thumbnail?: string;
description?: string;
}
export interface ICinemetaTrailerStream {
title?: string;
ytId?: string;
}
export interface ICinemetaLink {
name?: string;
category?: string;
url?: string;
}
export interface ICinemetaBehaviorHints {
defaultVideoId?: null;
hasScheduledVideos?: boolean;

4
src/node/consumer/src/lib/interfaces/kitsu_metadata.ts
View File

@@ -4,6 +4,7 @@ export interface IKitsuJsonResponse {
cacheMaxAge?: number;
meta?: IKitsuMeta;
}
export interface IKitsuMeta {
aliases?: string[];
animeType?: string;
@@ -29,16 +30,19 @@ export interface IKitsuMeta {
videos?: IKitsuVideo[];
year?: string;
}
export interface IKitsuVideo extends ICommonVideoMetadata {
imdbEpisode?: number;
imdbSeason?: number;
imdb_id?: string;
thumbnail?: string;
}
export interface IKitsuTrailer {
source?: string;
type?: string;
}
export interface IKitsuLink {
name?: string;
category?: string;

3
src/node/consumer/src/lib/interfaces/logging_service.ts
View File

@@ -1,6 +1,9 @@
export interface ILoggingService {
info(message: string, ...args: any[]): void;
error(message: string, ...args: any[]): void;
debug(message: string, ...args: any[]): void;
warn(message: string, ...args: any[]): void;
}

1
src/node/consumer/src/lib/models/composition_root.ts
View File

@@ -10,6 +10,7 @@ export class CompositionalRoot implements ICompositionalRoot {
private trackerService: ITrackerService;
private databaseRepository: IDatabaseRepository;
private processTorrentsJob: IProcessTorrentsJob;
constructor(@inject(IocTypes.ITrackerService) trackerService: ITrackerService,
@inject(IocTypes.IDatabaseRepository) databaseRepository: IDatabaseRepository,
@inject(IocTypes.IProcessTorrentsJob) processTorrentsJob: IProcessTorrentsJob) {

1
src/node/consumer/src/lib/models/configuration/torrent_config.ts
View File

@@ -1,4 +1,5 @@
export const torrentConfig = {
MAX_CONNECTIONS_PER_TORRENT: parseInt(process.env.MAX_SINGLE_TORRENT_CONNECTIONS || "20", 10),
MAX_CONNECTIONS_OVERALL: parseInt(process.env.MAX_CONNECTIONS_OVERALL || "100", 10),
TIMEOUT: parseInt(process.env.TORRENT_TIMEOUT || "30000", 10)
};

2
src/node/consumer/src/lib/models/inversify_config.ts
View File

@@ -32,11 +32,11 @@ serviceContainer.bind<ICompositionalRoot>(IocTypes.ICompositionalRoot).to(Compos
serviceContainer.bind<ICacheService>(IocTypes.ICacheService).to(CacheService).inSingletonScope();
serviceContainer.bind<ILoggingService>(IocTypes.ILoggingService).to(LoggingService).inSingletonScope();
serviceContainer.bind<ITrackerService>(IocTypes.ITrackerService).to(TrackerService).inSingletonScope();
serviceContainer.bind<ITorrentDownloadService>(IocTypes.ITorrentDownloadService).to(TorrentDownloadService).inSingletonScope();
serviceContainer.bind<ITorrentFileService>(IocTypes.ITorrentFileService).to(TorrentFileService);
serviceContainer.bind<ITorrentProcessingService>(IocTypes.ITorrentProcessingService).to(TorrentProcessingService);
serviceContainer.bind<ITorrentSubtitleService>(IocTypes.ITorrentSubtitleService).to(TorrentSubtitleService);
serviceContainer.bind<ITorrentEntriesService>(IocTypes.ITorrentEntriesService).to(TorrentEntriesService);
serviceContainer.bind<ITorrentDownloadService>(IocTypes.ITorrentDownloadService).to(TorrentDownloadService);
serviceContainer.bind<IMetadataService>(IocTypes.IMetadataService).to(MetadataService);
serviceContainer.bind<IDatabaseRepository>(IocTypes.IDatabaseRepository).to(DatabaseRepository);
serviceContainer.bind<IProcessTorrentsJob>(IocTypes.IProcessTorrentsJob).to(ProcessTorrentsJob);

1
src/node/consumer/src/lib/services/metadata_service.ts
View File

@@ -20,6 +20,7 @@ const TIMEOUT = 20000;
@injectable()
export class MetadataService implements IMetadataService {
private cacheService: ICacheService;
constructor(@inject(IocTypes.ICacheService) cacheService: ICacheService) {
this.cacheService = cacheService;
}

80
src/node/consumer/src/lib/services/torrent_download_service.ts
View File

@@ -1,5 +1,4 @@
import {encode} from 'magnet-uri';
import torrentStream from 'torrent-stream';
import {configurationService} from './configuration_service';
import {ExtensionHelpers} from '../helpers/extension_helpers';
import {ITorrentFileCollection} from "../interfaces/torrent_file_collection";
@@ -9,7 +8,10 @@ import {ISubtitleAttributes} from "../../repository/interfaces/subtitle_attribut
import {IContentAttributes} from "../../repository/interfaces/content_attributes";
import {parse} from "parse-torrent-title";
import {ITorrentDownloadService} from "../interfaces/torrent_download_service";
import {injectable} from "inversify";
import {inject, injectable} from "inversify";
import {ILoggingService} from "../interfaces/logging_service";
import {IocTypes} from "../models/ioc_types";
import WebTorrent from "webtorrent";
interface ITorrentFile {
name: string;
@@ -18,15 +20,28 @@ interface ITorrentFile {
fileIndex: number;
}
const clientOptions = {
maxConns: configurationService.torrentConfig.MAX_CONNECTIONS_OVERALL,
}
const torrentOptions = {
skipVerify: true,
addUID: true,
destroyStoreOnDestroy: true,
private: true,
maxWebConns: configurationService.torrentConfig.MAX_CONNECTIONS_PER_TORRENT,
}
@injectable()
export class TorrentDownloadService implements ITorrentDownloadService {
private engineOptions: TorrentStream.TorrentEngineOptions = {
connections: configurationService.torrentConfig.MAX_CONNECTIONS_PER_TORRENT,
uploads: 0,
verify: false,
dht: false,
tracker: true,
};
private torrentClient: WebTorrent.Instance;
private logger: ILoggingService;
constructor(@inject(IocTypes.ILoggingService) logger: ILoggingService) {
this.logger = logger;
this.torrentClient = new WebTorrent(clientOptions);
this.torrentClient.on('error', errors => this.logClientErrors(errors));
}
public getTorrentFiles = async (torrent: IParsedTorrent, timeout: number = 30000): Promise<ITorrentFileCollection> => {
const torrentFiles: ITorrentFile[] = await this.filesFromTorrentStream(torrent, timeout);
@@ -48,29 +63,32 @@ export class TorrentDownloadService implements ITorrentDownloadService {
}
const magnet = encode({infoHash: torrent.infoHash, announce: torrent.trackers.split(',')});
return new Promise((resolve, reject) => {
let engine: TorrentStream.TorrentEngine;
this.logger.debug(`Constructing torrent stream for ${torrent.title} with magnet ${magnet}`);
return new Promise((resolve, reject) => {
const timeoutId = setTimeout(() => {
engine.destroy(() => {
});
this.torrentClient.remove(magnet, {destroyStore: true});
reject(new Error('No available connections for torrent!'));
}, timeout);
engine = torrentStream(magnet, this.engineOptions);
this.logger.debug(`Adding torrent with infoHash ${torrent.infoHash}`);
engine.on("ready", () => {
const files: ITorrentFile[] = engine.files.map((file, fileId) => ({
...file,
this.torrentClient.add(magnet, torrentOptions, (torrent) => {
this.logger.debug(`torrent with infoHash ${torrent.infoHash} added to client.`);
const files: ITorrentFile[] = torrent.files.map((file, fileId) => ({
fileIndex: fileId,
size: file.length,
title: file.name
length: file.length,
name: file.name,
path: file.path,
}));
this.logger.debug(`Found ${files.length} files in torrent ${torrent.infoHash}`);
resolve(files);
engine.destroy(() => {
});
this.torrentClient.remove(magnet, {destroyStore: true});
clearTimeout(timeoutId);
});
});
@@ -86,12 +104,12 @@ export class TorrentDownloadService implements ITorrentDownloadService {
const minAnimeExtraRatio = 5;
const minRedundantRatio = videos.length <= 3 ? 30 : Number.MAX_VALUE;
const isSample = (video: ITorrentFile) => video.path?.match(/sample|bonus|promo/i) && maxSize / parseInt(video.path.toString()) > minSampleRatio;
const isRedundant = (video: ITorrentFile) => maxSize / parseInt(video.path.toString()) > minRedundantRatio;
const isExtra = (video: ITorrentFile) => video.path?.match(/extras?\//i);
const isAnimeExtra = (video: ITorrentFile) => video.path?.match(/(?:\b|_)(?:NC)?(?:ED|OP|PV)(?:v?\d\d?)?(?:\b|_)/i)
const isSample = (video: ITorrentFile) => video.path.toString()?.match(/sample|bonus|promo/i) && maxSize / video.length > minSampleRatio;
const isRedundant = (video: ITorrentFile) => maxSize / video.length > minRedundantRatio;
const isExtra = (video: ITorrentFile) => video.path.toString()?.match(/extras?\//i);
const isAnimeExtra = (video: ITorrentFile) => video.path.toString()?.match(/(?:\b|_)(?:NC)?(?:ED|OP|PV)(?:v?\d\d?)?(?:\b|_)/i)
&& maxSize / parseInt(video.length.toString()) > minAnimeExtraRatio;
const isWatermark = (video: ITorrentFile) => video.path?.match(/^[A-Z-]+(?:\.[A-Z]+)?\.\w{3,4}$/)
const isWatermark = (video: ITorrentFile) => video.path.toString()?.match(/^[A-Z-]+(?:\.[A-Z]+)?\.\w{3,4}$/)
&& maxSize / parseInt(video.length.toString()) > minAnimeExtraRatio
return videos
@@ -109,6 +127,7 @@ export class TorrentDownloadService implements ITorrentDownloadService {
private createContent = (torrent: IParsedTorrent, torrentFiles: ITorrentFile[]): IContentAttributes[] => torrentFiles.map(file => this.mapTorrentFileToContentAttributes(torrent, file));
private mapTorrentFileToFileAttributes = (torrent: IParsedTorrent, file: ITorrentFile): IFileAttributes => {
try {
const videoFile: IFileAttributes = {
title: file.name,
size: file.length,
@@ -117,11 +136,14 @@ export class TorrentDownloadService implements ITorrentDownloadService {
imdbId: torrent.imdbId.toString(),
imdbSeason: torrent.season || 0,
imdbEpisode: torrent.episode || 0,
kitsuId: parseInt(torrent.kitsuId.toString()) || 0,
kitsuId: parseInt(torrent.kitsuId?.toString()) || 0,
kitsuEpisode: torrent.episode || 0
};
return {...videoFile, ...parse(file.name)};
} catch (error) {
throw new Error(`Error parsing file ${file.name} from torrent ${torrent.infoHash}: ${error}`);
}
};
private mapTorrentFileToSubtitleAttributes = (torrent: IParsedTorrent, file: ITorrentFile): ISubtitleAttributes => ({
@@ -138,5 +160,9 @@ export class TorrentDownloadService implements ITorrentDownloadService {
path: file.path,
size: file.length,
});
private logClientErrors(errors: Error | string) {
this.logger.error(`Error in torrent client: ${errors}`);
}
}

13
src/node/consumer/src/lib/services/torrent_file_service.ts
View File

@@ -27,6 +27,10 @@ export class TorrentFileService implements ITorrentFileService {
private metadataService: IMetadataService;
private torrentDownloadService: ITorrentDownloadService;
private logger: ILoggingService;
private readonly imdb_limiter: Bottleneck = new Bottleneck({
maxConcurrent: configurationService.metadataConfig.IMDB_CONCURRENT,
minTime: configurationService.metadataConfig.IMDB_INTERVAL_MS
});
constructor(@inject(IocTypes.IMetadataService) metadataService: IMetadataService,
@inject(IocTypes.ITorrentDownloadService) torrentDownloadService: ITorrentDownloadService,
@@ -36,11 +40,6 @@ export class TorrentFileService implements ITorrentFileService {
this.logger = logger;
}
private readonly imdb_limiter: Bottleneck = new Bottleneck({
maxConcurrent: configurationService.metadataConfig.IMDB_CONCURRENT,
minTime: configurationService.metadataConfig.IMDB_INTERVAL_MS
});
public parseTorrentFiles = async (torrent: IParsedTorrent): Promise<ITorrentFileCollection> => {
const parsedTorrentName = parse(torrent.title);
const query: IMetaDataQuery = {
@@ -100,8 +99,8 @@ export class TorrentFileService implements ITorrentFileService {
fileIndex: video.fileIndex,
title: video.path || torrent.title,
size: video.size || torrent.size,
imdbId: torrent.imdbId.toString() || metadata && metadata.imdbId.toString(),
kitsuId: parseInt(torrent.kitsuId.toString() || metadata && metadata.kitsuId.toString())
imdbId: torrent.imdbId?.toString() || metadata && metadata.imdbId?.toString(),
kitsuId: parseInt(torrent.kitsuId?.toString() || metadata && metadata.kitsuId?.toString())
}));
return {...fileCollection, videos: parsedVideos};
}

1
src/node/consumer/src/lib/services/torrent_processing_service.ts
View File

@@ -13,6 +13,7 @@ export class TorrentProcessingService implements ITorrentProcessingService {
private torrentEntriesService: ITorrentEntriesService;
private logger: ILoggingService;
private trackerService: ITrackerService;
constructor(@inject(IocTypes.ITorrentEntriesService) torrentEntriesService: ITorrentEntriesService,
@inject(IocTypes.ILoggingService) logger: ILoggingService,
@inject(IocTypes.ITrackerService) trackerService: ITrackerService) {

8
src/node/consumer/src/repository/database_repository.ts
View File

@@ -41,7 +41,8 @@ export class DatabaseRepository implements IDatabaseRepository {
public connect = async () => {
try {
await this.database.sync({alter: configurationService.databaseConfig.AUTO_CREATE_AND_APPLY_MIGRATIONS});
} catch {
} catch (error) {
this.logger.debug('Failed to sync database', error);
this.logger.error('Failed syncing database');
process.exit(1);
}
@@ -112,9 +113,14 @@ export class DatabaseRepository implements IDatabaseRepository {
});
public createTorrent = async (torrent: Torrent): Promise<void> => {
try {
await Torrent.upsert(torrent);
await this.createContents(torrent.infoHash, torrent.contents);
await this.createSubtitles(torrent.infoHash, torrent.subtitles);
} catch (error) {
this.logger.error(`Failed to create torrent: ${torrent.infoHash}`);
this.logger.debug(error);
}
};
public setTorrentSeeders = async (torrent: ITorrentAttributes, seeders: number): Promise<[number]> => {

2
src/node/consumer/src/repository/models/content.ts
View File

@@ -1,4 +1,4 @@
import {Table, Column, Model, HasMany, DataType, BelongsTo, ForeignKey} from 'sequelize-typescript';
import {BelongsTo, Column, DataType, ForeignKey, Model, Table} from 'sequelize-typescript';
import {IContentAttributes, IContentCreationAttributes} from "../interfaces/content_attributes";
import {Torrent} from "./torrent";

3
src/node/consumer/src/repository/models/file.ts
View File

@@ -1,8 +1,7 @@
import {Table, Column, Model, HasMany, DataType, BelongsTo, ForeignKey} from 'sequelize-typescript';
import {BelongsTo, Column, DataType, ForeignKey, HasMany, Model, Table} from 'sequelize-typescript';
import {IFileAttributes, IFileCreationAttributes} from "../interfaces/file_attributes";
import {Torrent} from "./torrent";
import {Subtitle} from "./subtitle";
import {ISubtitleAttributes} from "../interfaces/subtitle_attributes";
const indexes = [
{

2
src/node/consumer/src/repository/models/ingestedPage.ts
View File

@@ -1,4 +1,4 @@
import { Table, Column, Model, HasMany, DataType } from 'sequelize-typescript';
import {Column, DataType, Model, Table} from 'sequelize-typescript';
import {IIngestedPageAttributes, IIngestedPageCreationAttributes} from "../interfaces/ingested_page_attributes";
const indexes = [

2
src/node/consumer/src/repository/models/ingestedTorrent.ts
View File

@@ -1,4 +1,4 @@
import { Table, Column, Model, HasMany, DataType } from 'sequelize-typescript';
import {Column, DataType, Model, Table} from 'sequelize-typescript';
import {IIngestedTorrentAttributes, IIngestedTorrentCreationAttributes} from "../interfaces/ingested_torrent_attributes";
const indexes = [

2
src/node/consumer/src/repository/models/provider.ts
View File

@@ -1,4 +1,4 @@
import { Table, Column, Model, HasMany, DataType } from 'sequelize-typescript';
import {Column, DataType, Model, Table} from 'sequelize-typescript';
import {IProviderAttributes, IProviderCreationAttributes} from "../interfaces/provider_attributes";
@Table({modelName: 'provider', timestamps: false})

2
src/node/consumer/src/repository/models/skipTorrent.ts
View File

@@ -1,4 +1,4 @@
import { Table, Column, Model, HasMany, DataType } from 'sequelize-typescript';
import {Column, DataType, Model, Table} from 'sequelize-typescript';
import {ISkipTorrentAttributes, ISkipTorrentCreationAttributes} from "../interfaces/skip_torrent_attributes";

3
src/node/consumer/src/repository/models/subtitle.ts
View File

@@ -1,7 +1,6 @@
import {Table, Column, Model, HasMany, DataType, BelongsTo, ForeignKey} from 'sequelize-typescript';
import {BelongsTo, Column, DataType, ForeignKey, Model, Table} from 'sequelize-typescript';
import {ISubtitleAttributes, ISubtitleCreationAttributes} from "../interfaces/subtitle_attributes";
import {File} from "./file";
import {Torrent} from "./torrent";
const indexes = [
{

2
src/node/consumer/src/repository/models/torrent.ts
View File

@@ -1,4 +1,4 @@
import { Table, Column, Model, HasMany, DataType } from 'sequelize-typescript';
import {Column, DataType, HasMany, Model, Table} from 'sequelize-typescript';
import {ITorrentAttributes, ITorrentCreationAttributes} from "../interfaces/torrent_attributes";
import {Content} from "./content";
import {File} from "./file";

9
src/node/consumer/tsconfig.json
View File

@@ -8,8 +8,13 @@
"rootDir": "./src",
"sourceMap": true,
"target": "ES6",
"lib": ["es6"],
"types": ["node", "reflect-metadata"],
"lib": [
"es6"
],
"types": [
"node",
"reflect-metadata"
],
"esModuleInterop": true,
"experimentalDecorators": true,
"emitDecoratorMetadata": true,