replace torrent-stream (high vunerabilities) with webtorrent, gives us clean package audit

src/node/consumer/.eslintrc

@@ -64,7 +64,12 @@
   },
   "overrides": [
     {
-      "files": ["*.ts", "*.mts", "*.cts", "*.tsx"],
+      "files": [
+        "*.ts",
+        "*.mts",
+        "*.cts",
+        "*.tsx"
+      ],
       "rules": {
         "@typescript-eslint/explicit-function-return-type": "error",
         "@typescript-eslint/consistent-type-assertions": [

src/node/consumer/package.json

@@ -27,15 +27,12 @@
     "reflect-metadata": "^0.2.1",
     "sequelize": "^6.36.0",
     "sequelize-typescript": "^2.1.6",
-    "torrent-stream": "^1.2.1",
+    "webtorrent": "^2.1.35"
-    "user-agents": "^1.0.1444"
   },
   "devDependencies": {
     "@types/amqplib": "^0.10.4",
     "@types/magnet-uri": "^5.1.5",
     "@types/node": "^20.11.16",
-    "@types/stremio-addon-sdk": "^1.6.10",
-    "@types/torrent-stream": "^0.0.9",
     "@types/validator": "^13.11.8",
     "@typescript-eslint/eslint-plugin": "^6.21.0",
     "@typescript-eslint/parser": "^6.21.0",

src/node/consumer/src/lib/interfaces/cinemeta_metadata.ts

@@ -6,6 +6,7 @@ export interface ICinemetaJsonResponse {
     links?: ICinemetaLink[];
     behaviorHints?: ICinemetaBehaviorHints;
 }
+
 export interface ICinemetaMetaData {
     awards?: string;
     cast?: string[];
@@ -37,6 +38,7 @@ export interface ICinemetaMetaData {
     releaseInfo?: string;
     videos?: ICinemetaVideo[];
 }
+
 export interface ICinemetaPopularities {
     PXS_TEST?: number;
     PXS?: number;
@@ -49,10 +51,12 @@ export interface ICinemetaPopularities {
     stremio?: number;
     stremio_lib?: number;
 }
+
 export interface ICinemetaTrailer {
     source?: string;
     type?: string;
 }
+
 export interface ICinemetaVideo extends ICommonVideoMetadata {
     name?: string;
     number?: number;
@@ -63,15 +67,18 @@ export interface ICinemetaVideo extends ICommonVideoMetadata {
     thumbnail?: string;
     description?: string;
 }
+
 export interface ICinemetaTrailerStream {
     title?: string;
     ytId?: string;
 }
+
 export interface ICinemetaLink {
     name?: string;
     category?: string;
     url?: string;
 }
+
 export interface ICinemetaBehaviorHints {
     defaultVideoId?: null;
     hasScheduledVideos?: boolean;

src/node/consumer/src/lib/interfaces/kitsu_metadata.ts

@@ -4,6 +4,7 @@ export interface IKitsuJsonResponse {
     cacheMaxAge?: number;
     meta?: IKitsuMeta;
 }
+
 export interface IKitsuMeta {
     aliases?: string[];
     animeType?: string;
@@ -29,16 +30,19 @@ export interface IKitsuMeta {
     videos?: IKitsuVideo[];
     year?: string;
 }
+
 export interface IKitsuVideo extends ICommonVideoMetadata {
     imdbEpisode?: number;
     imdbSeason?: number;
     imdb_id?: string;
     thumbnail?: string;
 }
+
 export interface IKitsuTrailer {
     source?: string;
     type?: string;
 }
+
 export interface IKitsuLink {
     name?: string;
     category?: string;

src/node/consumer/src/lib/interfaces/logging_service.ts

@@ -1,6 +1,9 @@
 export interface ILoggingService {
     info(message: string, ...args: any[]): void;
+
     error(message: string, ...args: any[]): void;
+
     debug(message: string, ...args: any[]): void;
+
     warn(message: string, ...args: any[]): void;
 }

src/node/consumer/src/lib/models/composition_root.ts

@@ -10,6 +10,7 @@ export class CompositionalRoot implements ICompositionalRoot {
     private trackerService: ITrackerService;
     private databaseRepository: IDatabaseRepository;
     private processTorrentsJob: IProcessTorrentsJob;
+
     constructor(@inject(IocTypes.ITrackerService) trackerService: ITrackerService,
                 @inject(IocTypes.IDatabaseRepository) databaseRepository: IDatabaseRepository,
                 @inject(IocTypes.IProcessTorrentsJob) processTorrentsJob: IProcessTorrentsJob) {

src/node/consumer/src/lib/models/configuration/torrent_config.ts

@@ -1,4 +1,5 @@
 export const torrentConfig = {
     MAX_CONNECTIONS_PER_TORRENT: parseInt(process.env.MAX_SINGLE_TORRENT_CONNECTIONS || "20", 10),
+    MAX_CONNECTIONS_OVERALL: parseInt(process.env.MAX_CONNECTIONS_OVERALL || "100", 10),
     TIMEOUT: parseInt(process.env.TORRENT_TIMEOUT || "30000", 10)
 };

src/node/consumer/src/lib/models/inversify_config.ts

@@ -32,11 +32,11 @@ serviceContainer.bind<ICompositionalRoot>(IocTypes.ICompositionalRoot).to(Compos
 serviceContainer.bind<ICacheService>(IocTypes.ICacheService).to(CacheService).inSingletonScope();
 serviceContainer.bind<ILoggingService>(IocTypes.ILoggingService).to(LoggingService).inSingletonScope();
 serviceContainer.bind<ITrackerService>(IocTypes.ITrackerService).to(TrackerService).inSingletonScope();
+serviceContainer.bind<ITorrentDownloadService>(IocTypes.ITorrentDownloadService).to(TorrentDownloadService).inSingletonScope();
 serviceContainer.bind<ITorrentFileService>(IocTypes.ITorrentFileService).to(TorrentFileService);
 serviceContainer.bind<ITorrentProcessingService>(IocTypes.ITorrentProcessingService).to(TorrentProcessingService);
 serviceContainer.bind<ITorrentSubtitleService>(IocTypes.ITorrentSubtitleService).to(TorrentSubtitleService);
 serviceContainer.bind<ITorrentEntriesService>(IocTypes.ITorrentEntriesService).to(TorrentEntriesService);
-serviceContainer.bind<ITorrentDownloadService>(IocTypes.ITorrentDownloadService).to(TorrentDownloadService);
 serviceContainer.bind<IMetadataService>(IocTypes.IMetadataService).to(MetadataService);
 serviceContainer.bind<IDatabaseRepository>(IocTypes.IDatabaseRepository).to(DatabaseRepository);
 serviceContainer.bind<IProcessTorrentsJob>(IocTypes.IProcessTorrentsJob).to(ProcessTorrentsJob);

src/node/consumer/src/lib/services/metadata_service.ts

@@ -20,6 +20,7 @@ const TIMEOUT = 20000;
 @injectable()
 export class MetadataService implements IMetadataService {
     private cacheService: ICacheService;
+
     constructor(@inject(IocTypes.ICacheService) cacheService: ICacheService) {
         this.cacheService = cacheService;
     }

src/node/consumer/src/lib/services/torrent_download_service.ts

@@ -1,5 +1,4 @@
 import {encode} from 'magnet-uri';
-import torrentStream from 'torrent-stream';
 import {configurationService} from './configuration_service';
 import {ExtensionHelpers} from '../helpers/extension_helpers';
 import {ITorrentFileCollection} from "../interfaces/torrent_file_collection";
@@ -9,7 +8,10 @@ import {ISubtitleAttributes} from "../../repository/interfaces/subtitle_attribut
 import {IContentAttributes} from "../../repository/interfaces/content_attributes";
 import {parse} from "parse-torrent-title";
 import {ITorrentDownloadService} from "../interfaces/torrent_download_service";
-import {injectable} from "inversify";
+import {inject, injectable} from "inversify";
+import {ILoggingService} from "../interfaces/logging_service";
+import {IocTypes} from "../models/ioc_types";
+import WebTorrent from "webtorrent";
 
 interface ITorrentFile {
     name: string;
@@ -18,15 +20,28 @@ interface ITorrentFile {
     fileIndex: number;
 }
 
+const clientOptions = {
+    maxConns: configurationService.torrentConfig.MAX_CONNECTIONS_OVERALL,
+}
+
+const torrentOptions = {
+    skipVerify: true,
+    addUID: true,
+    destroyStoreOnDestroy: true,
+    private: true,
+    maxWebConns: configurationService.torrentConfig.MAX_CONNECTIONS_PER_TORRENT,
+}
+
 @injectable()
 export class TorrentDownloadService implements ITorrentDownloadService {
-    private engineOptions: TorrentStream.TorrentEngineOptions = {
+    private torrentClient: WebTorrent.Instance;
-        connections: configurationService.torrentConfig.MAX_CONNECTIONS_PER_TORRENT,
+    private logger: ILoggingService;
-        uploads: 0,
+
-        verify: false,
+    constructor(@inject(IocTypes.ILoggingService) logger: ILoggingService) {
-        dht: false,
+        this.logger = logger;
-        tracker: true,
+        this.torrentClient = new WebTorrent(clientOptions);
-    };
+        this.torrentClient.on('error', errors => this.logClientErrors(errors));
+    }
 
     public getTorrentFiles = async (torrent: IParsedTorrent, timeout: number = 30000): Promise<ITorrentFileCollection> => {
         const torrentFiles: ITorrentFile[] = await this.filesFromTorrentStream(torrent, timeout);
@@ -48,29 +63,32 @@ export class TorrentDownloadService implements ITorrentDownloadService {
         }
         const magnet = encode({infoHash: torrent.infoHash, announce: torrent.trackers.split(',')});
 
-        return new Promise((resolve, reject) => {
+        this.logger.debug(`Constructing torrent stream for ${torrent.title} with magnet ${magnet}`);
-            let engine: TorrentStream.TorrentEngine;
 
+        return new Promise((resolve, reject) => {
             const timeoutId = setTimeout(() => {
-                engine.destroy(() => {
+                this.torrentClient.remove(magnet, {destroyStore: true});
-                });
                 reject(new Error('No available connections for torrent!'));
             }, timeout);
 
-            engine = torrentStream(magnet, this.engineOptions);
+            this.logger.debug(`Adding torrent with infoHash ${torrent.infoHash}`);
             
-            engine.on("ready", () => {
+            this.torrentClient.add(magnet, torrentOptions, (torrent) => {
-                const files: ITorrentFile[] = engine.files.map((file, fileId) => ({
+
-                    ...file,
+                this.logger.debug(`torrent with infoHash ${torrent.infoHash} added to client.`);
+                
+                const files: ITorrentFile[] = torrent.files.map((file, fileId) => ({
                     fileIndex: fileId,
-                    size: file.length,
+                    length: file.length,
-                    title: file.name
+                    name: file.name,
+                    path: file.path,
                 }));
 
+                this.logger.debug(`Found ${files.length} files in torrent ${torrent.infoHash}`);
+
                 resolve(files);
 
-                engine.destroy(() => {
+                this.torrentClient.remove(magnet, {destroyStore: true});
-                });
                 clearTimeout(timeoutId);
             });
         });
@@ -86,12 +104,12 @@ export class TorrentDownloadService implements ITorrentDownloadService {
         const minAnimeExtraRatio = 5;
         const minRedundantRatio = videos.length <= 3 ? 30 : Number.MAX_VALUE;
 
-        const isSample = (video: ITorrentFile) => video.path?.match(/sample|bonus|promo/i) && maxSize / parseInt(video.path.toString()) > minSampleRatio;
+        const isSample = (video: ITorrentFile) => video.path.toString()?.match(/sample|bonus|promo/i) && maxSize / video.length > minSampleRatio;
-        const isRedundant = (video: ITorrentFile) => maxSize / parseInt(video.path.toString()) > minRedundantRatio;
+        const isRedundant = (video: ITorrentFile) => maxSize / video.length > minRedundantRatio;
-        const isExtra = (video: ITorrentFile) => video.path?.match(/extras?\//i);
+        const isExtra = (video: ITorrentFile) => video.path.toString()?.match(/extras?\//i);
-        const isAnimeExtra = (video: ITorrentFile) => video.path?.match(/(?:\b|_)(?:NC)?(?:ED|OP|PV)(?:v?\d\d?)?(?:\b|_)/i)
+        const isAnimeExtra = (video: ITorrentFile) => video.path.toString()?.match(/(?:\b|_)(?:NC)?(?:ED|OP|PV)(?:v?\d\d?)?(?:\b|_)/i)
             && maxSize / parseInt(video.length.toString()) > minAnimeExtraRatio;
-        const isWatermark = (video: ITorrentFile) => video.path?.match(/^[A-Z-]+(?:\.[A-Z]+)?\.\w{3,4}$/)
+        const isWatermark = (video: ITorrentFile) => video.path.toString()?.match(/^[A-Z-]+(?:\.[A-Z]+)?\.\w{3,4}$/)
             && maxSize / parseInt(video.length.toString()) > minAnimeExtraRatio
 
         return videos
@@ -109,6 +127,7 @@ export class TorrentDownloadService implements ITorrentDownloadService {
     private createContent = (torrent: IParsedTorrent, torrentFiles: ITorrentFile[]): IContentAttributes[] => torrentFiles.map(file => this.mapTorrentFileToContentAttributes(torrent, file));
 
     private mapTorrentFileToFileAttributes = (torrent: IParsedTorrent, file: ITorrentFile): IFileAttributes => {
+        try {
             const videoFile: IFileAttributes = {
                 title: file.name,
                 size: file.length,
@@ -117,11 +136,14 @@ export class TorrentDownloadService implements ITorrentDownloadService {
                 imdbId: torrent.imdbId.toString(),
                 imdbSeason: torrent.season || 0,
                 imdbEpisode: torrent.episode || 0,
-            kitsuId: parseInt(torrent.kitsuId.toString()) || 0,
+                kitsuId: parseInt(torrent.kitsuId?.toString()) || 0,
                 kitsuEpisode: torrent.episode || 0
             };
 
             return {...videoFile, ...parse(file.name)};
+        } catch (error) {
+            throw new Error(`Error parsing file ${file.name} from torrent ${torrent.infoHash}: ${error}`);
+        }
     };
 
     private mapTorrentFileToSubtitleAttributes = (torrent: IParsedTorrent, file: ITorrentFile): ISubtitleAttributes => ({
@@ -138,5 +160,9 @@ export class TorrentDownloadService implements ITorrentDownloadService {
         path: file.path,
         size: file.length,
     });
+
+    private logClientErrors(errors: Error | string) {
+        this.logger.error(`Error in torrent client: ${errors}`);
+    }
 }
 

src/node/consumer/src/lib/services/torrent_file_service.ts

@@ -27,6 +27,10 @@ export class TorrentFileService implements ITorrentFileService {
     private metadataService: IMetadataService;
     private torrentDownloadService: ITorrentDownloadService;
     private logger: ILoggingService;
+    private readonly imdb_limiter: Bottleneck = new Bottleneck({
+        maxConcurrent: configurationService.metadataConfig.IMDB_CONCURRENT,
+        minTime: configurationService.metadataConfig.IMDB_INTERVAL_MS
+    });
 
     constructor(@inject(IocTypes.IMetadataService) metadataService: IMetadataService,
                 @inject(IocTypes.ITorrentDownloadService) torrentDownloadService: ITorrentDownloadService,
@@ -36,11 +40,6 @@ export class TorrentFileService implements ITorrentFileService {
         this.logger = logger;
     }
 
-    private readonly imdb_limiter: Bottleneck = new Bottleneck({
-        maxConcurrent: configurationService.metadataConfig.IMDB_CONCURRENT,
-        minTime: configurationService.metadataConfig.IMDB_INTERVAL_MS
-    });
-
     public parseTorrentFiles = async (torrent: IParsedTorrent): Promise<ITorrentFileCollection> => {
         const parsedTorrentName = parse(torrent.title);
         const query: IMetaDataQuery = {
@@ -100,8 +99,8 @@ export class TorrentFileService implements ITorrentFileService {
                 fileIndex: video.fileIndex,
                 title: video.path || torrent.title,
                 size: video.size || torrent.size,
-                imdbId: torrent.imdbId.toString() || metadata && metadata.imdbId.toString(),
+                imdbId: torrent.imdbId?.toString() || metadata && metadata.imdbId?.toString(),
-                kitsuId: parseInt(torrent.kitsuId.toString() || metadata && metadata.kitsuId.toString())
+                kitsuId: parseInt(torrent.kitsuId?.toString() || metadata && metadata.kitsuId?.toString())
             }));
             return {...fileCollection, videos: parsedVideos};
         }

src/node/consumer/src/lib/services/torrent_processing_service.ts

@@ -13,6 +13,7 @@ export class TorrentProcessingService implements ITorrentProcessingService {
     private torrentEntriesService: ITorrentEntriesService;
     private logger: ILoggingService;
     private trackerService: ITrackerService;
+
     constructor(@inject(IocTypes.ITorrentEntriesService) torrentEntriesService: ITorrentEntriesService,
                 @inject(IocTypes.ILoggingService) logger: ILoggingService,
                 @inject(IocTypes.ITrackerService) trackerService: ITrackerService) {

src/node/consumer/src/repository/database_repository.ts

@@ -41,7 +41,8 @@ export class DatabaseRepository implements IDatabaseRepository {
     public connect = async () => {
         try {
             await this.database.sync({alter: configurationService.databaseConfig.AUTO_CREATE_AND_APPLY_MIGRATIONS});
-        } catch {
+        } catch (error) {
+            this.logger.debug('Failed to sync database', error);
             this.logger.error('Failed syncing database');
             process.exit(1);
         }
@@ -112,9 +113,14 @@ export class DatabaseRepository implements IDatabaseRepository {
     });
 
     public createTorrent = async (torrent: Torrent): Promise<void> => {
+        try {
             await Torrent.upsert(torrent);
             await this.createContents(torrent.infoHash, torrent.contents);
             await this.createSubtitles(torrent.infoHash, torrent.subtitles);
+        } catch (error) {
+            this.logger.error(`Failed to create torrent: ${torrent.infoHash}`);
+            this.logger.debug(error);
+        }
     };
 
     public setTorrentSeeders = async (torrent: ITorrentAttributes, seeders: number): Promise<[number]> => {

src/node/consumer/src/repository/models/content.ts

@@ -1,4 +1,4 @@
-import {Table, Column, Model, HasMany, DataType, BelongsTo, ForeignKey} from 'sequelize-typescript';
+import {BelongsTo, Column, DataType, ForeignKey, Model, Table} from 'sequelize-typescript';
 import {IContentAttributes, IContentCreationAttributes} from "../interfaces/content_attributes";
 import {Torrent} from "./torrent";
 

src/node/consumer/src/repository/models/file.ts

@@ -1,8 +1,7 @@
-import {Table, Column, Model, HasMany, DataType, BelongsTo, ForeignKey} from 'sequelize-typescript';
+import {BelongsTo, Column, DataType, ForeignKey, HasMany, Model, Table} from 'sequelize-typescript';
 import {IFileAttributes, IFileCreationAttributes} from "../interfaces/file_attributes";
 import {Torrent} from "./torrent";
 import {Subtitle} from "./subtitle";
-import {ISubtitleAttributes} from "../interfaces/subtitle_attributes";
 
 const indexes = [
     {

src/node/consumer/src/repository/models/ingestedPage.ts

@@ -1,4 +1,4 @@
-import { Table, Column, Model, HasMany, DataType } from 'sequelize-typescript';
+import {Column, DataType, Model, Table} from 'sequelize-typescript';
 import {IIngestedPageAttributes, IIngestedPageCreationAttributes} from "../interfaces/ingested_page_attributes";
 
 const indexes = [

src/node/consumer/src/repository/models/ingestedTorrent.ts

@@ -1,4 +1,4 @@
-import { Table, Column, Model, HasMany, DataType } from 'sequelize-typescript';
+import {Column, DataType, Model, Table} from 'sequelize-typescript';
 import {IIngestedTorrentAttributes, IIngestedTorrentCreationAttributes} from "../interfaces/ingested_torrent_attributes";
 
 const indexes = [

src/node/consumer/src/repository/models/provider.ts

@@ -1,4 +1,4 @@
-import { Table, Column, Model, HasMany, DataType } from 'sequelize-typescript';
+import {Column, DataType, Model, Table} from 'sequelize-typescript';
 import {IProviderAttributes, IProviderCreationAttributes} from "../interfaces/provider_attributes";
 
 @Table({modelName: 'provider', timestamps: false})

src/node/consumer/src/repository/models/skipTorrent.ts

@@ -1,4 +1,4 @@
-import { Table, Column, Model, HasMany, DataType } from 'sequelize-typescript';
+import {Column, DataType, Model, Table} from 'sequelize-typescript';
 import {ISkipTorrentAttributes, ISkipTorrentCreationAttributes} from "../interfaces/skip_torrent_attributes";
 
 

src/node/consumer/src/repository/models/subtitle.ts

@@ -1,7 +1,6 @@
-import {Table, Column, Model, HasMany, DataType, BelongsTo, ForeignKey} from 'sequelize-typescript';
+import {BelongsTo, Column, DataType, ForeignKey, Model, Table} from 'sequelize-typescript';
 import {ISubtitleAttributes, ISubtitleCreationAttributes} from "../interfaces/subtitle_attributes";
 import {File} from "./file";
-import {Torrent} from "./torrent";
 
 const indexes = [
     {

src/node/consumer/src/repository/models/torrent.ts

@@ -1,4 +1,4 @@
-import { Table, Column, Model, HasMany, DataType } from 'sequelize-typescript';
+import {Column, DataType, HasMany, Model, Table} from 'sequelize-typescript';
 import {ITorrentAttributes, ITorrentCreationAttributes} from "../interfaces/torrent_attributes";
 import {Content} from "./content";
 import {File} from "./file";

src/node/consumer/tsconfig.json

@@ -8,8 +8,13 @@
     "rootDir": "./src",
     "sourceMap": true,
     "target": "ES6",
-    "lib": ["es6"],
+    "lib": [
-    "types": ["node", "reflect-metadata"],
+      "es6"
+    ],
+    "types": [
+      "node",
+      "reflect-metadata"
+    ],
     "esModuleInterop": true,
     "experimentalDecorators": true,
     "emitDecoratorMetadata": true,