Plug in the new mongo imdb lookups into the consumer

src/node/consumer/package-lock.json

@@ -18,6 +18,7 @@
         "inversify": "^6.0.2",
         "magnet-uri": "^6.2.0",
         "moment": "^2.30.1",
+        "mongoose": "^8.2.0",
         "name-to-imdb": "^3.0.4",
         "parse-torrent-title": "https://github.com/TheBeastLT/parse-torrent-title.git#022408972c2a040f846331a912a6a8487746a654",
         "pg": "^8.11.3",
@@ -6401,6 +6402,14 @@
         "randombytes": "^2.0.3"
       }
     },
+    "node_modules/kareem": {
+      "version": "2.5.1",
+      "resolved": "https://registry.npmjs.org/kareem/-/kareem-2.5.1.tgz",
+      "integrity": "sha512-7jFxRVm+jD+rkq3kY0iZDJfsO2/t4BBPeEb2qKn2lR/9KhuksYk5hxzfRYWMPV8P/x2d0kHD306YyWLzjjH+uA==",
+      "engines": {
+        "node": ">=12.0.0"
+      }
+    },
     "node_modules/keyv": {
       "version": "4.5.4",
       "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz",
@@ -6742,6 +6751,51 @@
         "whatwg-url": "^13.0.0"
       }
     },
+    "node_modules/mongoose": {
+      "version": "8.2.0",
+      "resolved": "https://registry.npmjs.org/mongoose/-/mongoose-8.2.0.tgz",
+      "integrity": "sha512-la93n6zCYRbPS+c5N9oTDAktvREy5OT9OCljp1Tah0y3+p8UPMTAoabWaLZMdzYruOtF9/9GRf6MasaZjiZP1A==",
+      "dependencies": {
+        "bson": "^6.2.0",
+        "kareem": "2.5.1",
+        "mongodb": "6.3.0",
+        "mpath": "0.9.0",
+        "mquery": "5.0.0",
+        "ms": "2.1.3",
+        "sift": "16.0.1"
+      },
+      "engines": {
+        "node": ">=16.20.1"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/mongoose"
+      }
+    },
+    "node_modules/mongoose/node_modules/ms": {
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
+      "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
+    },
+    "node_modules/mpath": {
+      "version": "0.9.0",
+      "resolved": "https://registry.npmjs.org/mpath/-/mpath-0.9.0.tgz",
+      "integrity": "sha512-ikJRQTk8hw5DEoFVxHG1Gn9T/xcjtdnOKIU1JTmGjZZlg9LST2mBLmcX3/ICIbgJydT2GOc15RnNy5mHmzfSew==",
+      "engines": {
+        "node": ">=4.0.0"
+      }
+    },
+    "node_modules/mquery": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/mquery/-/mquery-5.0.0.tgz",
+      "integrity": "sha512-iQMncpmEK8R8ncT8HJGsGc9Dsp8xcgYMVSbs5jgnm1lFHTZqMJTUWTDx1LBO8+mK3tPNZWFLBghQEIOULSTHZg==",
+      "dependencies": {
+        "debug": "4.x"
+      },
+      "engines": {
+        "node": ">=14.0.0"
+      }
+    },
     "node_modules/ms": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
@@ -8555,6 +8609,11 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/sift": {
+      "version": "16.0.1",
+      "resolved": "https://registry.npmjs.org/sift/-/sift-16.0.1.tgz",
+      "integrity": "sha512-Wv6BjQ5zbhW7VFefWusVP33T/EM0vYikCaQ2qR8yULbsilAT8/wQaXvuQ3ptGLpoKx+lihJE3y2UTgKDyyNHZQ=="
+    },
     "node_modules/signal-exit": {
       "version": "3.0.7",
       "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz",

src/node/consumer/package.json

@@ -23,6 +23,7 @@
     "inversify": "^6.0.2",
     "magnet-uri": "^6.2.0",
     "moment": "^2.30.1",
+    "mongoose": "^8.2.0",
     "name-to-imdb": "^3.0.4",
     "parse-torrent-title": "https://github.com/TheBeastLT/parse-torrent-title.git#022408972c2a040f846331a912a6a8487746a654",
     "pg": "^8.11.3",

src/node/consumer/src/lib/mongo/interfaces/imdb_entry_attributes.ts

@@ -0,0 +1,13 @@
+import {Document} from "mongoose";
+
+export interface IImdbEntry extends Document {
+    _id: string;
+    EndYear: string;
+    Genres: string;
+    IsAdult: string;
+    OriginalTitle: string;
+    PrimaryTitle: string;
+    RuntimeMinutes: string;
+    StartYear: string;
+    TitleType: string;
+}

src/node/consumer/src/lib/mongo/interfaces/mongo_metadata_query.ts

@@ -0,0 +1,5 @@
+export interface IMongoMetadataQuery {
+    PrimaryTitle: { $regex: RegExp };
+    TitleType: {$in: string[]};
+    StartYear?: string;
+}

src/node/consumer/src/lib/mongo/interfaces/mongo_repository.ts

@@ -0,0 +1,4 @@
+export interface IMongoRepository {
+    connect(): Promise<void>;
+    getImdbId(title: string, category: string, year?: string | number): Promise<string | null>;
+}

src/node/consumer/src/lib/mongo/models/imdb_entries_model.ts

@@ -0,0 +1,16 @@
+import {IImdbEntry} from "@mongo/interfaces/imdb_entry_attributes";
+import mongoose, {Schema} from "mongoose";
+
+const ImdbEntriesSchema: Schema = new Schema({
+    _id: { type: String, required: true },
+    EndYear: { type: String, default: "" },
+    Genres: { type: String, default: "" },
+    IsAdult: { type: String, default: "0" },
+    OriginalTitle: { type: String, default: "" },
+    PrimaryTitle: { type: String, required: true },
+    RuntimeMinutes: { type: String, default: "" },
+    StartYear: { type: String, required: true },
+    TitleType: { type: String, default: "" },
+});
+
+export const ImdbEntryModel = mongoose.model<IImdbEntry>('ImdbEntry', ImdbEntriesSchema, 'imdb-entries');

src/node/consumer/src/lib/mongo/mongo_repository.ts

@@ -0,0 +1,41 @@
+import {TorrentType} from "@enums/torrent_types";
+import {IMongoMetadataQuery} from "@mongo/interfaces/mongo_metadata_query";
+import {IMongoRepository} from "@mongo/interfaces/mongo_repository";
+import {ImdbEntryModel} from "@mongo/models/imdb_entries_model";
+import {configurationService} from '@services/configuration_service';
+import {injectable} from "inversify";
+import mongoose from 'mongoose';
+
+@injectable()
+export class MongoRepository implements IMongoRepository {
+    private db: typeof mongoose = mongoose;
+
+    async connect() : Promise<void> {
+        await this.db.connect(configurationService.cacheConfig.MONGO_URI, {directConnection: true});
+    }
+
+    async getImdbId(title: string, category: string, year?: string | number) : Promise<string | null> {
+        const seriesTypes : string[] = ['tvSeries'];
+        const movieTypes : string[] = ['movie', 'tvMovie'];
+
+        let titleTypes: string[] = [];
+
+        if (category === TorrentType.Series) {
+            titleTypes = seriesTypes;
+        } else if (category === TorrentType.Movie) {
+            titleTypes = movieTypes;
+        }
+                
+        const query: IMongoMetadataQuery = {
+            PrimaryTitle: { $regex: new RegExp(title, 'i') },
+            TitleType: {$in: titleTypes}
+        };
+        
+        if (year) {
+            query.StartYear = year.toString();
+        }
+
+        const result = await ImdbEntryModel.findOne(query);
+        return result ? result._id : null;
+    }
+}

src/node/consumer/src/lib/services/metadata_service.ts

@@ -7,6 +7,7 @@ import {IKitsuJsonResponse} from "@interfaces/kitsu_metadata";
 import {IMetaDataQuery} from "@interfaces/metadata_query";
 import {IMetadataResponse} from "@interfaces/metadata_response";
 import {IMetadataService} from "@interfaces/metadata_service";
+import {IMongoRepository} from "@mongo/interfaces/mongo_repository";
 import {IocTypes} from "@setup/ioc_types";
 import axios from 'axios';
 import {ResultTypes, search} from 'google-sr';
@@ -20,6 +21,7 @@ const TIMEOUT = 60000;
 @injectable()
 export class MetadataService implements IMetadataService {
     @inject(IocTypes.ICacheService) private cacheService: ICacheService;
+    @inject(IocTypes.IMongoRepository) private mongoRepository: IMongoRepository;
 
     async getKitsuId(info: IMetaDataQuery): Promise<number | Error> {
         const title = this.escapeTitle(info.title!.replace(/\s\|\s.*/, ''));
@@ -47,6 +49,12 @@ export class MetadataService implements IMetadataService {
         const query = `${name} ${year || ''} ${info.type} imdb`;
         const fallbackQuery = `${name} ${info.type} imdb`;
         const googleQuery = year ? query : fallbackQuery;
+        
+        const imdbInMongo = await this.mongoRepository.getImdbId(name, info.type, year);
+        
+        if (imdbInMongo) {
+            return imdbInMongo;
+        }
 
         try {
             const imdbId = await this.cacheService.cacheWrapImdbId(key,

src/node/consumer/src/setup/composition_root.ts

@@ -1,5 +1,6 @@
 import {IProcessTorrentsJob} from "@interfaces/process_torrents_job";
 import {ITrackerService} from "@interfaces/tracker_service";
+import {IMongoRepository} from "@mongo/interfaces/mongo_repository";
 import {IDatabaseRepository} from "@repository/interfaces/database_repository";
 import {IocTypes} from "@setup/ioc_types";
 import {inject, injectable} from "inversify";
@@ -12,11 +13,13 @@ export interface ICompositionalRoot {
 export class CompositionalRoot implements ICompositionalRoot {
     @inject(IocTypes.ITrackerService) trackerService: ITrackerService;
     @inject(IocTypes.IDatabaseRepository) databaseRepository: IDatabaseRepository;
+    @inject(IocTypes.IMongoRepository) mongoRepository: IMongoRepository;
     @inject(IocTypes.IProcessTorrentsJob) processTorrentsJob: IProcessTorrentsJob;
 
     async start(): Promise<void> {
         await this.trackerService.getTrackers();
         await this.databaseRepository.connect();
+        await this.mongoRepository.connect();
         await this.processTorrentsJob.listenToQueue();
     }
 }

src/node/consumer/src/setup/inversify_config.ts

@@ -9,6 +9,8 @@ import {ITorrentProcessingService} from "@interfaces/torrent_processing_service"
 import {ITorrentSubtitleService} from "@interfaces/torrent_subtitle_service";
 import {ITrackerService} from "@interfaces/tracker_service";
 import {ProcessTorrentsJob} from "@jobs/process_torrents_job";
+import {IMongoRepository} from "@mongo/interfaces/mongo_repository";
+import {MongoRepository} from "@mongo/mongo_repository";
 import {DatabaseRepository} from "@repository/database_repository";
 import {IDatabaseRepository} from "@repository/interfaces/database_repository";
 import {CacheService} from "@services/cache_service";
@@ -37,6 +39,7 @@ serviceContainer.bind<ITorrentSubtitleService>(IocTypes.ITorrentSubtitleService)
 serviceContainer.bind<ITorrentEntriesService>(IocTypes.ITorrentEntriesService).to(TorrentEntriesService);
 serviceContainer.bind<IMetadataService>(IocTypes.IMetadataService).to(MetadataService);
 serviceContainer.bind<IDatabaseRepository>(IocTypes.IDatabaseRepository).to(DatabaseRepository);
+serviceContainer.bind<IMongoRepository>(IocTypes.IMongoRepository).to(MongoRepository);
 serviceContainer.bind<IProcessTorrentsJob>(IocTypes.IProcessTorrentsJob).to(ProcessTorrentsJob);
 
 export {serviceContainer};

src/node/consumer/src/setup/ioc_types.ts

@@ -13,6 +13,7 @@ export const IocTypes = {
     ITrackerService: Symbol.for("ITrackerService"),
     // DAL
     IDatabaseRepository: Symbol.for("IDatabaseRepository"),
+    IMongoRepository: Symbol.for("IMongoRepository"),
     // Jobs
     IProcessTorrentsJob: Symbol.for("IProcessTorrentsJob"),
 };

src/node/consumer/test/services/mongo_repository.test.ts

@@ -0,0 +1,73 @@
+import "reflect-metadata"; // required
+import {TorrentType} from "@enums/torrent_types";
+import {MongoRepository} from "@mongo/mongo_repository";
+import {Container} from "inversify";
+
+jest.mock('@services/configuration_service', () => {
+    return {
+        configurationService: {
+            cacheConfig: {
+                MONGODB_HOST: 'localhost',
+                MONGODB_PORT: '27017',
+                MONGODB_DB: 'knightcrawler',
+                MONGO_INITDB_ROOT_USERNAME: 'mongo',
+                MONGO_INITDB_ROOT_PASSWORD: 'mongo',
+                get MONGO_URI(): string {
+                    return `mongodb://${this.MONGO_INITDB_ROOT_USERNAME}:${this.MONGO_INITDB_ROOT_PASSWORD}@${this.MONGODB_HOST}:${this.MONGODB_PORT}/${this.MONGODB_DB}?authSource=admin`;
+                }
+            },
+        }
+    }
+});
+
+describe('MongoRepository Tests', () => {
+    let mongoRepository: MongoRepository;
+        
+    beforeEach(() => {
+        jest.clearAllMocks();
+        process.env.LOG_LEVEL = 'debug';
+        const container = new Container();
+        container.bind<MongoRepository>(MongoRepository).toSelf();
+        mongoRepository = container.get(MongoRepository);
+    });
+
+    afterEach(() => {
+        jest.clearAllMocks();
+    });
+
+    it('should get The Flash 2014 imdbId correctly', async () => {
+        await mongoRepository.connect();
+        const result = await mongoRepository.getImdbId('The Flash', TorrentType.Series, 2014);
+        expect(result).toBe('tt3107288');
+    });
+
+    it('should get The Flash 1990 imdbId correctly', async () => {
+        await mongoRepository.connect();
+        const result = await mongoRepository.getImdbId('The Flash', TorrentType.Series, 1990);
+        expect(result).toBe('tt0098798');
+    });
+
+    it('should get Wrath of Khan imdbId correctly', async () => {
+        await mongoRepository.connect();
+        const result = await mongoRepository.getImdbId('Star Trek II: The Wrath of Khan', TorrentType.Movie, 1982);
+        expect(result).toBe('tt0084726');
+    }, 30000);
+
+    it('should get Wrath of Khan imdbId correctly', async () => {
+        await mongoRepository.connect();
+        const result = await mongoRepository.getImdbId('Wrath of Khan', TorrentType.Movie, 1982);
+        expect(result).toBe('tt0084726');
+    }, 30000);
+
+    it('should get Return of the Jedi correctly', async () => {
+        await mongoRepository.connect();
+        const result = await mongoRepository.getImdbId('Star Wars: Episode VI - Return of the Jedi', TorrentType.Movie, 1983);
+        expect(result).toBe('tt0086190');
+    }, 30000);
+
+    it('should get Return of the Jedi correctly', async () => {
+        await mongoRepository.connect();
+        const result = await mongoRepository.getImdbId('Return of the Jedi', TorrentType.Movie, 1983);
+        expect(result).toBe('tt0086190');
+    }, 30000);
+});

src/node/consumer/tsconfig.json

@@ -34,6 +34,9 @@
       "@repository/*": [
         "lib/repository/*"
       ],
+      "@mongo/*": [
+        "lib/mongo/*"
+      ],
       "@interfaces/*": [
         "lib/interfaces/*"
       ],