From 9c8cbf6468d42a0140c49f2676b3d547db1229da Mon Sep 17 00:00:00 2001 From: iPromKnight Date: Wed, 28 Feb 2024 04:03:50 +0000 Subject: [PATCH] Plug in the new mongo imdb lookups into the consumer --- src/node/consumer/package-lock.json | 59 +++++++++++++++ src/node/consumer/package.json | 1 + .../mongo/interfaces/imdb_entry_attributes.ts | 13 ++++ .../mongo/interfaces/mongo_metadata_query.ts | 5 ++ .../lib/mongo/interfaces/mongo_repository.ts | 4 + .../lib/mongo/models/imdb_entries_model.ts | 16 ++++ .../src/lib/mongo/mongo_repository.ts | 41 +++++++++++ .../src/lib/services/metadata_service.ts | 8 ++ .../consumer/src/setup/composition_root.ts | 3 + .../consumer/src/setup/inversify_config.ts | 3 + src/node/consumer/src/setup/ioc_types.ts | 1 + .../test/services/mongo_repository.test.ts | 73 +++++++++++++++++++ src/node/consumer/tsconfig.json | 3 + 13 files changed, 230 insertions(+) create mode 100644 src/node/consumer/src/lib/mongo/interfaces/imdb_entry_attributes.ts create mode 100644 src/node/consumer/src/lib/mongo/interfaces/mongo_metadata_query.ts create mode 100644 src/node/consumer/src/lib/mongo/interfaces/mongo_repository.ts create mode 100644 src/node/consumer/src/lib/mongo/models/imdb_entries_model.ts create mode 100644 src/node/consumer/src/lib/mongo/mongo_repository.ts create mode 100644 src/node/consumer/test/services/mongo_repository.test.ts diff --git a/src/node/consumer/package-lock.json b/src/node/consumer/package-lock.json index 41b4c50..c9e5770 100644 --- a/src/node/consumer/package-lock.json +++ b/src/node/consumer/package-lock.json @@ -18,6 +18,7 @@ "inversify": "^6.0.2", "magnet-uri": "^6.2.0", "moment": "^2.30.1", + "mongoose": "^8.2.0", "name-to-imdb": "^3.0.4", "parse-torrent-title": "https://github.com/TheBeastLT/parse-torrent-title.git#022408972c2a040f846331a912a6a8487746a654", "pg": "^8.11.3", @@ -6401,6 +6402,14 @@ "randombytes": "^2.0.3" } }, + "node_modules/kareem": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/kareem/-/kareem-2.5.1.tgz", + "integrity": "sha512-7jFxRVm+jD+rkq3kY0iZDJfsO2/t4BBPeEb2qKn2lR/9KhuksYk5hxzfRYWMPV8P/x2d0kHD306YyWLzjjH+uA==", + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/keyv": { "version": "4.5.4", "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", @@ -6742,6 +6751,51 @@ "whatwg-url": "^13.0.0" } }, + "node_modules/mongoose": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/mongoose/-/mongoose-8.2.0.tgz", + "integrity": "sha512-la93n6zCYRbPS+c5N9oTDAktvREy5OT9OCljp1Tah0y3+p8UPMTAoabWaLZMdzYruOtF9/9GRf6MasaZjiZP1A==", + "dependencies": { + "bson": "^6.2.0", + "kareem": "2.5.1", + "mongodb": "6.3.0", + "mpath": "0.9.0", + "mquery": "5.0.0", + "ms": "2.1.3", + "sift": "16.0.1" + }, + "engines": { + "node": ">=16.20.1" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mongoose" + } + }, + "node_modules/mongoose/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + }, + "node_modules/mpath": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/mpath/-/mpath-0.9.0.tgz", + "integrity": "sha512-ikJRQTk8hw5DEoFVxHG1Gn9T/xcjtdnOKIU1JTmGjZZlg9LST2mBLmcX3/ICIbgJydT2GOc15RnNy5mHmzfSew==", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/mquery": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/mquery/-/mquery-5.0.0.tgz", + "integrity": "sha512-iQMncpmEK8R8ncT8HJGsGc9Dsp8xcgYMVSbs5jgnm1lFHTZqMJTUWTDx1LBO8+mK3tPNZWFLBghQEIOULSTHZg==", + "dependencies": { + "debug": "4.x" + }, + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -8555,6 +8609,11 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/sift": { + "version": "16.0.1", + "resolved": "https://registry.npmjs.org/sift/-/sift-16.0.1.tgz", + "integrity": "sha512-Wv6BjQ5zbhW7VFefWusVP33T/EM0vYikCaQ2qR8yULbsilAT8/wQaXvuQ3ptGLpoKx+lihJE3y2UTgKDyyNHZQ==" + }, "node_modules/signal-exit": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", diff --git a/src/node/consumer/package.json b/src/node/consumer/package.json index d144b5e..33e4c55 100644 --- a/src/node/consumer/package.json +++ b/src/node/consumer/package.json @@ -23,6 +23,7 @@ "inversify": "^6.0.2", "magnet-uri": "^6.2.0", "moment": "^2.30.1", + "mongoose": "^8.2.0", "name-to-imdb": "^3.0.4", "parse-torrent-title": "https://github.com/TheBeastLT/parse-torrent-title.git#022408972c2a040f846331a912a6a8487746a654", "pg": "^8.11.3", diff --git a/src/node/consumer/src/lib/mongo/interfaces/imdb_entry_attributes.ts b/src/node/consumer/src/lib/mongo/interfaces/imdb_entry_attributes.ts new file mode 100644 index 0000000..7f1b15c --- /dev/null +++ b/src/node/consumer/src/lib/mongo/interfaces/imdb_entry_attributes.ts @@ -0,0 +1,13 @@ +import {Document} from "mongoose"; + +export interface IImdbEntry extends Document { + _id: string; + EndYear: string; + Genres: string; + IsAdult: string; + OriginalTitle: string; + PrimaryTitle: string; + RuntimeMinutes: string; + StartYear: string; + TitleType: string; +} \ No newline at end of file diff --git a/src/node/consumer/src/lib/mongo/interfaces/mongo_metadata_query.ts b/src/node/consumer/src/lib/mongo/interfaces/mongo_metadata_query.ts new file mode 100644 index 0000000..af5f9df --- /dev/null +++ b/src/node/consumer/src/lib/mongo/interfaces/mongo_metadata_query.ts @@ -0,0 +1,5 @@ +export interface IMongoMetadataQuery { + PrimaryTitle: { $regex: RegExp }; + TitleType: {$in: string[]}; + StartYear?: string; +} \ No newline at end of file diff --git a/src/node/consumer/src/lib/mongo/interfaces/mongo_repository.ts b/src/node/consumer/src/lib/mongo/interfaces/mongo_repository.ts new file mode 100644 index 0000000..fca1ff7 --- /dev/null +++ b/src/node/consumer/src/lib/mongo/interfaces/mongo_repository.ts @@ -0,0 +1,4 @@ +export interface IMongoRepository { + connect(): Promise; + getImdbId(title: string, category: string, year?: string | number): Promise; +} \ No newline at end of file diff --git a/src/node/consumer/src/lib/mongo/models/imdb_entries_model.ts b/src/node/consumer/src/lib/mongo/models/imdb_entries_model.ts new file mode 100644 index 0000000..9d1877a --- /dev/null +++ b/src/node/consumer/src/lib/mongo/models/imdb_entries_model.ts @@ -0,0 +1,16 @@ +import {IImdbEntry} from "@mongo/interfaces/imdb_entry_attributes"; +import mongoose, {Schema} from "mongoose"; + +const ImdbEntriesSchema: Schema = new Schema({ + _id: { type: String, required: true }, + EndYear: { type: String, default: "" }, + Genres: { type: String, default: "" }, + IsAdult: { type: String, default: "0" }, + OriginalTitle: { type: String, default: "" }, + PrimaryTitle: { type: String, required: true }, + RuntimeMinutes: { type: String, default: "" }, + StartYear: { type: String, required: true }, + TitleType: { type: String, default: "" }, +}); + +export const ImdbEntryModel = mongoose.model('ImdbEntry', ImdbEntriesSchema, 'imdb-entries'); \ No newline at end of file diff --git a/src/node/consumer/src/lib/mongo/mongo_repository.ts b/src/node/consumer/src/lib/mongo/mongo_repository.ts new file mode 100644 index 0000000..d697d46 --- /dev/null +++ b/src/node/consumer/src/lib/mongo/mongo_repository.ts @@ -0,0 +1,41 @@ +import {TorrentType} from "@enums/torrent_types"; +import {IMongoMetadataQuery} from "@mongo/interfaces/mongo_metadata_query"; +import {IMongoRepository} from "@mongo/interfaces/mongo_repository"; +import {ImdbEntryModel} from "@mongo/models/imdb_entries_model"; +import {configurationService} from '@services/configuration_service'; +import {injectable} from "inversify"; +import mongoose from 'mongoose'; + +@injectable() +export class MongoRepository implements IMongoRepository { + private db: typeof mongoose = mongoose; + + async connect() : Promise { + await this.db.connect(configurationService.cacheConfig.MONGO_URI, {directConnection: true}); + } + + async getImdbId(title: string, category: string, year?: string | number) : Promise { + const seriesTypes : string[] = ['tvSeries']; + const movieTypes : string[] = ['movie', 'tvMovie']; + + let titleTypes: string[] = []; + + if (category === TorrentType.Series) { + titleTypes = seriesTypes; + } else if (category === TorrentType.Movie) { + titleTypes = movieTypes; + } + + const query: IMongoMetadataQuery = { + PrimaryTitle: { $regex: new RegExp(title, 'i') }, + TitleType: {$in: titleTypes} + }; + + if (year) { + query.StartYear = year.toString(); + } + + const result = await ImdbEntryModel.findOne(query); + return result ? result._id : null; + } +} \ No newline at end of file diff --git a/src/node/consumer/src/lib/services/metadata_service.ts b/src/node/consumer/src/lib/services/metadata_service.ts index 3dfeb1d..f489a15 100644 --- a/src/node/consumer/src/lib/services/metadata_service.ts +++ b/src/node/consumer/src/lib/services/metadata_service.ts @@ -7,6 +7,7 @@ import {IKitsuJsonResponse} from "@interfaces/kitsu_metadata"; import {IMetaDataQuery} from "@interfaces/metadata_query"; import {IMetadataResponse} from "@interfaces/metadata_response"; import {IMetadataService} from "@interfaces/metadata_service"; +import {IMongoRepository} from "@mongo/interfaces/mongo_repository"; import {IocTypes} from "@setup/ioc_types"; import axios from 'axios'; import {ResultTypes, search} from 'google-sr'; @@ -20,6 +21,7 @@ const TIMEOUT = 60000; @injectable() export class MetadataService implements IMetadataService { @inject(IocTypes.ICacheService) private cacheService: ICacheService; + @inject(IocTypes.IMongoRepository) private mongoRepository: IMongoRepository; async getKitsuId(info: IMetaDataQuery): Promise { const title = this.escapeTitle(info.title!.replace(/\s\|\s.*/, '')); @@ -47,6 +49,12 @@ export class MetadataService implements IMetadataService { const query = `${name} ${year || ''} ${info.type} imdb`; const fallbackQuery = `${name} ${info.type} imdb`; const googleQuery = year ? query : fallbackQuery; + + const imdbInMongo = await this.mongoRepository.getImdbId(name, info.type, year); + + if (imdbInMongo) { + return imdbInMongo; + } try { const imdbId = await this.cacheService.cacheWrapImdbId(key, diff --git a/src/node/consumer/src/setup/composition_root.ts b/src/node/consumer/src/setup/composition_root.ts index ce28aa0..2980202 100644 --- a/src/node/consumer/src/setup/composition_root.ts +++ b/src/node/consumer/src/setup/composition_root.ts @@ -1,5 +1,6 @@ import {IProcessTorrentsJob} from "@interfaces/process_torrents_job"; import {ITrackerService} from "@interfaces/tracker_service"; +import {IMongoRepository} from "@mongo/interfaces/mongo_repository"; import {IDatabaseRepository} from "@repository/interfaces/database_repository"; import {IocTypes} from "@setup/ioc_types"; import {inject, injectable} from "inversify"; @@ -12,11 +13,13 @@ export interface ICompositionalRoot { export class CompositionalRoot implements ICompositionalRoot { @inject(IocTypes.ITrackerService) trackerService: ITrackerService; @inject(IocTypes.IDatabaseRepository) databaseRepository: IDatabaseRepository; + @inject(IocTypes.IMongoRepository) mongoRepository: IMongoRepository; @inject(IocTypes.IProcessTorrentsJob) processTorrentsJob: IProcessTorrentsJob; async start(): Promise { await this.trackerService.getTrackers(); await this.databaseRepository.connect(); + await this.mongoRepository.connect(); await this.processTorrentsJob.listenToQueue(); } } \ No newline at end of file diff --git a/src/node/consumer/src/setup/inversify_config.ts b/src/node/consumer/src/setup/inversify_config.ts index 796bacd..d82a4fc 100644 --- a/src/node/consumer/src/setup/inversify_config.ts +++ b/src/node/consumer/src/setup/inversify_config.ts @@ -9,6 +9,8 @@ import {ITorrentProcessingService} from "@interfaces/torrent_processing_service" import {ITorrentSubtitleService} from "@interfaces/torrent_subtitle_service"; import {ITrackerService} from "@interfaces/tracker_service"; import {ProcessTorrentsJob} from "@jobs/process_torrents_job"; +import {IMongoRepository} from "@mongo/interfaces/mongo_repository"; +import {MongoRepository} from "@mongo/mongo_repository"; import {DatabaseRepository} from "@repository/database_repository"; import {IDatabaseRepository} from "@repository/interfaces/database_repository"; import {CacheService} from "@services/cache_service"; @@ -37,6 +39,7 @@ serviceContainer.bind(IocTypes.ITorrentSubtitleService) serviceContainer.bind(IocTypes.ITorrentEntriesService).to(TorrentEntriesService); serviceContainer.bind(IocTypes.IMetadataService).to(MetadataService); serviceContainer.bind(IocTypes.IDatabaseRepository).to(DatabaseRepository); +serviceContainer.bind(IocTypes.IMongoRepository).to(MongoRepository); serviceContainer.bind(IocTypes.IProcessTorrentsJob).to(ProcessTorrentsJob); export {serviceContainer}; diff --git a/src/node/consumer/src/setup/ioc_types.ts b/src/node/consumer/src/setup/ioc_types.ts index ebe437b..a7d3819 100644 --- a/src/node/consumer/src/setup/ioc_types.ts +++ b/src/node/consumer/src/setup/ioc_types.ts @@ -13,6 +13,7 @@ export const IocTypes = { ITrackerService: Symbol.for("ITrackerService"), // DAL IDatabaseRepository: Symbol.for("IDatabaseRepository"), + IMongoRepository: Symbol.for("IMongoRepository"), // Jobs IProcessTorrentsJob: Symbol.for("IProcessTorrentsJob"), }; \ No newline at end of file diff --git a/src/node/consumer/test/services/mongo_repository.test.ts b/src/node/consumer/test/services/mongo_repository.test.ts new file mode 100644 index 0000000..f1168f2 --- /dev/null +++ b/src/node/consumer/test/services/mongo_repository.test.ts @@ -0,0 +1,73 @@ +import "reflect-metadata"; // required +import {TorrentType} from "@enums/torrent_types"; +import {MongoRepository} from "@mongo/mongo_repository"; +import {Container} from "inversify"; + +jest.mock('@services/configuration_service', () => { + return { + configurationService: { + cacheConfig: { + MONGODB_HOST: 'localhost', + MONGODB_PORT: '27017', + MONGODB_DB: 'knightcrawler', + MONGO_INITDB_ROOT_USERNAME: 'mongo', + MONGO_INITDB_ROOT_PASSWORD: 'mongo', + get MONGO_URI(): string { + return `mongodb://${this.MONGO_INITDB_ROOT_USERNAME}:${this.MONGO_INITDB_ROOT_PASSWORD}@${this.MONGODB_HOST}:${this.MONGODB_PORT}/${this.MONGODB_DB}?authSource=admin`; + } + }, + } + } +}); + +describe('MongoRepository Tests', () => { + let mongoRepository: MongoRepository; + + beforeEach(() => { + jest.clearAllMocks(); + process.env.LOG_LEVEL = 'debug'; + const container = new Container(); + container.bind(MongoRepository).toSelf(); + mongoRepository = container.get(MongoRepository); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + it('should get The Flash 2014 imdbId correctly', async () => { + await mongoRepository.connect(); + const result = await mongoRepository.getImdbId('The Flash', TorrentType.Series, 2014); + expect(result).toBe('tt3107288'); + }); + + it('should get The Flash 1990 imdbId correctly', async () => { + await mongoRepository.connect(); + const result = await mongoRepository.getImdbId('The Flash', TorrentType.Series, 1990); + expect(result).toBe('tt0098798'); + }); + + it('should get Wrath of Khan imdbId correctly', async () => { + await mongoRepository.connect(); + const result = await mongoRepository.getImdbId('Star Trek II: The Wrath of Khan', TorrentType.Movie, 1982); + expect(result).toBe('tt0084726'); + }, 30000); + + it('should get Wrath of Khan imdbId correctly', async () => { + await mongoRepository.connect(); + const result = await mongoRepository.getImdbId('Wrath of Khan', TorrentType.Movie, 1982); + expect(result).toBe('tt0084726'); + }, 30000); + + it('should get Return of the Jedi correctly', async () => { + await mongoRepository.connect(); + const result = await mongoRepository.getImdbId('Star Wars: Episode VI - Return of the Jedi', TorrentType.Movie, 1983); + expect(result).toBe('tt0086190'); + }, 30000); + + it('should get Return of the Jedi correctly', async () => { + await mongoRepository.connect(); + const result = await mongoRepository.getImdbId('Return of the Jedi', TorrentType.Movie, 1983); + expect(result).toBe('tt0086190'); + }, 30000); +}); \ No newline at end of file diff --git a/src/node/consumer/tsconfig.json b/src/node/consumer/tsconfig.json index f1b203c..e16895a 100644 --- a/src/node/consumer/tsconfig.json +++ b/src/node/consumer/tsconfig.json @@ -34,6 +34,9 @@ "@repository/*": [ "lib/repository/*" ], + "@mongo/*": [ + "lib/mongo/*" + ], "@interfaces/*": [ "lib/interfaces/*" ],