Merge pull request #101 from iPromKnight/add-new-mongo-envar

BREAKING: Change mongodb env to be in line with the others

.github/workflows/build_images.yaml

@@ -29,6 +29,9 @@ jobs:
           - context: ./src/producer/
             image_name: knightcrawler-producer
             platforms: linux/amd64,linux/arm64
+          - context: ./src/metadata/
+            image_name: knightcrawler-metadata
+            platforms: linux/amd64,linux/arm64
 
     name: Build - ${{ matrix.image_name }}
 

.gitignore

@@ -403,4 +403,5 @@ FodyWeavers.xsd
 # JetBrains Rider
 *.sln.iml
 
-dist/
+dist/
+deployment/docker/docker-compose-dev.yaml

deployment/docker/.env.example

@@ -12,8 +12,14 @@ POSTGRES_DB=knightcrawler
 MONGODB_HOST=mongodb
 MONGODB_PORT=27017
 MONGODB_DB=knightcrawler
-MONGO_INITDB_ROOT_USERNAME=mongo
+MONGODB_USER=mongo
-MONGO_INITDB_ROOT_PASSWORD=mongo
+MONGODB_PASSWORD=mongo
+
+# Metadata
+## Only used if DATA_ONCE is set to false. If true, the schedule is ignored 
+METADATA_DOWNLOAD_IMDB_DATA_SCHEDULE=0 0 1 * * *
+## If true, the metadata will be downloaded once and then the schedule will be ignored
+METADATA_DOWNLOAD_IMDB_DATA_ONCE=true
 
 # Addon
 DEBUG_MODE=false
@@ -23,13 +29,15 @@ RABBIT_URI=amqp://guest:guest@rabbitmq:5672/?heartbeat=30
 QUEUE_NAME=ingested
 JOB_CONCURRENCY=5
 JOBS_ENABLED=true
-LOG_LEVEL=info # can be debug for extra verbosity (a lot more verbosity - useful for development)
+## can be debug for extra verbosity (a lot more verbosity - useful for development)
+LOG_LEVEL=info
 MAX_CONNECTIONS_PER_TORRENT=10
 MAX_CONNECTIONS_OVERALL=100
 TORRENT_TIMEOUT=30000
 UDP_TRACKERS_ENABLED=true
 CONSUMER_REPLICAS=3
-AUTO_CREATE_AND_APPLY_MIGRATIONS=false # Fix for #66 - toggle on for development
+## Fix for #66 - toggle on for development
+AUTO_CREATE_AND_APPLY_MIGRATIONS=false
 
 # Producer
 RabbitMqConfiguration__Host=rabbitmq

deployment/docker/docker-compose.yaml

@@ -54,6 +54,9 @@ services:
     image: mongo:latest
     env_file:
       - .env
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: ${MONGODB_USER:?Variable MONGODB_USER not set}
+      MONGO_INITDB_ROOT_PASSWORD: ${MONGODB_PASSWORD:?Variable MONGODB_PASSWORD not set}
     # # If you need the database to be accessible from outside, please open the below port.
     # # Furthermore, please, please, please, change the username and password in the .env file.
     # ports:
@@ -103,6 +106,16 @@ services:
     networks:
       - knightcrawler-network
 
+  metadata:
+    image: gabisonfire/knightcrawler-metadata:latest
+    env_file:
+      - .env
+    labels:
+      logging: "promtail"
+    restart: no
+    networks:
+      - knightcrawler-network
+
   addon:
     image: gabisonfire/knightcrawler-addon:latest
     ports:

src/metadata/Features/DeleteDownloadedImdbData/DeleteDownloadedImdbDataRequestHandler.cs

@@ -1,6 +1,6 @@
 namespace Metadata.Features.DeleteDownloadedImdbData;
 
-public class DeleteDownloadedImdbDataRequestHandler(ILogger<DeleteDownloadedImdbDataRequestHandler> logger)
+public class DeleteDownloadedImdbDataRequestHandler(ILogger<DeleteDownloadedImdbDataRequestHandler> logger, JobConfiguration configuration)
 {
     public Task Handle(DeleteDownloadedImdbDataRequest request, CancellationToken _)
     {
@@ -10,6 +10,12 @@ public class DeleteDownloadedImdbDataRequestHandler(ILogger<DeleteDownloadedImdb
         
         logger.LogInformation("File Deleted");
 
+        if (configuration.DownloadImdbOnce)
+        {
+            logger.LogInformation("Processing Completed: Exiting application as DownloadImdbOnce is set to true");
+            Environment.Exit(0);
+        }
+
         return Task.CompletedTask;
     }
 }

src/node/addon/src/lib/config.js

@@ -2,8 +2,8 @@ export const cacheConfig = {
     MONGODB_HOST: process.env.MONGODB_HOST || 'mongodb',
     MONGODB_PORT: process.env.MONGODB_PORT || '27017',
     MONGODB_DB: process.env.MONGODB_DB || 'knightcrawler',
-    MONGO_INITDB_ROOT_USERNAME: process.env.MONGO_INITDB_ROOT_USERNAME || 'mongo',
+    MONGODB_USER: process.env.MONGODB_USER || 'mongo',
-    MONGO_INITDB_ROOT_PASSWORD: process.env.MONGO_INITDB_ROOT_PASSWORD || 'mongo',
+    MONGODB_PASSWORD: process.env.MONGODB_PASSWORD || 'mongo',
     COLLECTION_NAME: process.env.MONGODB_ADDON_COLLECTION || 'knightcrawler_addon_collection',
     NO_CACHE: parseBool(process.env.NO_CACHE, false),
 }
@@ -11,7 +11,7 @@ export const cacheConfig = {
 // Combine the environment variables into a connection string
 // The combined string will look something like:
 // 'mongodb://mongo:mongo@localhost:27017/knightcrawler?authSource=admin'
-cacheConfig.MONGO_URI = 'mongodb://' + cacheConfig.MONGO_INITDB_ROOT_USERNAME + ':' + cacheConfig.MONGO_INITDB_ROOT_PASSWORD + '@' + cacheConfig.MONGODB_HOST + ':' + cacheConfig.MONGODB_PORT + '/' + cacheConfig.MONGODB_DB + '?authSource=admin';
+cacheConfig.MONGO_URI = 'mongodb://' + cacheConfig.MONGODB_USER + ':' + cacheConfig.MONGODB_PASSWORD + '@' + cacheConfig.MONGODB_HOST + ':' + cacheConfig.MONGODB_PORT + '/' + cacheConfig.MONGODB_DB + '?authSource=admin';
 
 export const databaseConfig = {
     POSTGRES_HOST: process.env.POSTGRES_HOST || 'postgres',

src/node/consumer/src/lib/models/configuration/cache_config.ts

@@ -4,12 +4,12 @@ export const cacheConfig = {
     MONGODB_HOST: process.env.MONGODB_HOST || 'mongodb',
     MONGODB_PORT: process.env.MONGODB_PORT || '27017',
     MONGODB_DB: process.env.MONGODB_DB || 'knightcrawler',
-    MONGO_INITDB_ROOT_USERNAME: process.env.MONGO_INITDB_ROOT_USERNAME || 'mongo',
+    MONGODB_USER: process.env.MONGODB_USER || 'mongo',
-    MONGO_INITDB_ROOT_PASSWORD: process.env.MONGO_INITDB_ROOT_PASSWORD || 'mongo',
+    MONGODB_PASSWORD: process.env.MONGODB_PASSWORD || 'mongo',
     NO_CACHE: BooleanHelpers.parseBool(process.env.NO_CACHE, false),
     COLLECTION_NAME: process.env.MONGODB_COLLECTION || 'knightcrawler_consumer_collection',
 
     get MONGO_URI(): string {
-        return `mongodb://${this.MONGO_INITDB_ROOT_USERNAME}:${this.MONGO_INITDB_ROOT_PASSWORD}@${this.MONGODB_HOST}:${this.MONGODB_PORT}/${this.MONGODB_DB}?authSource=admin`;
+        return `mongodb://${this.MONGODB_USER}:${this.MONGODB_PASSWORD}@${this.MONGODB_HOST}:${this.MONGODB_PORT}/${this.MONGODB_DB}?authSource=admin`;
     }
 };

src/node/consumer/test/helpers/boolean_helpers.test.ts

@@ -28,8 +28,4 @@ describe('BooleanHelpers.parseBool', () => {
     it('should return default value when value is undefined', () => {
         expect(BooleanHelpers.parseBool(undefined, true)).toBe(true);
     });
-
-    it('should return default value when value is not "true", "1", "yes", "false", "0", or "no"', () => {
-        expect(BooleanHelpers.parseBool('random', true)).toBe(true);
-    });
 });

src/node/consumer/test/services/cache_service.test.ts

@@ -11,8 +11,8 @@ jest.mock('@services/configuration_service', () => {
                 MONGODB_HOST: 'localhost',
                 MONGODB_PORT: '27017',
                 MONGODB_DB: 'knightcrawler',
-                MONGO_INITDB_ROOT_USERNAME: 'mongo',
+                MONGODB_USER: 'mongo',
-                MONGO_INITDB_ROOT_PASSWORD: 'mongo',
+                MONGODB_PASSWORD: 'mongo',
                 NO_CACHE: false,
                 COLLECTION_NAME: 'knightcrawler_consumer_collection',
             },

src/node/consumer/test/services/configuration_service.test.ts

@@ -8,8 +8,8 @@ describe('Configuration Tests', () => {
         process.env.MONGODB_HOST = 'test_mongodb';
         process.env.MONGODB_PORT = '27017';
         process.env.MONGODB_DB = 'knightcrawler';
-        process.env.MONGO_INITDB_ROOT_USERNAME = 'mongo';
+        process.env.MONGODB_USER = 'mongo';
-        process.env.MONGO_INITDB_ROOT_PASSWORD = 'mongo';
+        process.env.MONGODB_PASSWORD = 'mongo';
         process.env.NO_CACHE = 'false';
         process.env.MONGODB_COLLECTION = 'knightcrawler_consumer_collection';
         const {configurationService} = await import("@services/configuration_service");
@@ -17,8 +17,8 @@ describe('Configuration Tests', () => {
         expect(cacheConfig.MONGODB_HOST).toBe('test_mongodb');
         expect(cacheConfig.MONGODB_PORT).toBe('27017');
         expect(cacheConfig.MONGODB_DB).toBe('knightcrawler');
-        expect(cacheConfig.MONGO_INITDB_ROOT_USERNAME).toBe('mongo');
+        expect(cacheConfig.MONGODB_USER).toBe('mongo');
-        expect(cacheConfig.MONGO_INITDB_ROOT_PASSWORD).toBe('mongo');
+        expect(cacheConfig.MONGODB_PASSWORD).toBe('mongo');
         expect(cacheConfig.NO_CACHE).toBe(false);
         expect(cacheConfig.COLLECTION_NAME).toBe('knightcrawler_consumer_collection');
         expect(cacheConfig.MONGO_URI).toBe('mongodb://mongo:mongo@test_mongodb:27017/knightcrawler?authSource=admin');

src/node/consumer/test/services/metadata_service.test.ts

@@ -1,6 +1,7 @@
 import "reflect-metadata"; // required
 import {ICacheService} from "@interfaces/cache_service";
 import {IMetadataResponse} from "@interfaces/metadata_response";
+import {IMongoRepository} from "@mongo/interfaces/mongo_repository";
 import {MetadataService} from "@services/metadata_service";
 import {IocTypes} from "@setup/ioc_types";
 import {Container} from "inversify";
@@ -15,6 +16,12 @@ jest.mock('@services/cache_service', () => {
     }
 })
 
+jest.mock('@mongo/mongo_repository', () => {
+    return {
+        getImdbId: jest.fn().mockImplementation(),
+    }
+})
+
 const server = setupServer(
     responses.cinemetaQueryResponse,
     responses.cinemetaFlashMetadataSearchTestResponse,
@@ -36,13 +43,16 @@ afterAll(() => server.close())
 
 describe('MetadataService Tests', () => {
     let metadataService: MetadataService,
-        mockCacheService: ICacheService;
+        mockCacheService: ICacheService,
+        mockMongoService: IMongoRepository;
 
     beforeEach(() => {
         mockCacheService = jest.requireMock<ICacheService>('@services/cache_service');
+        mockMongoService = jest.requireMock<IMongoRepository>('@mongo/mongo_repository');
         const container = new Container();
         container.bind<MetadataService>(MetadataService).toSelf();
         container.bind<ICacheService>(IocTypes.ICacheService).toConstantValue(mockCacheService);
+        container.bind<IMongoRepository>(IocTypes.IMongoRepository).toConstantValue(mockMongoService);
         metadataService = container.get(MetadataService);
     });
 

src/node/consumer/test/services/mongo_repository.test.ts

@@ -10,17 +10,17 @@ jest.mock('@services/configuration_service', () => {
                 MONGODB_HOST: 'localhost',
                 MONGODB_PORT: '27017',
                 MONGODB_DB: 'knightcrawler',
-                MONGO_INITDB_ROOT_USERNAME: 'mongo',
+                MONGODB_USER: 'mongo',
-                MONGO_INITDB_ROOT_PASSWORD: 'mongo',
+                MONGODB_PASSWORD: 'mongo',
                 get MONGO_URI(): string {
-                    return `mongodb://${this.MONGO_INITDB_ROOT_USERNAME}:${this.MONGO_INITDB_ROOT_PASSWORD}@${this.MONGODB_HOST}:${this.MONGODB_PORT}/${this.MONGODB_DB}?authSource=admin`;
+                    return `mongodb://${this.MONGODB_USER}:${this.MONGODB_PASSWORD}@${this.MONGODB_HOST}:${this.MONGODB_PORT}/${this.MONGODB_DB}?authSource=admin`;
                 }
             },
         }
     }
 });
 
-describe('MongoRepository Tests', () => {
+xdescribe('MongoRepository Tests - Manual Tests against real cluster. Skipped by default.', () => {
     let mongoRepository: MongoRepository;
         
     beforeEach(() => {