diff --git a/.github/workflows/build_images.yaml b/.github/workflows/build_images.yaml index fd92012..10f9b6d 100644 --- a/.github/workflows/build_images.yaml +++ b/.github/workflows/build_images.yaml @@ -29,6 +29,9 @@ jobs: - context: ./src/producer/ image_name: knightcrawler-producer platforms: linux/amd64,linux/arm64 + - context: ./src/metadata/ + image_name: knightcrawler-metadata + platforms: linux/amd64,linux/arm64 name: Build - ${{ matrix.image_name }} diff --git a/.gitignore b/.gitignore index 6a4f0ab..c521616 100644 --- a/.gitignore +++ b/.gitignore @@ -403,4 +403,5 @@ FodyWeavers.xsd # JetBrains Rider *.sln.iml -dist/ \ No newline at end of file +dist/ +deployment/docker/docker-compose-dev.yaml diff --git a/deployment/docker/.env.example b/deployment/docker/.env.example index 1d84fe9..905c8dd 100644 --- a/deployment/docker/.env.example +++ b/deployment/docker/.env.example @@ -12,8 +12,14 @@ POSTGRES_DB=knightcrawler MONGODB_HOST=mongodb MONGODB_PORT=27017 MONGODB_DB=knightcrawler -MONGO_INITDB_ROOT_USERNAME=mongo -MONGO_INITDB_ROOT_PASSWORD=mongo +MONGODB_USER=mongo +MONGODB_PASSWORD=mongo + +# Metadata +## Only used if DATA_ONCE is set to false. If true, the schedule is ignored +METADATA_DOWNLOAD_IMDB_DATA_SCHEDULE=0 0 1 * * * +## If true, the metadata will be downloaded once and then the schedule will be ignored +METADATA_DOWNLOAD_IMDB_DATA_ONCE=true # Addon DEBUG_MODE=false @@ -23,13 +29,15 @@ RABBIT_URI=amqp://guest:guest@rabbitmq:5672/?heartbeat=30 QUEUE_NAME=ingested JOB_CONCURRENCY=5 JOBS_ENABLED=true -LOG_LEVEL=info # can be debug for extra verbosity (a lot more verbosity - useful for development) +## can be debug for extra verbosity (a lot more verbosity - useful for development) +LOG_LEVEL=info MAX_CONNECTIONS_PER_TORRENT=10 MAX_CONNECTIONS_OVERALL=100 TORRENT_TIMEOUT=30000 UDP_TRACKERS_ENABLED=true CONSUMER_REPLICAS=3 -AUTO_CREATE_AND_APPLY_MIGRATIONS=false # Fix for #66 - toggle on for development +## Fix for #66 - toggle on for development +AUTO_CREATE_AND_APPLY_MIGRATIONS=false # Producer RabbitMqConfiguration__Host=rabbitmq diff --git a/deployment/docker/docker-compose.yaml b/deployment/docker/docker-compose.yaml index a24a1e7..8df9ddb 100644 --- a/deployment/docker/docker-compose.yaml +++ b/deployment/docker/docker-compose.yaml @@ -54,6 +54,9 @@ services: image: mongo:latest env_file: - .env + environment: + MONGO_INITDB_ROOT_USERNAME: ${MONGODB_USER:?Variable MONGODB_USER not set} + MONGO_INITDB_ROOT_PASSWORD: ${MONGODB_PASSWORD:?Variable MONGODB_PASSWORD not set} # # If you need the database to be accessible from outside, please open the below port. # # Furthermore, please, please, please, change the username and password in the .env file. # ports: @@ -103,6 +106,16 @@ services: networks: - knightcrawler-network + metadata: + image: gabisonfire/knightcrawler-metadata:latest + env_file: + - .env + labels: + logging: "promtail" + restart: no + networks: + - knightcrawler-network + addon: image: gabisonfire/knightcrawler-addon:latest ports: diff --git a/src/metadata/Features/DeleteDownloadedImdbData/DeleteDownloadedImdbDataRequestHandler.cs b/src/metadata/Features/DeleteDownloadedImdbData/DeleteDownloadedImdbDataRequestHandler.cs index 4e22ef5..bc6a4fe 100644 --- a/src/metadata/Features/DeleteDownloadedImdbData/DeleteDownloadedImdbDataRequestHandler.cs +++ b/src/metadata/Features/DeleteDownloadedImdbData/DeleteDownloadedImdbDataRequestHandler.cs @@ -1,6 +1,6 @@ namespace Metadata.Features.DeleteDownloadedImdbData; -public class DeleteDownloadedImdbDataRequestHandler(ILogger logger) +public class DeleteDownloadedImdbDataRequestHandler(ILogger logger, JobConfiguration configuration) { public Task Handle(DeleteDownloadedImdbDataRequest request, CancellationToken _) { @@ -10,6 +10,12 @@ public class DeleteDownloadedImdbDataRequestHandler(ILogger { it('should return default value when value is undefined', () => { expect(BooleanHelpers.parseBool(undefined, true)).toBe(true); }); - - it('should return default value when value is not "true", "1", "yes", "false", "0", or "no"', () => { - expect(BooleanHelpers.parseBool('random', true)).toBe(true); - }); }); \ No newline at end of file diff --git a/src/node/consumer/test/services/cache_service.test.ts b/src/node/consumer/test/services/cache_service.test.ts index 6a5e8b6..0a8df99 100644 --- a/src/node/consumer/test/services/cache_service.test.ts +++ b/src/node/consumer/test/services/cache_service.test.ts @@ -11,8 +11,8 @@ jest.mock('@services/configuration_service', () => { MONGODB_HOST: 'localhost', MONGODB_PORT: '27017', MONGODB_DB: 'knightcrawler', - MONGO_INITDB_ROOT_USERNAME: 'mongo', - MONGO_INITDB_ROOT_PASSWORD: 'mongo', + MONGODB_USER: 'mongo', + MONGODB_PASSWORD: 'mongo', NO_CACHE: false, COLLECTION_NAME: 'knightcrawler_consumer_collection', }, diff --git a/src/node/consumer/test/services/configuration_service.test.ts b/src/node/consumer/test/services/configuration_service.test.ts index a82cea5..21a979c 100644 --- a/src/node/consumer/test/services/configuration_service.test.ts +++ b/src/node/consumer/test/services/configuration_service.test.ts @@ -8,8 +8,8 @@ describe('Configuration Tests', () => { process.env.MONGODB_HOST = 'test_mongodb'; process.env.MONGODB_PORT = '27017'; process.env.MONGODB_DB = 'knightcrawler'; - process.env.MONGO_INITDB_ROOT_USERNAME = 'mongo'; - process.env.MONGO_INITDB_ROOT_PASSWORD = 'mongo'; + process.env.MONGODB_USER = 'mongo'; + process.env.MONGODB_PASSWORD = 'mongo'; process.env.NO_CACHE = 'false'; process.env.MONGODB_COLLECTION = 'knightcrawler_consumer_collection'; const {configurationService} = await import("@services/configuration_service"); @@ -17,8 +17,8 @@ describe('Configuration Tests', () => { expect(cacheConfig.MONGODB_HOST).toBe('test_mongodb'); expect(cacheConfig.MONGODB_PORT).toBe('27017'); expect(cacheConfig.MONGODB_DB).toBe('knightcrawler'); - expect(cacheConfig.MONGO_INITDB_ROOT_USERNAME).toBe('mongo'); - expect(cacheConfig.MONGO_INITDB_ROOT_PASSWORD).toBe('mongo'); + expect(cacheConfig.MONGODB_USER).toBe('mongo'); + expect(cacheConfig.MONGODB_PASSWORD).toBe('mongo'); expect(cacheConfig.NO_CACHE).toBe(false); expect(cacheConfig.COLLECTION_NAME).toBe('knightcrawler_consumer_collection'); expect(cacheConfig.MONGO_URI).toBe('mongodb://mongo:mongo@test_mongodb:27017/knightcrawler?authSource=admin'); diff --git a/src/node/consumer/test/services/metadata_service.test.ts b/src/node/consumer/test/services/metadata_service.test.ts index 0b94220..bae718a 100644 --- a/src/node/consumer/test/services/metadata_service.test.ts +++ b/src/node/consumer/test/services/metadata_service.test.ts @@ -1,6 +1,7 @@ import "reflect-metadata"; // required import {ICacheService} from "@interfaces/cache_service"; import {IMetadataResponse} from "@interfaces/metadata_response"; +import {IMongoRepository} from "@mongo/interfaces/mongo_repository"; import {MetadataService} from "@services/metadata_service"; import {IocTypes} from "@setup/ioc_types"; import {Container} from "inversify"; @@ -15,6 +16,12 @@ jest.mock('@services/cache_service', () => { } }) +jest.mock('@mongo/mongo_repository', () => { + return { + getImdbId: jest.fn().mockImplementation(), + } +}) + const server = setupServer( responses.cinemetaQueryResponse, responses.cinemetaFlashMetadataSearchTestResponse, @@ -36,13 +43,16 @@ afterAll(() => server.close()) describe('MetadataService Tests', () => { let metadataService: MetadataService, - mockCacheService: ICacheService; + mockCacheService: ICacheService, + mockMongoService: IMongoRepository; beforeEach(() => { mockCacheService = jest.requireMock('@services/cache_service'); + mockMongoService = jest.requireMock('@mongo/mongo_repository'); const container = new Container(); container.bind(MetadataService).toSelf(); container.bind(IocTypes.ICacheService).toConstantValue(mockCacheService); + container.bind(IocTypes.IMongoRepository).toConstantValue(mockMongoService); metadataService = container.get(MetadataService); }); diff --git a/src/node/consumer/test/services/mongo_repository.test.ts b/src/node/consumer/test/services/mongo_repository.test.ts index f1168f2..9af4b5f 100644 --- a/src/node/consumer/test/services/mongo_repository.test.ts +++ b/src/node/consumer/test/services/mongo_repository.test.ts @@ -10,17 +10,17 @@ jest.mock('@services/configuration_service', () => { MONGODB_HOST: 'localhost', MONGODB_PORT: '27017', MONGODB_DB: 'knightcrawler', - MONGO_INITDB_ROOT_USERNAME: 'mongo', - MONGO_INITDB_ROOT_PASSWORD: 'mongo', + MONGODB_USER: 'mongo', + MONGODB_PASSWORD: 'mongo', get MONGO_URI(): string { - return `mongodb://${this.MONGO_INITDB_ROOT_USERNAME}:${this.MONGO_INITDB_ROOT_PASSWORD}@${this.MONGODB_HOST}:${this.MONGODB_PORT}/${this.MONGODB_DB}?authSource=admin`; + return `mongodb://${this.MONGODB_USER}:${this.MONGODB_PASSWORD}@${this.MONGODB_HOST}:${this.MONGODB_PORT}/${this.MONGODB_DB}?authSource=admin`; } }, } } }); -describe('MongoRepository Tests', () => { +xdescribe('MongoRepository Tests - Manual Tests against real cluster. Skipped by default.', () => { let mongoRepository: MongoRepository; beforeEach(() => {