From 1b9a01c677ec32bb334adf85d920f403f9af2b6f Mon Sep 17 00:00:00 2001 From: iPromKnight Date: Wed, 28 Feb 2024 12:57:55 +0000 Subject: [PATCH] BREAKING: Cleanup RabbitMQ env vars, and Github Pat --- deployment/docker/.env.example | 22 +++--- .../lib/models/configuration/rabbit_config.ts | 13 +++- .../services/configuration_service.test.ts | 15 ++-- src/producer/Configuration/github.json | 5 -- src/producer/Configuration/rabbitmq.json | 12 ---- .../Sites/DebridMediaManagerCrawler.cs | 2 - .../Extensions/ConfigurationExtensions.cs | 4 -- .../Extensions/EnvironmentExtensions.cs | 68 +++++++++++++++++++ .../Extensions/ServiceCollectionExtensions.cs | 14 ++-- src/producer/GlobalUsings.cs | 2 + .../Configuration/GithubConfiguration.cs | 6 +- .../Configuration/PostgresConfiguration.cs | 20 ++---- .../Configuration/RabbitMqConfiguration.cs | 29 +++++--- src/producer/Producer.csproj | 9 --- src/producer/Program.cs | 6 +- src/producer/Services/DapperDataStorage.cs | 2 - src/producer/Services/TorrentPublisher.cs | 4 +- 17 files changed, 136 insertions(+), 97 deletions(-) delete mode 100644 src/producer/Configuration/github.json delete mode 100644 src/producer/Configuration/rabbitmq.json create mode 100644 src/producer/Extensions/EnvironmentExtensions.cs diff --git a/deployment/docker/.env.example b/deployment/docker/.env.example index 905c8dd..fba1e4d 100644 --- a/deployment/docker/.env.example +++ b/deployment/docker/.env.example @@ -15,6 +15,16 @@ MONGODB_DB=knightcrawler MONGODB_USER=mongo MONGODB_PASSWORD=mongo +# RabbitMQ +RABBITMQ_HOST=rabbitmq +RABBITMQ_USER=guest +RABBITMQ_PASSWORD=guest +RABBITMQ_QUEUE_NAME=ingested +RABBITMQ_DURABLE=true +RABBITMQ_MAX_QUEUE_SIZE=0 +RABBITMQ_MAX_PUBLISH_BATCH_SIZE=500 +RABBITMQ_PUBLISH_INTERVAL_IN_SECONDS=10 + # Metadata ## Only used if DATA_ONCE is set to false. If true, the schedule is ignored METADATA_DOWNLOAD_IMDB_DATA_SCHEDULE=0 0 1 * * * @@ -25,8 +35,6 @@ METADATA_DOWNLOAD_IMDB_DATA_ONCE=true DEBUG_MODE=false # Consumer -RABBIT_URI=amqp://guest:guest@rabbitmq:5672/?heartbeat=30 -QUEUE_NAME=ingested JOB_CONCURRENCY=5 JOBS_ENABLED=true ## can be debug for extra verbosity (a lot more verbosity - useful for development) @@ -40,12 +48,4 @@ CONSUMER_REPLICAS=3 AUTO_CREATE_AND_APPLY_MIGRATIONS=false # Producer -RabbitMqConfiguration__Host=rabbitmq -RabbitMqConfiguration__QueueName=ingested -RabbitMqConfiguration__Username=guest -RabbitMqConfiguration__Password=guest -RabbitMqConfiguration__Durable=true -RabbitMqConfiguration__MaxQueueSize=0 -RabbitMqConfiguration__MaxPublishBatchSize=500 -RabbitMqConfiguration__PublishIntervalInSeconds=10 -GithubSettings__PAT= +GITHUB_PAT= diff --git a/src/node/consumer/src/lib/models/configuration/rabbit_config.ts b/src/node/consumer/src/lib/models/configuration/rabbit_config.ts index b06ff9f..a0f5954 100644 --- a/src/node/consumer/src/lib/models/configuration/rabbit_config.ts +++ b/src/node/consumer/src/lib/models/configuration/rabbit_config.ts @@ -1,4 +1,13 @@ +import {BooleanHelpers} from "@helpers/boolean_helpers"; + export const rabbitConfig = { - RABBIT_URI: process.env.RABBIT_URI || 'amqp://localhost', - QUEUE_NAME: process.env.QUEUE_NAME || 'test-queue' + HOST: process.env.RABBITMQ_HOST || 'rabbitmq', + USER: process.env.RABBITMQ_USER || 'guest', + PASSWORD: process.env.RABBITMQ_PASSWORD || 'guest', + QUEUE_NAME: process.env.RABBITMQ_QUEUE_NAME || 'ingested', + DURABLE: BooleanHelpers.parseBool(process.env.RABBITMQ_DURABLE, true), + + get RABBIT_URI(): string { + return `amqp://${this.USER}:${this.PASSWORD}@${this.HOST}?heartbeat=30`; + } }; \ No newline at end of file diff --git a/src/node/consumer/test/services/configuration_service.test.ts b/src/node/consumer/test/services/configuration_service.test.ts index 21a979c..06ada2f 100644 --- a/src/node/consumer/test/services/configuration_service.test.ts +++ b/src/node/consumer/test/services/configuration_service.test.ts @@ -110,12 +110,19 @@ describe('Configuration Tests', () => { }); it('should populate rabbitConfig correctly', async () => { - process.env.RABBIT_URI = 'amqp://localhost'; - process.env.QUEUE_NAME = 'test-queue'; + process.env.RABBITMQ_HOST = 'rabbitmq'; + process.env.RABBITMQ_USER = 'guest'; + process.env.RABBITMQ_PASSWORD = 'guest'; + process.env.RABBITMQ_QUEUE_NAME = 'ingested'; + process.env.RABBITMQ_DURABLE = 'true'; const {configurationService} = await import("@services/configuration_service"); const {rabbitConfig} = configurationService; - expect(rabbitConfig.RABBIT_URI).toBe('amqp://localhost'); - expect(rabbitConfig.QUEUE_NAME).toBe('test-queue'); + expect(rabbitConfig.HOST).toBe('rabbitmq'); + expect(rabbitConfig.USER).toBe('guest'); + expect(rabbitConfig.PASSWORD).toBe('guest'); + expect(rabbitConfig.QUEUE_NAME).toBe('ingested'); + expect(rabbitConfig.DURABLE).toBe(true); + expect(rabbitConfig.RABBIT_URI).toBe('amqp://guest:guest@rabbitmq?heartbeat=30'); }); it('should populate torrentConfig correctly', async () => { diff --git a/src/producer/Configuration/github.json b/src/producer/Configuration/github.json deleted file mode 100644 index 1660747..0000000 --- a/src/producer/Configuration/github.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "GithubSettings": { - "PAT": "" - } -} \ No newline at end of file diff --git a/src/producer/Configuration/rabbitmq.json b/src/producer/Configuration/rabbitmq.json deleted file mode 100644 index 3544149..0000000 --- a/src/producer/Configuration/rabbitmq.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "RabbitMqConfiguration": { - "Host": "localhost", - "Username": "guest", - "Password": "guest", - "QueueName": "test-queue", - "Durable": true, - "MaxQueueSize": 0, - "MaxPublishBatchSize": 1, - "PublishIntervalInSeconds": 10 - } -} \ No newline at end of file diff --git a/src/producer/Crawlers/Sites/DebridMediaManagerCrawler.cs b/src/producer/Crawlers/Sites/DebridMediaManagerCrawler.cs index 1c5e49e..e18ed09 100644 --- a/src/producer/Crawlers/Sites/DebridMediaManagerCrawler.cs +++ b/src/producer/Crawlers/Sites/DebridMediaManagerCrawler.cs @@ -1,5 +1,3 @@ -using Producer.Models.Configuration; - namespace Producer.Crawlers.Sites; public partial class DebridMediaManagerCrawler( diff --git a/src/producer/Extensions/ConfigurationExtensions.cs b/src/producer/Extensions/ConfigurationExtensions.cs index 08fb531..a4d93b4 100644 --- a/src/producer/Extensions/ConfigurationExtensions.cs +++ b/src/producer/Extensions/ConfigurationExtensions.cs @@ -1,5 +1,3 @@ -using Producer.Models.Configuration; - namespace Producer.Extensions; public static class ConfigurationExtensions @@ -13,8 +11,6 @@ public static class ConfigurationExtensions configuration.AddJsonFile(LoggingConfig, false, true); configuration.AddJsonFile(ScrapeConfiguration.Filename, false, true); - configuration.AddJsonFile(RabbitMqConfiguration.Filename, false, true); - configuration.AddJsonFile(GithubConfiguration.Filename, false, true); configuration.AddEnvironmentVariables(); diff --git a/src/producer/Extensions/EnvironmentExtensions.cs b/src/producer/Extensions/EnvironmentExtensions.cs new file mode 100644 index 0000000..bdae5f3 --- /dev/null +++ b/src/producer/Extensions/EnvironmentExtensions.cs @@ -0,0 +1,68 @@ +namespace Producer.Extensions; + +public static class EnvironmentExtensions +{ + public static bool GetEnvironmentVariableAsBool(this string prefix, string varName, bool fallback = false) + { + var fullVarName = GetFullVariableName(prefix, varName); + + var str = Environment.GetEnvironmentVariable(fullVarName); + + if (string.IsNullOrEmpty(str)) + { + return fallback; + } + + return str.Trim().ToLower() switch + { + "true" => true, + "yes" => true, + "1" => true, + _ => false, + }; + } + + public static int GetEnvironmentVariableAsInt(this string prefix, string varName, int fallback = 0) + { + var fullVarName = GetFullVariableName(prefix, varName); + + var str = Environment.GetEnvironmentVariable(fullVarName); + + if (string.IsNullOrEmpty(str)) + { + return fallback; + } + + return int.TryParse(str, out var result) ? result : fallback; + } + + public static string GetRequiredEnvironmentVariableAsString(this string prefix, string varName) + { + var fullVarName = GetFullVariableName(prefix, varName); + + var str = Environment.GetEnvironmentVariable(fullVarName); + + if (string.IsNullOrEmpty(str)) + { + throw new InvalidOperationException($"Environment variable {fullVarName} is not set"); + } + + return str; + } + + public static string GetOptionalEnvironmentVariableAsString(this string prefix, string varName, string? fallback = null) + { + var fullVarName = GetFullVariableName(prefix, varName); + + var str = Environment.GetEnvironmentVariable(fullVarName); + + if (string.IsNullOrEmpty(str)) + { + return fallback; + } + + return str; + } + + private static string GetFullVariableName(string prefix, string varName) => $"{prefix}_{varName}"; +} \ No newline at end of file diff --git a/src/producer/Extensions/ServiceCollectionExtensions.cs b/src/producer/Extensions/ServiceCollectionExtensions.cs index 2774e62..01a7edb 100644 --- a/src/producer/Extensions/ServiceCollectionExtensions.cs +++ b/src/producer/Extensions/ServiceCollectionExtensions.cs @@ -1,5 +1,3 @@ -using Producer.Models.Configuration; - namespace Producer.Extensions; public static class ServiceCollectionExtensions @@ -29,13 +27,9 @@ public static class ServiceCollectionExtensions return services; } - internal static IServiceCollection RegisterMassTransit(this IServiceCollection services, IConfiguration configuration) + internal static IServiceCollection RegisterMassTransit(this IServiceCollection services) { - var rabbitConfig = configuration.GetSection(RabbitMqConfiguration.SectionName).Get(); - - ArgumentNullException.ThrowIfNull(rabbitConfig, nameof(rabbitConfig)); - - services.AddSingleton(rabbitConfig); + var rabbitConfig = services.LoadConfigurationFromEnv(); services.AddMassTransit(busConfigurator => { @@ -56,8 +50,8 @@ public static class ServiceCollectionExtensions internal static IServiceCollection AddQuartz(this IServiceCollection services, IConfiguration configuration) { var scrapeConfiguration = services.LoadConfigurationFromConfig(configuration, ScrapeConfiguration.SectionName); - var githubConfiguration = services.LoadConfigurationFromConfig(configuration, GithubConfiguration.SectionName); - var rabbitConfig = services.LoadConfigurationFromConfig(configuration, RabbitMqConfiguration.SectionName); + var githubConfiguration = services.LoadConfigurationFromEnv(); + var rabbitConfig = services.LoadConfigurationFromEnv(); services .AddTransient() diff --git a/src/producer/GlobalUsings.cs b/src/producer/GlobalUsings.cs index 0bf23aa..f44a4e6 100644 --- a/src/producer/GlobalUsings.cs +++ b/src/producer/GlobalUsings.cs @@ -16,8 +16,10 @@ global using Npgsql; global using Quartz; global using Producer.Crawlers; global using Producer.Crawlers.Sites; +global using Producer.Extensions; global using Producer.Interfaces; global using Producer.Jobs; global using Producer.Models; +global using Producer.Models.Configuration; global using Producer.Services; global using Serilog; \ No newline at end of file diff --git a/src/producer/Models/Configuration/GithubConfiguration.cs b/src/producer/Models/Configuration/GithubConfiguration.cs index 0e25749..277308c 100644 --- a/src/producer/Models/Configuration/GithubConfiguration.cs +++ b/src/producer/Models/Configuration/GithubConfiguration.cs @@ -2,8 +2,8 @@ public class GithubConfiguration { - public const string SectionName = "GithubSettings"; - public const string Filename = "github.json"; + private const string Prefix = "GITHUB"; + private const string PatVariable = "PAT"; - public string? PAT { get; set; } + public string? PAT { get; init; } = Prefix.GetOptionalEnvironmentVariableAsString(PatVariable); } \ No newline at end of file diff --git a/src/producer/Models/Configuration/PostgresConfiguration.cs b/src/producer/Models/Configuration/PostgresConfiguration.cs index fd3fb8d..c753b4f 100644 --- a/src/producer/Models/Configuration/PostgresConfiguration.cs +++ b/src/producer/Models/Configuration/PostgresConfiguration.cs @@ -9,21 +9,11 @@ public class PostgresConfiguration private const string DatabaseVariable = "DB"; private const string PortVariable = "PORT"; - private string Host { get; init; } = Environment.GetEnvironmentVariable($"{Prefix}_{HostVariable}") ?? - throw new InvalidOperationException($"Environment variable {Prefix}_{HostVariable} is not set"); - - private string Username { get; init; } = Environment.GetEnvironmentVariable($"{Prefix}_{UsernameVariable}") ?? - throw new InvalidOperationException($"Environment variable {Prefix}_{UsernameVariable} is not set"); - - private string Password { get; init; } = Environment.GetEnvironmentVariable($"{Prefix}_{PasswordVariable}") ?? - throw new InvalidOperationException($"Environment variable {Prefix}_{PasswordVariable} is not set"); - - private string Database { get; init; } = Environment.GetEnvironmentVariable($"{Prefix}_{DatabaseVariable}") ?? - throw new InvalidOperationException($"Environment variable {Prefix}_{DatabaseVariable} is not set"); - - private int PORT { get; init; } = int.Parse( - Environment.GetEnvironmentVariable($"{Prefix}_{PortVariable}") ?? - throw new InvalidOperationException($"Environment variable {Prefix}_{PortVariable} is not set")); + private string Host { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(HostVariable); + private string Username { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(UsernameVariable); + private string Password { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(PasswordVariable); + private string Database { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(DatabaseVariable); + private int PORT { get; init; } = Prefix.GetEnvironmentVariableAsInt(PortVariable, 5432); public string StorageConnectionString => $"Host={Host};Port={PORT};Username={Username};Password={Password};Database={Database};"; } \ No newline at end of file diff --git a/src/producer/Models/Configuration/RabbitMqConfiguration.cs b/src/producer/Models/Configuration/RabbitMqConfiguration.cs index da2f17a..d4eacb1 100644 --- a/src/producer/Models/Configuration/RabbitMqConfiguration.cs +++ b/src/producer/Models/Configuration/RabbitMqConfiguration.cs @@ -1,18 +1,25 @@ -namespace Producer.Models.Configuration; +namespace Producer.Models.Configuration; public class RabbitMqConfiguration { - public const string SectionName = "RabbitMqConfiguration"; - public const string Filename = "rabbitmq.json"; + private const string Prefix = "RABBITMQ"; + private const string HostVariable = "HOST"; + private const string UsernameVariable = "USER"; + private const string PasswordVariable = "PASSWORD"; + private const string QueueNameVariable = "QUEUE_NAME"; + private const string DurableVariable = "DURABLE"; + private const string MaxQueueSizeVariable = "MAX_QUEUE_SIZE"; + private const string MaxPublishBatchSizeVariable = "MAX_PUBLISH_BATCH_SIZE"; + private const string PublishIntervalInSecondsVariable = "PUBLISH_INTERVAL_IN_SECONDS"; - public string? Host { get; set; } - public string? Username { get; set; } - public string? Password { get; set; } - public string? QueueName { get; set; } - public bool Durable { get; set; } - public int MaxQueueSize { get; set; } - public int MaxPublishBatchSize { get; set; } = 500; - public int PublishIntervalInSeconds { get; set; } = 1000 * 10; + public string? Username { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(UsernameVariable); + public string? Host { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(HostVariable); + public string? Password { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(PasswordVariable); + public string? QueueName { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(QueueNameVariable); + public bool Durable { get; init; } = Prefix.GetEnvironmentVariableAsBool(DurableVariable, true); + public int MaxQueueSize { get; init; } = Prefix.GetEnvironmentVariableAsInt(MaxQueueSizeVariable, 0); + public int MaxPublishBatchSize { get; set; } = Prefix.GetEnvironmentVariableAsInt(MaxPublishBatchSizeVariable, 500); + public int PublishIntervalInSeconds { get; set; } = Prefix.GetEnvironmentVariableAsInt(PublishIntervalInSecondsVariable, 1000 * 10); public void Validate() { diff --git a/src/producer/Producer.csproj b/src/producer/Producer.csproj index 787cb5c..8cc3862 100644 --- a/src/producer/Producer.csproj +++ b/src/producer/Producer.csproj @@ -32,15 +32,6 @@ Always - - - Always - - - - Always - - diff --git a/src/producer/Program.cs b/src/producer/Program.cs index 5102b75..da1257d 100644 --- a/src/producer/Program.cs +++ b/src/producer/Program.cs @@ -1,6 +1,4 @@ -using Producer.Extensions; - -var builder = WebApplication.CreateBuilder(args); +var builder = WebApplication.CreateBuilder(args); builder.Configuration .AddScrapeConfiguration(); @@ -9,7 +7,7 @@ builder.Host .SetupSerilog(builder.Configuration); builder.Services - .RegisterMassTransit(builder.Configuration) + .RegisterMassTransit() .AddDataStorage() .AddCrawlers() .AddQuartz(builder.Configuration); diff --git a/src/producer/Services/DapperDataStorage.cs b/src/producer/Services/DapperDataStorage.cs index 0c6fa3a..c08b082 100644 --- a/src/producer/Services/DapperDataStorage.cs +++ b/src/producer/Services/DapperDataStorage.cs @@ -1,5 +1,3 @@ -using Producer.Models.Configuration; - namespace Producer.Services; public class DapperDataStorage(PostgresConfiguration configuration, RabbitMqConfiguration rabbitConfig, ILogger logger) : IDataStorage diff --git a/src/producer/Services/TorrentPublisher.cs b/src/producer/Services/TorrentPublisher.cs index cb8434b..ac39745 100644 --- a/src/producer/Services/TorrentPublisher.cs +++ b/src/producer/Services/TorrentPublisher.cs @@ -1,6 +1,4 @@ -using Producer.Models.Configuration; - -namespace Producer.Services; +namespace Producer.Services; public class TorrentPublisher( ISendEndpointProvider sendEndpointProvider,