From e461e26b0f42236ec99a929d8f78445152ae2087 Mon Sep 17 00:00:00 2001 From: iPromKnight Date: Sun, 4 Feb 2024 15:03:07 +0000 Subject: [PATCH] Change postgres configuration in the producer to use the env vars from the stack --- .env | 3 +- src/producer/Configuration/scrapers.json | 1 - .../Sites/DebridMediaManagerCrawler.cs | 2 + .../Extensions/ConfigurationExtensions.cs | 2 + .../Extensions/ServiceCollectionExtensions.cs | 60 +++++++------------ src/producer/GlobalUsings.cs | 1 + .../GithubConfiguration.cs | 2 +- .../Configuration/PostgresConfiguration.cs | 29 +++++++++ .../Configuration/RabbitMqConfiguration.cs | 39 ++++++++++++ .../ScrapeConfiguration.cs | 3 +- src/producer/Models/RabbitMqConfiguration.cs | 16 ----- src/producer/Producer.csproj | 1 + src/producer/Services/DapperDataStorage.cs | 4 +- src/producer/Services/TorrentPublisher.cs | 4 +- 14 files changed, 106 insertions(+), 61 deletions(-) rename src/producer/Models/{ => Configuration}/GithubConfiguration.cs (80%) create mode 100644 src/producer/Models/Configuration/PostgresConfiguration.cs create mode 100644 src/producer/Models/Configuration/RabbitMqConfiguration.cs rename src/producer/Models/{ => Configuration}/ScrapeConfiguration.cs (70%) delete mode 100644 src/producer/Models/RabbitMqConfiguration.cs diff --git a/.env b/.env index 7b00344..21a715c 100644 --- a/.env +++ b/.env @@ -28,7 +28,6 @@ TORRENT_TIMEOUT=30000 UDP_TRACKERS_ENABLED=true # Producer -ScrapeConfiguration__StorageConnectionString=host=postgres;username=postgres;password=postgres;database=knightcrawler; RabbitMqConfiguration__Host=rabbitmq RabbitMqConfiguration__QueueName=ingested RabbitMqConfiguration__Username=guest @@ -37,4 +36,4 @@ RabbitMqConfiguration__Durable=true RabbitMqConfiguration__MaxQueueSize=0 RabbitMqConfiguration__MaxPublishBatchSize=500 RabbitMqConfiguration__PublishIntervalInSeconds=10 -GithubSettings__PAT= \ No newline at end of file +GithubSettings__PAT= diff --git a/src/producer/Configuration/scrapers.json b/src/producer/Configuration/scrapers.json index aa30591..5cd9b5d 100644 --- a/src/producer/Configuration/scrapers.json +++ b/src/producer/Configuration/scrapers.json @@ -1,6 +1,5 @@ { "ScrapeConfiguration": { - "StorageConnectionString": "", "Scrapers": [ { "Name": "SyncEzTvJob", diff --git a/src/producer/Crawlers/Sites/DebridMediaManagerCrawler.cs b/src/producer/Crawlers/Sites/DebridMediaManagerCrawler.cs index dabb738..8daf3ae 100644 --- a/src/producer/Crawlers/Sites/DebridMediaManagerCrawler.cs +++ b/src/producer/Crawlers/Sites/DebridMediaManagerCrawler.cs @@ -1,3 +1,5 @@ +using Producer.Models.Configuration; + namespace Producer.Crawlers.Sites; public partial class DebridMediaManagerCrawler( diff --git a/src/producer/Extensions/ConfigurationExtensions.cs b/src/producer/Extensions/ConfigurationExtensions.cs index b321cab..08fb531 100644 --- a/src/producer/Extensions/ConfigurationExtensions.cs +++ b/src/producer/Extensions/ConfigurationExtensions.cs @@ -1,3 +1,5 @@ +using Producer.Models.Configuration; + namespace Producer.Extensions; public static class ConfigurationExtensions diff --git a/src/producer/Extensions/ServiceCollectionExtensions.cs b/src/producer/Extensions/ServiceCollectionExtensions.cs index 7ce067a..3461e77 100644 --- a/src/producer/Extensions/ServiceCollectionExtensions.cs +++ b/src/producer/Extensions/ServiceCollectionExtensions.cs @@ -1,3 +1,5 @@ +using Producer.Models.Configuration; + namespace Producer.Extensions; public static class ServiceCollectionExtensions @@ -20,6 +22,7 @@ public static class ServiceCollectionExtensions internal static IServiceCollection AddDataStorage(this IServiceCollection services) { + services.LoadConfigurationFromEnv(); services.AddTransient(); services.AddTransient(); return services; @@ -36,9 +39,9 @@ public static class ServiceCollectionExtensions services.AddMassTransit(busConfigurator => { busConfigurator.SetKebabCaseEndpointNameFormatter(); - busConfigurator.UsingRabbitMq((context, busFactoryConfigurator) => + busConfigurator.UsingRabbitMq((_, busFactoryConfigurator) => { - busFactoryConfigurator.Host(rabbitConfig!.Host, hostConfigurator => + busFactoryConfigurator.Host(rabbitConfig.Host, hostConfigurator => { hostConfigurator.Username(rabbitConfig.Username); hostConfigurator.Password(rabbitConfig.Password); @@ -51,9 +54,9 @@ public static class ServiceCollectionExtensions internal static IServiceCollection AddQuartz(this IServiceCollection services, IConfiguration configuration) { - var scrapeConfiguration = LoadScrapeConfiguration(services, configuration); - var githubConfiguration = LoadGithubConfiguration(services, configuration); - var rabbitConfig = LoadRabbitMQConfiguration(services, configuration); + var scrapeConfiguration = services.LoadConfigurationFromConfig(configuration, ScrapeConfiguration.SectionName); + var githubConfiguration = services.LoadConfigurationFromConfig(configuration, GithubConfiguration.SectionName); + var rabbitConfig = services.LoadConfigurationFromConfig(configuration, RabbitMqConfiguration.SectionName); services .AddTransient() @@ -92,46 +95,29 @@ public static class ServiceCollectionExtensions return services; } - - private static GithubConfiguration LoadGithubConfiguration(IServiceCollection services, IConfiguration configuration) + + private static TConfiguration LoadConfigurationFromConfig(this IServiceCollection services, IConfiguration configuration, string sectionName) + where TConfiguration : class { - var githubConfiguration = configuration.GetSection(GithubConfiguration.SectionName).Get(); + var instance = configuration.GetSection(sectionName).Get(); - ArgumentNullException.ThrowIfNull(githubConfiguration, nameof(githubConfiguration)); - - services.TryAddSingleton(githubConfiguration); + ArgumentNullException.ThrowIfNull(instance, nameof(instance)); - return githubConfiguration; + services.TryAddSingleton(instance); + + return instance; } - - private static RabbitMqConfiguration LoadRabbitMQConfiguration(IServiceCollection services, IConfiguration configuration) + + private static TConfiguration LoadConfigurationFromEnv(this IServiceCollection services) + where TConfiguration : class { - var rabbitConfiguration = configuration.GetSection(RabbitMqConfiguration.SectionName).Get(); - - ArgumentNullException.ThrowIfNull(rabbitConfiguration, nameof(rabbitConfiguration)); - - if (rabbitConfiguration.MaxQueueSize > 0) - { - if (rabbitConfiguration.MaxPublishBatchSize > rabbitConfiguration.MaxQueueSize) - { - throw new InvalidOperationException("MaxPublishBatchSize cannot be greater than MaxQueueSize in RabbitMqConfiguration"); - } - } - - services.TryAddSingleton(rabbitConfiguration); - - return rabbitConfiguration; - } - - private static ScrapeConfiguration LoadScrapeConfiguration(IServiceCollection services, IConfiguration configuration) - { - var scrapeConfiguration = configuration.GetSection(ScrapeConfiguration.SectionName).Get(); + var instance = Activator.CreateInstance(); - ArgumentNullException.ThrowIfNull(scrapeConfiguration, nameof(scrapeConfiguration)); + ArgumentNullException.ThrowIfNull(instance, nameof(instance)); - services.TryAddSingleton(scrapeConfiguration); + services.TryAddSingleton(instance); - return scrapeConfiguration; + return instance; } private static void AddJobWithTrigger( diff --git a/src/producer/GlobalUsings.cs b/src/producer/GlobalUsings.cs index 1a7749d..0bf23aa 100644 --- a/src/producer/GlobalUsings.cs +++ b/src/producer/GlobalUsings.cs @@ -2,6 +2,7 @@ global using System.Text; global using System.Text.Json; +global using System.Text.Json.Serialization; global using System.Text.RegularExpressions; global using System.Xml.Linq; global using Dapper; diff --git a/src/producer/Models/GithubConfiguration.cs b/src/producer/Models/Configuration/GithubConfiguration.cs similarity index 80% rename from src/producer/Models/GithubConfiguration.cs rename to src/producer/Models/Configuration/GithubConfiguration.cs index 429b8d0..0e25749 100644 --- a/src/producer/Models/GithubConfiguration.cs +++ b/src/producer/Models/Configuration/GithubConfiguration.cs @@ -1,4 +1,4 @@ -namespace Producer.Models; +namespace Producer.Models.Configuration; public class GithubConfiguration { diff --git a/src/producer/Models/Configuration/PostgresConfiguration.cs b/src/producer/Models/Configuration/PostgresConfiguration.cs new file mode 100644 index 0000000..fd3fb8d --- /dev/null +++ b/src/producer/Models/Configuration/PostgresConfiguration.cs @@ -0,0 +1,29 @@ +namespace Producer.Models.Configuration; + +public class PostgresConfiguration +{ + private const string Prefix = "POSTGRES"; + private const string HostVariable = "HOST"; + private const string UsernameVariable = "USER"; + private const string PasswordVariable = "PASSWORD"; + private const string DatabaseVariable = "DB"; + private const string PortVariable = "PORT"; + + private string Host { get; init; } = Environment.GetEnvironmentVariable($"{Prefix}_{HostVariable}") ?? + throw new InvalidOperationException($"Environment variable {Prefix}_{HostVariable} is not set"); + + private string Username { get; init; } = Environment.GetEnvironmentVariable($"{Prefix}_{UsernameVariable}") ?? + throw new InvalidOperationException($"Environment variable {Prefix}_{UsernameVariable} is not set"); + + private string Password { get; init; } = Environment.GetEnvironmentVariable($"{Prefix}_{PasswordVariable}") ?? + throw new InvalidOperationException($"Environment variable {Prefix}_{PasswordVariable} is not set"); + + private string Database { get; init; } = Environment.GetEnvironmentVariable($"{Prefix}_{DatabaseVariable}") ?? + throw new InvalidOperationException($"Environment variable {Prefix}_{DatabaseVariable} is not set"); + + private int PORT { get; init; } = int.Parse( + Environment.GetEnvironmentVariable($"{Prefix}_{PortVariable}") ?? + throw new InvalidOperationException($"Environment variable {Prefix}_{PortVariable} is not set")); + + public string StorageConnectionString => $"Host={Host};Port={PORT};Username={Username};Password={Password};Database={Database};"; +} \ No newline at end of file diff --git a/src/producer/Models/Configuration/RabbitMqConfiguration.cs b/src/producer/Models/Configuration/RabbitMqConfiguration.cs new file mode 100644 index 0000000..da2f17a --- /dev/null +++ b/src/producer/Models/Configuration/RabbitMqConfiguration.cs @@ -0,0 +1,39 @@ +namespace Producer.Models.Configuration; + +public class RabbitMqConfiguration +{ + public const string SectionName = "RabbitMqConfiguration"; + public const string Filename = "rabbitmq.json"; + + public string? Host { get; set; } + public string? Username { get; set; } + public string? Password { get; set; } + public string? QueueName { get; set; } + public bool Durable { get; set; } + public int MaxQueueSize { get; set; } + public int MaxPublishBatchSize { get; set; } = 500; + public int PublishIntervalInSeconds { get; set; } = 1000 * 10; + + public void Validate() + { + if (MaxQueueSize == 0) + { + return; + } + + if (MaxQueueSize < 0) + { + throw new InvalidOperationException("MaxQueueSize cannot be less than 0 in RabbitMqConfiguration"); + } + + if (MaxPublishBatchSize < 0) + { + throw new InvalidOperationException("MaxPublishBatchSize cannot be less than 0 in RabbitMqConfiguration"); + } + + if (MaxPublishBatchSize > MaxQueueSize) + { + throw new InvalidOperationException("MaxPublishBatchSize cannot be greater than MaxQueueSize in RabbitMqConfiguration"); + } + } +} \ No newline at end of file diff --git a/src/producer/Models/ScrapeConfiguration.cs b/src/producer/Models/Configuration/ScrapeConfiguration.cs similarity index 70% rename from src/producer/Models/ScrapeConfiguration.cs rename to src/producer/Models/Configuration/ScrapeConfiguration.cs index 716e32a..1073b0c 100644 --- a/src/producer/Models/ScrapeConfiguration.cs +++ b/src/producer/Models/Configuration/ScrapeConfiguration.cs @@ -1,4 +1,4 @@ -namespace Producer.Models; +namespace Producer.Models.Configuration; public class ScrapeConfiguration { @@ -6,5 +6,4 @@ public class ScrapeConfiguration public const string Filename = "scrapers.json"; public List Scrapers { get; set; } = []; - public string StorageConnectionString { get; set; } = ""; } \ No newline at end of file diff --git a/src/producer/Models/RabbitMqConfiguration.cs b/src/producer/Models/RabbitMqConfiguration.cs deleted file mode 100644 index ccc0277..0000000 --- a/src/producer/Models/RabbitMqConfiguration.cs +++ /dev/null @@ -1,16 +0,0 @@ -namespace Producer.Models; - -public class RabbitMqConfiguration -{ - public const string SectionName = "RabbitMqConfiguration"; - public const string Filename = "rabbitmq.json"; - - public string? Host { get; set; } - public string? Username { get; set; } - public string? Password { get; set; } - public string? QueueName { get; set; } - public bool Durable { get; set; } - public int MaxQueueSize { get; set; } - public int MaxPublishBatchSize { get; set; } = 500; - public int PublishIntervalInSeconds { get; set; } = 1000 * 10; -} \ No newline at end of file diff --git a/src/producer/Producer.csproj b/src/producer/Producer.csproj index d54c9ec..787cb5c 100644 --- a/src/producer/Producer.csproj +++ b/src/producer/Producer.csproj @@ -40,6 +40,7 @@ Always + diff --git a/src/producer/Services/DapperDataStorage.cs b/src/producer/Services/DapperDataStorage.cs index 5fe352e..0c6fa3a 100644 --- a/src/producer/Services/DapperDataStorage.cs +++ b/src/producer/Services/DapperDataStorage.cs @@ -1,6 +1,8 @@ +using Producer.Models.Configuration; + namespace Producer.Services; -public class DapperDataStorage(ScrapeConfiguration configuration, RabbitMqConfiguration rabbitConfig, ILogger logger) : IDataStorage +public class DapperDataStorage(PostgresConfiguration configuration, RabbitMqConfiguration rabbitConfig, ILogger logger) : IDataStorage { private const string InsertTorrentSql = """ diff --git a/src/producer/Services/TorrentPublisher.cs b/src/producer/Services/TorrentPublisher.cs index ac39745..cb8434b 100644 --- a/src/producer/Services/TorrentPublisher.cs +++ b/src/producer/Services/TorrentPublisher.cs @@ -1,4 +1,6 @@ -namespace Producer.Services; +using Producer.Models.Configuration; + +namespace Producer.Services; public class TorrentPublisher( ISendEndpointProvider sendEndpointProvider,