BREAKING: Cleanup RabbitMQ env vars, and Github Pat

deployment/docker/.env.example

@@ -15,6 +15,16 @@ MONGODB_DB=knightcrawler
 MONGODB_USER=mongo
 MONGODB_PASSWORD=mongo
 
+# RabbitMQ
+RABBITMQ_HOST=rabbitmq
+RABBITMQ_USER=guest
+RABBITMQ_PASSWORD=guest
+RABBITMQ_QUEUE_NAME=ingested
+RABBITMQ_DURABLE=true
+RABBITMQ_MAX_QUEUE_SIZE=0
+RABBITMQ_MAX_PUBLISH_BATCH_SIZE=500
+RABBITMQ_PUBLISH_INTERVAL_IN_SECONDS=10
+
 # Metadata
 ## Only used if DATA_ONCE is set to false. If true, the schedule is ignored 
 METADATA_DOWNLOAD_IMDB_DATA_SCHEDULE=0 0 1 * * *
@@ -25,8 +35,6 @@ METADATA_DOWNLOAD_IMDB_DATA_ONCE=true
 DEBUG_MODE=false
 
 # Consumer
-RABBIT_URI=amqp://guest:guest@rabbitmq:5672/?heartbeat=30
-QUEUE_NAME=ingested
 JOB_CONCURRENCY=5
 JOBS_ENABLED=true
 ## can be debug for extra verbosity (a lot more verbosity - useful for development)
@@ -40,12 +48,4 @@ CONSUMER_REPLICAS=3
 AUTO_CREATE_AND_APPLY_MIGRATIONS=false
 
 # Producer
-RabbitMqConfiguration__Host=rabbitmq
+GITHUB_PAT=
-RabbitMqConfiguration__QueueName=ingested
-RabbitMqConfiguration__Username=guest
-RabbitMqConfiguration__Password=guest
-RabbitMqConfiguration__Durable=true
-RabbitMqConfiguration__MaxQueueSize=0
-RabbitMqConfiguration__MaxPublishBatchSize=500
-RabbitMqConfiguration__PublishIntervalInSeconds=10
-GithubSettings__PAT=

src/node/consumer/src/lib/models/configuration/rabbit_config.ts

@@ -1,4 +1,13 @@
+import {BooleanHelpers} from "@helpers/boolean_helpers";
+
 export const rabbitConfig = {
-    RABBIT_URI: process.env.RABBIT_URI || 'amqp://localhost',
+    HOST: process.env.RABBITMQ_HOST || 'rabbitmq',
-    QUEUE_NAME: process.env.QUEUE_NAME || 'test-queue'
+    USER: process.env.RABBITMQ_USER || 'guest',
+    PASSWORD: process.env.RABBITMQ_PASSWORD || 'guest',
+    QUEUE_NAME: process.env.RABBITMQ_QUEUE_NAME || 'ingested',
+    DURABLE: BooleanHelpers.parseBool(process.env.RABBITMQ_DURABLE, true),
+    
+    get RABBIT_URI(): string {
+        return `amqp://${this.USER}:${this.PASSWORD}@${this.HOST}?heartbeat=30`;
+    }
 };

src/node/consumer/test/services/configuration_service.test.ts

@@ -110,12 +110,19 @@ describe('Configuration Tests', () => {
     });
 
     it('should populate rabbitConfig correctly', async () => {
-        process.env.RABBIT_URI = 'amqp://localhost';
+        process.env.RABBITMQ_HOST = 'rabbitmq';
-        process.env.QUEUE_NAME = 'test-queue';
+        process.env.RABBITMQ_USER = 'guest';
+        process.env.RABBITMQ_PASSWORD = 'guest';
+        process.env.RABBITMQ_QUEUE_NAME = 'ingested';
+        process.env.RABBITMQ_DURABLE = 'true';
         const {configurationService} = await import("@services/configuration_service");
         const {rabbitConfig} = configurationService;
-        expect(rabbitConfig.RABBIT_URI).toBe('amqp://localhost');
+        expect(rabbitConfig.HOST).toBe('rabbitmq');
-        expect(rabbitConfig.QUEUE_NAME).toBe('test-queue');
+        expect(rabbitConfig.USER).toBe('guest');
+        expect(rabbitConfig.PASSWORD).toBe('guest');
+        expect(rabbitConfig.QUEUE_NAME).toBe('ingested');
+        expect(rabbitConfig.DURABLE).toBe(true);
+        expect(rabbitConfig.RABBIT_URI).toBe('amqp://guest:guest@rabbitmq?heartbeat=30');
     });
 
     it('should populate torrentConfig correctly', async () => {

src/producer/Configuration/github.json

@@ -1,5 +0,0 @@
-{
-  "GithubSettings": {
-    "PAT": ""
-  }
-}

src/producer/Configuration/rabbitmq.json

@@ -1,12 +0,0 @@
-{
-  "RabbitMqConfiguration": {
-    "Host": "localhost",
-    "Username": "guest",
-    "Password": "guest",
-    "QueueName": "test-queue",
-    "Durable": true,
-    "MaxQueueSize": 0,
-    "MaxPublishBatchSize": 1,
-    "PublishIntervalInSeconds": 10
-  }
-}

src/producer/Crawlers/Sites/DebridMediaManagerCrawler.cs

@@ -1,5 +1,3 @@
-using Producer.Models.Configuration;
-
 namespace Producer.Crawlers.Sites;
 
 public partial class DebridMediaManagerCrawler(

src/producer/Extensions/ConfigurationExtensions.cs

@@ -1,5 +1,3 @@
-using Producer.Models.Configuration;
-
 namespace Producer.Extensions;
 
 public static class ConfigurationExtensions
@@ -13,8 +11,6 @@ public static class ConfigurationExtensions
         
         configuration.AddJsonFile(LoggingConfig, false, true);
         configuration.AddJsonFile(ScrapeConfiguration.Filename, false, true);
-        configuration.AddJsonFile(RabbitMqConfiguration.Filename, false, true);
-        configuration.AddJsonFile(GithubConfiguration.Filename, false, true);
         
         configuration.AddEnvironmentVariables();
 

src/producer/Extensions/EnvironmentExtensions.cs

@@ -0,0 +1,68 @@
+namespace Producer.Extensions;
+
+public static class EnvironmentExtensions
+{
+    public static bool GetEnvironmentVariableAsBool(this string prefix, string varName, bool fallback = false)
+    {
+        var fullVarName = GetFullVariableName(prefix, varName);
+        
+        var str = Environment.GetEnvironmentVariable(fullVarName);
+        
+        if (string.IsNullOrEmpty(str))
+        {
+            return fallback;
+        }
+
+        return str.Trim().ToLower() switch
+        {
+            "true" => true,
+            "yes" => true,
+            "1" => true,
+            _ => false,
+        };
+    }
+    
+    public static int GetEnvironmentVariableAsInt(this string prefix, string varName, int fallback = 0)
+    {
+        var fullVarName = GetFullVariableName(prefix, varName);
+        
+        var str = Environment.GetEnvironmentVariable(fullVarName);
+
+        if (string.IsNullOrEmpty(str))
+        {
+            return fallback;
+        }
+
+        return int.TryParse(str, out var result) ? result : fallback;
+    }
+    
+    public static string GetRequiredEnvironmentVariableAsString(this string prefix, string varName)
+    {
+        var fullVarName = GetFullVariableName(prefix, varName);
+        
+        var str = Environment.GetEnvironmentVariable(fullVarName);
+
+        if (string.IsNullOrEmpty(str))
+        {
+            throw new InvalidOperationException($"Environment variable {fullVarName} is not set");
+        }
+
+        return str;
+    }
+    
+    public static string GetOptionalEnvironmentVariableAsString(this string prefix, string varName, string? fallback = null)
+    {
+        var fullVarName = GetFullVariableName(prefix, varName);
+        
+        var str = Environment.GetEnvironmentVariable(fullVarName);
+
+        if (string.IsNullOrEmpty(str))
+        {
+            return fallback;
+        }
+
+        return str;
+    }
+    
+    private static string GetFullVariableName(string prefix, string varName) => $"{prefix}_{varName}";
+}

src/producer/Extensions/ServiceCollectionExtensions.cs

@@ -1,5 +1,3 @@
-using Producer.Models.Configuration;
-
 namespace Producer.Extensions;
 
 public static class ServiceCollectionExtensions
@@ -29,13 +27,9 @@ public static class ServiceCollectionExtensions
         return services;
     }
 
-    internal static IServiceCollection RegisterMassTransit(this IServiceCollection services, IConfiguration configuration)
+    internal static IServiceCollection RegisterMassTransit(this IServiceCollection services)
     {
-        var rabbitConfig = configuration.GetSection(RabbitMqConfiguration.SectionName).Get<RabbitMqConfiguration>();
+        var rabbitConfig = services.LoadConfigurationFromEnv<RabbitMqConfiguration>();
-        
-        ArgumentNullException.ThrowIfNull(rabbitConfig, nameof(rabbitConfig));
-
-        services.AddSingleton(rabbitConfig);
         
         services.AddMassTransit(busConfigurator =>
         {
@@ -56,8 +50,8 @@ public static class ServiceCollectionExtensions
     internal static IServiceCollection AddQuartz(this IServiceCollection services, IConfiguration configuration)
     {
         var scrapeConfiguration = services.LoadConfigurationFromConfig<ScrapeConfiguration>(configuration, ScrapeConfiguration.SectionName);
-        var githubConfiguration = services.LoadConfigurationFromConfig<GithubConfiguration>(configuration, GithubConfiguration.SectionName);
+        var githubConfiguration = services.LoadConfigurationFromEnv<GithubConfiguration>();
-        var rabbitConfig = services.LoadConfigurationFromConfig<RabbitMqConfiguration>(configuration, RabbitMqConfiguration.SectionName);
+        var rabbitConfig = services.LoadConfigurationFromEnv<RabbitMqConfiguration>();
 
         services
             .AddTransient<SyncEzTvJob>()

src/producer/GlobalUsings.cs

@@ -16,8 +16,10 @@ global using Npgsql;
 global using Quartz;
 global using Producer.Crawlers;
 global using Producer.Crawlers.Sites;
+global using Producer.Extensions;
 global using Producer.Interfaces;
 global using Producer.Jobs;
 global using Producer.Models;
+global using Producer.Models.Configuration;
 global using Producer.Services;
 global using Serilog;

src/producer/Models/Configuration/GithubConfiguration.cs

@@ -2,8 +2,8 @@
 
 public class GithubConfiguration
 {
-    public const string SectionName = "GithubSettings";
+    private const string Prefix = "GITHUB";
-    public const string Filename = "github.json";
+    private const string PatVariable = "PAT";
     
-    public string? PAT { get; set; }
+    public string? PAT { get; init; } =  Prefix.GetOptionalEnvironmentVariableAsString(PatVariable);
 }

src/producer/Models/Configuration/PostgresConfiguration.cs

@@ -9,21 +9,11 @@ public class PostgresConfiguration
     private const string DatabaseVariable = "DB";
     private const string PortVariable = "PORT";
 
-    private string Host { get; init; } = Environment.GetEnvironmentVariable($"{Prefix}_{HostVariable}") ??
+    private string Host { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(HostVariable);
-                                         throw new InvalidOperationException($"Environment variable {Prefix}_{HostVariable} is not set");
+    private string Username { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(UsernameVariable);
-
+    private string Password { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(PasswordVariable);
-    private string Username { get; init; } = Environment.GetEnvironmentVariable($"{Prefix}_{UsernameVariable}") ??
+    private string Database { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(DatabaseVariable);
-                                             throw new InvalidOperationException($"Environment variable {Prefix}_{UsernameVariable} is not set");
+    private int PORT { get; init; } = Prefix.GetEnvironmentVariableAsInt(PortVariable, 5432);
-
-    private string Password { get; init; } = Environment.GetEnvironmentVariable($"{Prefix}_{PasswordVariable}") ??
-                                             throw new InvalidOperationException($"Environment variable {Prefix}_{PasswordVariable} is not set");
-
-    private string Database { get; init; } = Environment.GetEnvironmentVariable($"{Prefix}_{DatabaseVariable}") ??
-                                             throw new InvalidOperationException($"Environment variable {Prefix}_{DatabaseVariable} is not set");
-
-    private int PORT { get; init; } = int.Parse(
-        Environment.GetEnvironmentVariable($"{Prefix}_{PortVariable}") ??
-        throw new InvalidOperationException($"Environment variable {Prefix}_{PortVariable} is not set"));
 
     public string StorageConnectionString => $"Host={Host};Port={PORT};Username={Username};Password={Password};Database={Database};";
 }

src/producer/Models/Configuration/RabbitMqConfiguration.cs

@@ -1,18 +1,25 @@
-namespace Producer.Models.Configuration;
+namespace Producer.Models.Configuration;
 
 public class RabbitMqConfiguration
 {
-    public const string SectionName = "RabbitMqConfiguration";
+    private const string Prefix = "RABBITMQ";
-    public const string Filename = "rabbitmq.json";
+    private const string HostVariable = "HOST";
+    private const string UsernameVariable = "USER";
+    private const string PasswordVariable = "PASSWORD";
+    private const string QueueNameVariable = "QUEUE_NAME";
+    private const string DurableVariable = "DURABLE";
+    private const string MaxQueueSizeVariable = "MAX_QUEUE_SIZE";
+    private const string MaxPublishBatchSizeVariable = "MAX_PUBLISH_BATCH_SIZE";
+    private const string PublishIntervalInSecondsVariable = "PUBLISH_INTERVAL_IN_SECONDS";
     
-    public string? Host { get; set; }
+    public string? Username { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(UsernameVariable);
-    public string? Username { get; set; }
+    public string? Host { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(HostVariable);
-    public string? Password { get; set; }
+    public string? Password { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(PasswordVariable);
-    public string? QueueName { get; set; }
+    public string? QueueName { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(QueueNameVariable);
-    public bool Durable { get; set; }
+    public bool Durable { get; init; } = Prefix.GetEnvironmentVariableAsBool(DurableVariable, true);
-    public int MaxQueueSize { get; set; }
+    public int MaxQueueSize { get; init; } = Prefix.GetEnvironmentVariableAsInt(MaxQueueSizeVariable, 0);
-    public int MaxPublishBatchSize { get; set; } = 500;
+    public int MaxPublishBatchSize { get; set; } = Prefix.GetEnvironmentVariableAsInt(MaxPublishBatchSizeVariable, 500);
-    public int PublishIntervalInSeconds { get; set; } = 1000 * 10;
+    public int PublishIntervalInSeconds { get; set; } = Prefix.GetEnvironmentVariableAsInt(PublishIntervalInSecondsVariable, 1000 * 10);
 
     public void Validate()
     {

src/producer/Producer.csproj

@@ -32,15 +32,6 @@
       <None Include="Configuration\logging.json">
         <CopyToOutputDirectory>Always</CopyToOutputDirectory>
       </None>
-      <Content Remove="Configuration\rabbitmq.json" />
-      <None Include="Configuration\rabbitmq.json">
-        <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-      </None>
-      <Content Remove="Configuration\github.json" />
-      <None Include="Configuration\github.json">
-        <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-      </None>
-      <Content Remove="Configuration\postgres.json" />
     </ItemGroup>
 
 </Project>

src/producer/Program.cs

@@ -1,6 +1,4 @@
-using Producer.Extensions;
+var builder = WebApplication.CreateBuilder(args);
-
-var builder = WebApplication.CreateBuilder(args);
 
 builder.Configuration
     .AddScrapeConfiguration();
@@ -9,7 +7,7 @@ builder.Host
     .SetupSerilog(builder.Configuration);
 
 builder.Services
-    .RegisterMassTransit(builder.Configuration)
+    .RegisterMassTransit()
     .AddDataStorage()
     .AddCrawlers()
     .AddQuartz(builder.Configuration);

src/producer/Services/DapperDataStorage.cs

@@ -1,5 +1,3 @@
-using Producer.Models.Configuration;
-
 namespace Producer.Services;
 
 public class DapperDataStorage(PostgresConfiguration configuration, RabbitMqConfiguration rabbitConfig, ILogger<DapperDataStorage> logger) : IDataStorage

src/producer/Services/TorrentPublisher.cs

@@ -1,6 +1,4 @@
-using Producer.Models.Configuration;
+namespace Producer.Services;
-
-namespace Producer.Services;
 
 public class TorrentPublisher(
     ISendEndpointProvider sendEndpointProvider,