brock/torrentio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Change postgres configuration in the producer to use the env vars from the stack

Browse Source
This commit is contained in:
iPromKnight
2024-02-04 15:03:07 +00:00
parent c20a92e8f0
commit e461e26b0f
14 changed files with 106 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.env
View File

@@ -28,7 +28,6 @@ TORRENT_TIMEOUT=30000
UDP_TRACKERS_ENABLED=true
# Producer
ScrapeConfiguration__StorageConnectionString=host=postgres;username=postgres;password=postgres;database=knightcrawler;
RabbitMqConfiguration__Host=rabbitmq
RabbitMqConfiguration__QueueName=ingested
RabbitMqConfiguration__Username=guest
@@ -37,4 +36,4 @@ RabbitMqConfiguration__Durable=true
RabbitMqConfiguration__MaxQueueSize=0
RabbitMqConfiguration__MaxPublishBatchSize=500
RabbitMqConfiguration__PublishIntervalInSeconds=10
GithubSettings__PAT=
GithubSettings__PAT=

1
src/producer/Configuration/scrapers.json
View File

@@ -1,6 +1,5 @@
{
"ScrapeConfiguration": {
"StorageConnectionString": "",
"Scrapers": [
{
"Name": "SyncEzTvJob",

2
src/producer/Crawlers/Sites/DebridMediaManagerCrawler.cs
View File

@@ -1,3 +1,5 @@
using Producer.Models.Configuration;
namespace Producer.Crawlers.Sites;
public partial class DebridMediaManagerCrawler(

2
src/producer/Extensions/ConfigurationExtensions.cs
View File

@@ -1,3 +1,5 @@
using Producer.Models.Configuration;
namespace Producer.Extensions;
public static class ConfigurationExtensions

60
src/producer/Extensions/ServiceCollectionExtensions.cs
View File

@@ -1,3 +1,5 @@
using Producer.Models.Configuration;
namespace Producer.Extensions;
public static class ServiceCollectionExtensions
@@ -20,6 +22,7 @@ public static class ServiceCollectionExtensions
internal static IServiceCollection AddDataStorage(this IServiceCollection services)
{
services.LoadConfigurationFromEnv<PostgresConfiguration>();
services.AddTransient<IDataStorage, DapperDataStorage>();
services.AddTransient<IMessagePublisher, TorrentPublisher>();
return services;
@@ -36,9 +39,9 @@ public static class ServiceCollectionExtensions
services.AddMassTransit(busConfigurator =>
{
busConfigurator.SetKebabCaseEndpointNameFormatter();
busConfigurator.UsingRabbitMq((context, busFactoryConfigurator) =>
busConfigurator.UsingRabbitMq((_, busFactoryConfigurator) =>
{
busFactoryConfigurator.Host(rabbitConfig!.Host, hostConfigurator =>
busFactoryConfigurator.Host(rabbitConfig.Host, hostConfigurator =>
{
hostConfigurator.Username(rabbitConfig.Username);
hostConfigurator.Password(rabbitConfig.Password);
@@ -51,9 +54,9 @@ public static class ServiceCollectionExtensions
internal static IServiceCollection AddQuartz(this IServiceCollection services, IConfiguration configuration)
{
var scrapeConfiguration = LoadScrapeConfiguration(services, configuration);
var githubConfiguration = LoadGithubConfiguration(services, configuration);
var rabbitConfig = LoadRabbitMQConfiguration(services, configuration);
var scrapeConfiguration = services.LoadConfigurationFromConfig<ScrapeConfiguration>(configuration, ScrapeConfiguration.SectionName);
var githubConfiguration = services.LoadConfigurationFromConfig<GithubConfiguration>(configuration, GithubConfiguration.SectionName);
var rabbitConfig = services.LoadConfigurationFromConfig<RabbitMqConfiguration>(configuration, RabbitMqConfiguration.SectionName);
services
.AddTransient<SyncEzTvJob>()
@@ -92,46 +95,29 @@ public static class ServiceCollectionExtensions
return services;
}
private static GithubConfiguration LoadGithubConfiguration(IServiceCollection services, IConfiguration configuration)
private static TConfiguration LoadConfigurationFromConfig<TConfiguration>(this IServiceCollection services, IConfiguration configuration, string sectionName)
where TConfiguration : class
{
var githubConfiguration = configuration.GetSection(GithubConfiguration.SectionName).Get<GithubConfiguration>();
var instance = configuration.GetSection(sectionName).Get<TConfiguration>();
ArgumentNullException.ThrowIfNull(githubConfiguration, nameof(githubConfiguration));
services.TryAddSingleton(githubConfiguration);
ArgumentNullException.ThrowIfNull(instance, nameof(instance));
return githubConfiguration;
services.TryAddSingleton(instance);
return instance;
}
private static RabbitMqConfiguration LoadRabbitMQConfiguration(IServiceCollection services, IConfiguration configuration)
private static TConfiguration LoadConfigurationFromEnv<TConfiguration>(this IServiceCollection services)
where TConfiguration : class
{
var rabbitConfiguration = configuration.GetSection(RabbitMqConfiguration.SectionName).Get<RabbitMqConfiguration>();
ArgumentNullException.ThrowIfNull(rabbitConfiguration, nameof(rabbitConfiguration));
if (rabbitConfiguration.MaxQueueSize > 0)
{
if (rabbitConfiguration.MaxPublishBatchSize > rabbitConfiguration.MaxQueueSize)
{
throw new InvalidOperationException("MaxPublishBatchSize cannot be greater than MaxQueueSize in RabbitMqConfiguration");
}
}
services.TryAddSingleton(rabbitConfiguration);
return rabbitConfiguration;
}
private static ScrapeConfiguration LoadScrapeConfiguration(IServiceCollection services, IConfiguration configuration)
{
var scrapeConfiguration = configuration.GetSection(ScrapeConfiguration.SectionName).Get<ScrapeConfiguration>();
var instance = Activator.CreateInstance<TConfiguration>();
ArgumentNullException.ThrowIfNull(scrapeConfiguration, nameof(scrapeConfiguration));
ArgumentNullException.ThrowIfNull(instance, nameof(instance));
services.TryAddSingleton(scrapeConfiguration);
services.TryAddSingleton(instance);
return scrapeConfiguration;
return instance;
}
private static void AddJobWithTrigger<TJobType>(

1
src/producer/GlobalUsings.cs
View File

@@ -2,6 +2,7 @@
global using System.Text;
global using System.Text.Json;
global using System.Text.Json.Serialization;
global using System.Text.RegularExpressions;
global using System.Xml.Linq;
global using Dapper;

2
src/producer/Models/GithubConfiguration.cs → src/producer/Models/Configuration/GithubConfiguration.cs
View File

@@ -1,4 +1,4 @@
namespace Producer.Models;
namespace Producer.Models.Configuration;
public class GithubConfiguration
{

29
src/producer/Models/Configuration/PostgresConfiguration.cs Normal file
View File

@@ -0,0 +1,29 @@
namespace Producer.Models.Configuration;
public class PostgresConfiguration
{
private const string Prefix = "POSTGRES";
private const string HostVariable = "HOST";
private const string UsernameVariable = "USER";
private const string PasswordVariable = "PASSWORD";
private const string DatabaseVariable = "DB";
private const string PortVariable = "PORT";
private string Host { get; init; } = Environment.GetEnvironmentVariable($"{Prefix}_{HostVariable}") ??
throw new InvalidOperationException($"Environment variable {Prefix}_{HostVariable} is not set");
private string Username { get; init; } = Environment.GetEnvironmentVariable($"{Prefix}_{UsernameVariable}") ??
throw new InvalidOperationException($"Environment variable {Prefix}_{UsernameVariable} is not set");
private string Password { get; init; } = Environment.GetEnvironmentVariable($"{Prefix}_{PasswordVariable}") ??
throw new InvalidOperationException($"Environment variable {Prefix}_{PasswordVariable} is not set");
private string Database { get; init; } = Environment.GetEnvironmentVariable($"{Prefix}_{DatabaseVariable}") ??
throw new InvalidOperationException($"Environment variable {Prefix}_{DatabaseVariable} is not set");
private int PORT { get; init; } = int.Parse(
Environment.GetEnvironmentVariable($"{Prefix}_{PortVariable}") ??
throw new InvalidOperationException($"Environment variable {Prefix}_{PortVariable} is not set"));
public string StorageConnectionString => $"Host={Host};Port={PORT};Username={Username};Password={Password};Database={Database};";
}

39
src/producer/Models/Configuration/RabbitMqConfiguration.cs Normal file
View File

@@ -0,0 +1,39 @@
namespace Producer.Models.Configuration;
public class RabbitMqConfiguration
{
public const string SectionName = "RabbitMqConfiguration";
public const string Filename = "rabbitmq.json";
public string? Host { get; set; }
public string? Username { get; set; }
public string? Password { get; set; }
public string? QueueName { get; set; }
public bool Durable { get; set; }
public int MaxQueueSize { get; set; }
public int MaxPublishBatchSize { get; set; } = 500;
public int PublishIntervalInSeconds { get; set; } = 1000 * 10;
public void Validate()
{
if (MaxQueueSize == 0)
{
return;
}
if (MaxQueueSize < 0)
{
throw new InvalidOperationException("MaxQueueSize cannot be less than 0 in RabbitMqConfiguration");
}
if (MaxPublishBatchSize < 0)
{
throw new InvalidOperationException("MaxPublishBatchSize cannot be less than 0 in RabbitMqConfiguration");
}
if (MaxPublishBatchSize > MaxQueueSize)
{
throw new InvalidOperationException("MaxPublishBatchSize cannot be greater than MaxQueueSize in RabbitMqConfiguration");
}
}
}

3
src/producer/Models/ScrapeConfiguration.cs → src/producer/Models/Configuration/ScrapeConfiguration.cs
View File

@@ -1,4 +1,4 @@
namespace Producer.Models;
namespace Producer.Models.Configuration;
public class ScrapeConfiguration
{
@@ -6,5 +6,4 @@ public class ScrapeConfiguration
public const string Filename = "scrapers.json";
public List<Scraper> Scrapers { get; set; } = [];
public string StorageConnectionString { get; set; } = "";
}

16
src/producer/Models/RabbitMqConfiguration.cs
View File

@@ -1,16 +0,0 @@
namespace Producer.Models;
public class RabbitMqConfiguration
{
public const string SectionName = "RabbitMqConfiguration";
public const string Filename = "rabbitmq.json";
public string? Host { get; set; }
public string? Username { get; set; }
public string? Password { get; set; }
public string? QueueName { get; set; }
public bool Durable { get; set; }
public int MaxQueueSize { get; set; }
public int MaxPublishBatchSize { get; set; } = 500;
public int PublishIntervalInSeconds { get; set; } = 1000 * 10;
}

1
src/producer/Producer.csproj
View File

@@ -40,6 +40,7 @@
<None Include="Configuration\github.json">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<Content Remove="Configuration\postgres.json" />
</ItemGroup>
</Project>

4
src/producer/Services/DapperDataStorage.cs
View File

@@ -1,6 +1,8 @@
using Producer.Models.Configuration;
namespace Producer.Services;
public class DapperDataStorage(ScrapeConfiguration configuration, RabbitMqConfiguration rabbitConfig, ILogger<DapperDataStorage> logger) : IDataStorage
public class DapperDataStorage(PostgresConfiguration configuration, RabbitMqConfiguration rabbitConfig, ILogger<DapperDataStorage> logger) : IDataStorage
{
private const string InsertTorrentSql =
"""

4
src/producer/Services/TorrentPublisher.cs
View File

@@ -1,4 +1,6 @@
namespace Producer.Services;
using Producer.Models.Configuration;
namespace Producer.Services;
public class TorrentPublisher(
ISendEndpointProvider sendEndpointProvider,