tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

BREAKING: Cleanup RabbitMQ env vars, and Github Pat

Browse Source
This commit is contained in:
iPromKnight
2024-02-28 12:57:55 +00:00
parent 9f35605e59
commit 1b9a01c677
17 changed files with 136 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/producer/Configuration/github.json
View File

@@ -1,5 +0,0 @@
{
"GithubSettings": {
"PAT": ""
}
}

12
src/producer/Configuration/rabbitmq.json
View File

@@ -1,12 +0,0 @@
{
"RabbitMqConfiguration": {
"Host": "localhost",
"Username": "guest",
"Password": "guest",
"QueueName": "test-queue",
"Durable": true,
"MaxQueueSize": 0,
"MaxPublishBatchSize": 1,
"PublishIntervalInSeconds": 10
}
}

2
src/producer/Crawlers/Sites/DebridMediaManagerCrawler.cs
View File

@@ -1,5 +1,3 @@
using Producer.Models.Configuration;
namespace Producer.Crawlers.Sites;
public partial class DebridMediaManagerCrawler(

4
src/producer/Extensions/ConfigurationExtensions.cs
View File

@@ -1,5 +1,3 @@
using Producer.Models.Configuration;
namespace Producer.Extensions;
public static class ConfigurationExtensions
@@ -13,8 +11,6 @@ public static class ConfigurationExtensions
configuration.AddJsonFile(LoggingConfig, false, true);
configuration.AddJsonFile(ScrapeConfiguration.Filename, false, true);
configuration.AddJsonFile(RabbitMqConfiguration.Filename, false, true);
configuration.AddJsonFile(GithubConfiguration.Filename, false, true);
configuration.AddEnvironmentVariables();

68
src/producer/Extensions/EnvironmentExtensions.cs Normal file
View File

@@ -0,0 +1,68 @@
namespace Producer.Extensions;
public static class EnvironmentExtensions
{
public static bool GetEnvironmentVariableAsBool(this string prefix, string varName, bool fallback = false)
{
var fullVarName = GetFullVariableName(prefix, varName);
var str = Environment.GetEnvironmentVariable(fullVarName);
if (string.IsNullOrEmpty(str))
{
return fallback;
}
return str.Trim().ToLower() switch
{
"true" => true,
"yes" => true,
"1" => true,
_ => false,
};
}
public static int GetEnvironmentVariableAsInt(this string prefix, string varName, int fallback = 0)
{
var fullVarName = GetFullVariableName(prefix, varName);
var str = Environment.GetEnvironmentVariable(fullVarName);
if (string.IsNullOrEmpty(str))
{
return fallback;
}
return int.TryParse(str, out var result) ? result : fallback;
}
public static string GetRequiredEnvironmentVariableAsString(this string prefix, string varName)
{
var fullVarName = GetFullVariableName(prefix, varName);
var str = Environment.GetEnvironmentVariable(fullVarName);
if (string.IsNullOrEmpty(str))
{
throw new InvalidOperationException($"Environment variable {fullVarName} is not set");
}
return str;
}
public static string GetOptionalEnvironmentVariableAsString(this string prefix, string varName, string? fallback = null)
{
var fullVarName = GetFullVariableName(prefix, varName);
var str = Environment.GetEnvironmentVariable(fullVarName);
if (string.IsNullOrEmpty(str))
{
return fallback;
}
return str;
}
private static string GetFullVariableName(string prefix, string varName) => $"{prefix}_{varName}";
}

14
src/producer/Extensions/ServiceCollectionExtensions.cs
View File

@@ -1,5 +1,3 @@
using Producer.Models.Configuration;
namespace Producer.Extensions;
public static class ServiceCollectionExtensions
@@ -29,13 +27,9 @@ public static class ServiceCollectionExtensions
return services;
}
internal static IServiceCollection RegisterMassTransit(this IServiceCollection services, IConfiguration configuration)
internal static IServiceCollection RegisterMassTransit(this IServiceCollection services)
{
var rabbitConfig = configuration.GetSection(RabbitMqConfiguration.SectionName).Get<RabbitMqConfiguration>();
ArgumentNullException.ThrowIfNull(rabbitConfig, nameof(rabbitConfig));
services.AddSingleton(rabbitConfig);
var rabbitConfig = services.LoadConfigurationFromEnv<RabbitMqConfiguration>();
services.AddMassTransit(busConfigurator =>
{
@@ -56,8 +50,8 @@ public static class ServiceCollectionExtensions
internal static IServiceCollection AddQuartz(this IServiceCollection services, IConfiguration configuration)
{
var scrapeConfiguration = services.LoadConfigurationFromConfig<ScrapeConfiguration>(configuration, ScrapeConfiguration.SectionName);
var githubConfiguration = services.LoadConfigurationFromConfig<GithubConfiguration>(configuration, GithubConfiguration.SectionName);
var rabbitConfig = services.LoadConfigurationFromConfig<RabbitMqConfiguration>(configuration, RabbitMqConfiguration.SectionName);
var githubConfiguration = services.LoadConfigurationFromEnv<GithubConfiguration>();
var rabbitConfig = services.LoadConfigurationFromEnv<RabbitMqConfiguration>();
services
.AddTransient<SyncEzTvJob>()

2
src/producer/GlobalUsings.cs
View File

@@ -16,8 +16,10 @@ global using Npgsql;
global using Quartz;
global using Producer.Crawlers;
global using Producer.Crawlers.Sites;
global using Producer.Extensions;
global using Producer.Interfaces;
global using Producer.Jobs;
global using Producer.Models;
global using Producer.Models.Configuration;
global using Producer.Services;
global using Serilog;

6
src/producer/Models/Configuration/GithubConfiguration.cs
View File

@@ -2,8 +2,8 @@
public class GithubConfiguration
{
public const string SectionName = "GithubSettings";
public const string Filename = "github.json";
private const string Prefix = "GITHUB";
private const string PatVariable = "PAT";
public string? PAT { get; set; }
public string? PAT { get; init; } = Prefix.GetOptionalEnvironmentVariableAsString(PatVariable);
}

20
src/producer/Models/Configuration/PostgresConfiguration.cs
View File

@@ -9,21 +9,11 @@ public class PostgresConfiguration
private const string DatabaseVariable = "DB";
private const string PortVariable = "PORT";
private string Host { get; init; } = Environment.GetEnvironmentVariable($"{Prefix}_{HostVariable}") ??
throw new InvalidOperationException($"Environment variable {Prefix}_{HostVariable} is not set");
private string Username { get; init; } = Environment.GetEnvironmentVariable($"{Prefix}_{UsernameVariable}") ??
throw new InvalidOperationException($"Environment variable {Prefix}_{UsernameVariable} is not set");
private string Password { get; init; } = Environment.GetEnvironmentVariable($"{Prefix}_{PasswordVariable}") ??
throw new InvalidOperationException($"Environment variable {Prefix}_{PasswordVariable} is not set");
private string Database { get; init; } = Environment.GetEnvironmentVariable($"{Prefix}_{DatabaseVariable}") ??
throw new InvalidOperationException($"Environment variable {Prefix}_{DatabaseVariable} is not set");
private int PORT { get; init; } = int.Parse(
Environment.GetEnvironmentVariable($"{Prefix}_{PortVariable}") ??
throw new InvalidOperationException($"Environment variable {Prefix}_{PortVariable} is not set"));
private string Host { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(HostVariable);
private string Username { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(UsernameVariable);
private string Password { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(PasswordVariable);
private string Database { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(DatabaseVariable);
private int PORT { get; init; } = Prefix.GetEnvironmentVariableAsInt(PortVariable, 5432);
public string StorageConnectionString => $"Host={Host};Port={PORT};Username={Username};Password={Password};Database={Database};";
}

29
src/producer/Models/Configuration/RabbitMqConfiguration.cs
View File

@@ -1,18 +1,25 @@
namespace Producer.Models.Configuration;
namespace Producer.Models.Configuration;
public class RabbitMqConfiguration
{
public const string SectionName = "RabbitMqConfiguration";
public const string Filename = "rabbitmq.json";
private const string Prefix = "RABBITMQ";
private const string HostVariable = "HOST";
private const string UsernameVariable = "USER";
private const string PasswordVariable = "PASSWORD";
private const string QueueNameVariable = "QUEUE_NAME";
private const string DurableVariable = "DURABLE";
private const string MaxQueueSizeVariable = "MAX_QUEUE_SIZE";
private const string MaxPublishBatchSizeVariable = "MAX_PUBLISH_BATCH_SIZE";
private const string PublishIntervalInSecondsVariable = "PUBLISH_INTERVAL_IN_SECONDS";
public string? Host { get; set; }
public string? Username { get; set; }
public string? Password { get; set; }
public string? QueueName { get; set; }
public bool Durable { get; set; }
public int MaxQueueSize { get; set; }
public int MaxPublishBatchSize { get; set; } = 500;
public int PublishIntervalInSeconds { get; set; } = 1000 * 10;
public string? Username { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(UsernameVariable);
public string? Host { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(HostVariable);
public string? Password { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(PasswordVariable);
public string? QueueName { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(QueueNameVariable);
public bool Durable { get; init; } = Prefix.GetEnvironmentVariableAsBool(DurableVariable, true);
public int MaxQueueSize { get; init; } = Prefix.GetEnvironmentVariableAsInt(MaxQueueSizeVariable, 0);
public int MaxPublishBatchSize { get; set; } = Prefix.GetEnvironmentVariableAsInt(MaxPublishBatchSizeVariable, 500);
public int PublishIntervalInSeconds { get; set; } = Prefix.GetEnvironmentVariableAsInt(PublishIntervalInSecondsVariable, 1000 * 10);
public void Validate()
{

9
src/producer/Producer.csproj
View File

@@ -32,15 +32,6 @@
<None Include="Configuration\logging.json">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<Content Remove="Configuration\rabbitmq.json" />
<None Include="Configuration\rabbitmq.json">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<Content Remove="Configuration\github.json" />
<None Include="Configuration\github.json">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
<Content Remove="Configuration\postgres.json" />
</ItemGroup>
</Project>

6
src/producer/Program.cs
View File

@@ -1,6 +1,4 @@
using Producer.Extensions;
var builder = WebApplication.CreateBuilder(args);
var builder = WebApplication.CreateBuilder(args);
builder.Configuration
.AddScrapeConfiguration();
@@ -9,7 +7,7 @@ builder.Host
.SetupSerilog(builder.Configuration);
builder.Services
.RegisterMassTransit(builder.Configuration)
.RegisterMassTransit()
.AddDataStorage()
.AddCrawlers()
.AddQuartz(builder.Configuration);

2
src/producer/Services/DapperDataStorage.cs
View File

@@ -1,5 +1,3 @@
using Producer.Models.Configuration;
namespace Producer.Services;
public class DapperDataStorage(PostgresConfiguration configuration, RabbitMqConfiguration rabbitConfig, ILogger<DapperDataStorage> logger) : IDataStorage

4
src/producer/Services/TorrentPublisher.cs
View File

@@ -1,6 +1,4 @@
using Producer.Models.Configuration;
namespace Producer.Services;
namespace Producer.Services;
public class TorrentPublisher(
ISendEndpointProvider sendEndpointProvider,