brock/torrentio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Run pre-commit

Browse Source
This commit is contained in:
purple_emily
2024-03-08 14:34:53 +00:00
parent 31e16df720
commit 79409915cf
221 changed files with 525 additions and 526 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/metadata/Configuration/logging.json
View File

@@ -16,16 +16,16 @@
}
},
"WriteTo": [
{
{
"Name": "Console",
"Args": {
"outputTemplate": "{Timestamp:HH:mm:ss} [{Level}] [{SourceContext}] {Message}{NewLine}{Exception}"
}
}
}
],
"Enrich": [ "FromLogContext", "WithMachineName", "WithThreadId" ],
"Properties": {
"Application": "Metadata"
}
}
}
}

18
src/metadata/Extensions/ConfigurationExtensions.cs
View File

@@ -4,41 +4,41 @@ public static class ConfigurationExtensions
{
private const string ConfigurationFolder = "Configuration";
private const string LoggingConfig = "logging.json";
public static IConfigurationBuilder AddServiceConfiguration(this IConfigurationBuilder configuration)
{
configuration.SetBasePath(Path.Combine(AppContext.BaseDirectory, ConfigurationFolder));
configuration.AddJsonFile(LoggingConfig, false, true);
configuration.AddEnvironmentVariables();
configuration.AddUserSecrets<Program>();
return configuration;
}
public static TConfiguration LoadConfigurationFromConfig<TConfiguration>(this IServiceCollection services, IConfiguration configuration, string sectionName)
where TConfiguration : class
{
var instance = configuration.GetSection(sectionName).Get<TConfiguration>();
ArgumentNullException.ThrowIfNull(instance, nameof(instance));
services.TryAddSingleton(instance);
return instance;
}
public static TConfiguration LoadConfigurationFromEnv<TConfiguration>(this IServiceCollection services)
where TConfiguration : class
{
var instance = Activator.CreateInstance<TConfiguration>();
ArgumentNullException.ThrowIfNull(instance, nameof(instance));
services.TryAddSingleton(instance);
return instance;
}
}
}

20
src/metadata/Extensions/EnvironmentExtensions.cs
View File

@@ -5,9 +5,9 @@ public static class EnvironmentExtensions
public static bool GetEnvironmentVariableAsBool(this string prefix, string varName, bool fallback = false)
{
var fullVarName = GetFullVariableName(prefix, varName);
var str = Environment.GetEnvironmentVariable(fullVarName);
if (string.IsNullOrEmpty(str))
{
return fallback;
@@ -21,11 +21,11 @@ public static class EnvironmentExtensions
_ => false,
};
}
public static int GetEnvironmentVariableAsInt(this string prefix, string varName, int fallback = 0)
{
var fullVarName = GetFullVariableName(prefix, varName);
var str = Environment.GetEnvironmentVariable(fullVarName);
if (string.IsNullOrEmpty(str))
@@ -35,11 +35,11 @@ public static class EnvironmentExtensions
return int.TryParse(str, out var result) ? result : fallback;
}
public static string GetRequiredEnvironmentVariableAsString(this string prefix, string varName)
{
var fullVarName = GetFullVariableName(prefix, varName);
var str = Environment.GetEnvironmentVariable(fullVarName);
if (string.IsNullOrEmpty(str))
@@ -49,11 +49,11 @@ public static class EnvironmentExtensions
return str;
}
public static string GetOptionalEnvironmentVariableAsString(this string prefix, string varName, string? fallback = null)
{
var fullVarName = GetFullVariableName(prefix, varName);
var str = Environment.GetEnvironmentVariable(fullVarName);
if (string.IsNullOrEmpty(str))
@@ -63,6 +63,6 @@ public static class EnvironmentExtensions
return str;
}
private static string GetFullVariableName(string prefix, string varName) => $"{prefix}_{varName}";
}
}

2
src/metadata/Extensions/JsonSerializerExtensions.cs
View File

@@ -3,4 +3,4 @@ namespace Metadata.Extensions;
public static class JsonSerializerExtensions
{
public static string ToJson<T>(this T value) => JsonSerializer.Serialize(value);
}
}

10
src/metadata/Extensions/ServiceCollectionExtensions.cs
View File

@@ -8,15 +8,15 @@ public static class ServiceCollectionExtensions
return services;
}
internal static IServiceCollection AddMongoDb(this IServiceCollection services)
{
services.LoadConfigurationFromEnv<MongoConfiguration>();
services.AddTransient<ImdbMongoDbService>();
return services;
}
internal static IServiceCollection AddJobSupport(this IServiceCollection services)
{
services.LoadConfigurationFromEnv<JobConfiguration>();
@@ -24,7 +24,7 @@ public static class ServiceCollectionExtensions
services.AddScheduler()
.AddTransient<DownloadImdbDataJob>()
.AddHostedService<JobScheduler>();
return services;
}
}
}

4
src/metadata/Extensions/WebApplicationBuilderExtensions.cs
View File

@@ -13,7 +13,7 @@ internal static class WebApplicationBuilderExtensions
{
options.DefaultExecutionTimeout = 6.Hours();
});
return builder;
}
}
}

4
src/metadata/Features/Configuration/JobConfiguration.cs
View File

@@ -6,8 +6,8 @@ public class JobConfiguration
private const string DownloadImdbDataVariable = "DOWNLOAD_IMDB_DATA_SCHEDULE";
private const string DownloadImdbDataOnceVariable = "DOWNLOAD_IMDB_DATA_ONCE";
private const string InsertBatchSizeVariable = "INSERT_BATCH_SIZE";
public int InsertBatchSize { get; init; } = Prefix.GetEnvironmentVariableAsInt(InsertBatchSizeVariable, 25_000);
public string DownloadImdbCronSchedule { get; init; } = Prefix.GetOptionalEnvironmentVariableAsString(DownloadImdbDataVariable, CronExpressions.EveryHour);
public bool DownloadImdbOnce { get; init; } = Prefix.GetEnvironmentVariableAsBool(DownloadImdbDataOnceVariable);
}
}

6
src/metadata/Features/Configuration/MongoConfiguration.cs
View File

@@ -8,13 +8,13 @@ public class MongoConfiguration
private const string DbVariable = "DB";
private const string UsernameVariable = "USER";
private const string PasswordVariable = "PASSWORD";
private string Host { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(HostVariable);
private int Port { get; init; } = Prefix.GetEnvironmentVariableAsInt(PortVariable, 27017);
private string Username { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(UsernameVariable);
private string Password { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(PasswordVariable);
public string DbName { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(DbVariable);
public string ConnectionString => $"mongodb://{Username}:{Password}@{Host}:{Port}/{DbName}?tls=false&directConnection=true&authSource=admin";
}
}

2
src/metadata/Features/DeleteDownloadedImdbData/DeleteDownloadedImdbDataRequest.cs
View File

@@ -1,3 +1,3 @@
namespace Metadata.Features.DeleteDownloadedImdbData;
public record DeleteDownloadedImdbDataRequest(string FilePath);
public record DeleteDownloadedImdbDataRequest(string FilePath);

6
src/metadata/Features/DeleteDownloadedImdbData/DeleteDownloadedImdbDataRequestHandler.cs
View File

@@ -5,9 +5,9 @@ public class DeleteDownloadedImdbDataRequestHandler(ILogger<DeleteDownloadedImdb
public Task Handle(DeleteDownloadedImdbDataRequest request, CancellationToken _)
{
logger.LogInformation("Deleting file {FilePath}", request.FilePath);
File.Delete(request.FilePath);
logger.LogInformation("File Deleted");
if (configuration.DownloadImdbOnce)
@@ -18,4 +18,4 @@ public class DeleteDownloadedImdbDataRequestHandler(ILogger<DeleteDownloadedImdb
return Task.CompletedTask;
}
}
}

2
src/metadata/Features/DownloadImdbData/DownloadImdbDataJob.cs
View File

@@ -5,4 +5,4 @@ public class DownloadImdbDataJob(IMessageBus messageBus, JobConfiguration config
public override bool IsScheduelable => !configuration.DownloadImdbOnce && !string.IsNullOrEmpty(configuration.DownloadImdbCronSchedule);
public override string JobName => nameof(DownloadImdbDataJob);
public override async Task Invoke() => await messageBus.SendAsync(new GetImdbDataRequest());
}
}

2
src/metadata/Features/DownloadImdbData/GetImdbDataRequest.cs
View File

@@ -1,3 +1,3 @@
namespace Metadata.Features.DownloadImdbData;
public record GetImdbDataRequest;
public record GetImdbDataRequest;

18
src/metadata/Features/DownloadImdbData/GetImdbDataRequestHandler.cs
View File

@@ -3,28 +3,28 @@ namespace Metadata.Features.DownloadImdbData;
public class GetImdbDataRequestHandler(IHttpClientFactory clientFactory, ILogger<GetImdbDataRequestHandler> logger)
{
private const string TitleBasicsFileName = "title.basics.tsv";
public async Task<ImportImdbDataRequest> Handle(GetImdbDataRequest _, CancellationToken cancellationToken)
{
logger.LogInformation("Downloading IMDB data");
var client = clientFactory.CreateClient("imdb-data");
var response = await client.GetAsync($"{TitleBasicsFileName}.gz", cancellationToken);
var tempFile = Path.Combine(Path.GetTempPath(), TitleBasicsFileName);
response.EnsureSuccessStatusCode();
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken);
await using var gzipStream = new GZipStream(stream, CompressionMode.Decompress);
await using var fileStream = File.Create(tempFile);
await gzipStream.CopyToAsync(fileStream, cancellationToken);
logger.LogInformation("Downloaded IMDB data to {TempFile}", tempFile);
fileStream.Close();
return new(tempFile);
}
}
}

2
src/metadata/Features/ImportImdbData/ImdbEntry.cs
View File

@@ -12,4 +12,4 @@ public class ImdbEntry
public string? EndYear { get; set; }
public string? RuntimeMinutes { get; set; }
public string? Genres { get; set; }
}
}

6
src/metadata/Features/ImportImdbData/ImdbMongoDbService.cs
View File

@@ -8,7 +8,7 @@ public class ImdbMongoDbService
public ImdbMongoDbService(MongoConfiguration configuration, ILogger<ImdbMongoDbService> logger)
{
_logger = logger;
var client = new MongoClient(configuration.ConnectionString);
var database = client.GetDatabase(configuration.DbName);
@@ -37,7 +37,7 @@ public class ImdbMongoDbService
await _imdbCollection.BulkWriteAsync(operations);
}
public bool IsDatabaseInitialized()
{
try
@@ -61,4 +61,4 @@ public class ImdbMongoDbService
return false;
}
}
}
}

2
src/metadata/Features/ImportImdbData/ImportImdbDataRequest.cs
View File

@@ -1,3 +1,3 @@
namespace Metadata.Features.ImportImdbData;
public record ImportImdbDataRequest(string FilePath);
public record ImportImdbDataRequest(string FilePath);

16
src/metadata/Features/ImportImdbData/ImportImdbDataRequestHandler.cs
View File

@@ -21,16 +21,16 @@ public class ImportImdbDataRequestHandler(ILogger<ImportImdbDataRequestHandler>
FullMode = BoundedChannelFullMode.Wait,
});
// Skip the header row
await csv.ReadAsync();
var batchInsertTask = CreateBatchOfEntries(channel, cancellationToken);
await ReadEntries(csv, channel, cancellationToken);
channel.Writer.Complete();
await batchInsertTask;
return new(request.FilePath);
@@ -45,7 +45,7 @@ public class ImportImdbDataRequestHandler(ILogger<ImportImdbDataRequestHandler>
{
return;
}
var batch = new List<ImdbEntry>
{
movieData,
@@ -63,7 +63,7 @@ public class ImportImdbDataRequestHandler(ILogger<ImportImdbDataRequestHandler>
}
}
}, cancellationToken);
private static async Task ReadEntries(CsvReader csv, Channel<ImdbEntry, ImdbEntry> channel, CancellationToken cancellationToken)
{
while (await csv.ReadAsync())
@@ -80,13 +80,13 @@ public class ImportImdbDataRequestHandler(ILogger<ImportImdbDataRequestHandler>
RuntimeMinutes = csv.GetField(7),
Genres = csv.GetField(8),
};
if (cancellationToken.IsCancellationRequested)
{
return;
}
await channel.Writer.WriteAsync(movieData, cancellationToken);
}
}
}
}

4
src/metadata/Features/Jobs/BaseJob.cs
View File

@@ -5,6 +5,6 @@ public abstract class BaseJob : IMetadataJob
public abstract bool IsScheduelable { get; }
public abstract string JobName { get; }
public abstract Task Invoke();
}
}

2
src/metadata/Features/Jobs/IMetadataJob.cs
View File

@@ -4,4 +4,4 @@ public interface IMetadataJob : IInvocable
{
bool IsScheduelable { get; }
string JobName { get; }
}
}

14
src/metadata/Features/Jobs/JobScheduler.cs
View File

@@ -5,14 +5,14 @@ public class JobScheduler(IServiceProvider serviceProvider) : IHostedService
public Task StartAsync(CancellationToken cancellationToken)
{
using var scope = serviceProvider.CreateAsyncScope();
var mongoDbService = scope.ServiceProvider.GetRequiredService<ImdbMongoDbService>();
if (!mongoDbService.IsDatabaseInitialized())
{
throw new InvalidOperationException("MongoDb is not initialized");
}
var jobConfigurations = scope.ServiceProvider.GetRequiredService<JobConfiguration>();
var downloadJob = scope.ServiceProvider.GetRequiredService<DownloadImdbDataJob>();
@@ -20,15 +20,15 @@ public class JobScheduler(IServiceProvider serviceProvider) : IHostedService
{
return downloadJob.Invoke();
}
var scheduler = scope.ServiceProvider.GetRequiredService<IScheduler>();
scheduler.Schedule<DownloadImdbDataJob>()
.Cron(jobConfigurations.DownloadImdbCronSchedule)
.PreventOverlapping(nameof(downloadJob.JobName));
return Task.CompletedTask;
}
public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
}
}

2
src/metadata/Features/Literals/CronExpressions.cs
View File

@@ -6,4 +6,4 @@ public static class CronExpressions
public const string EveryDay = "0 0 0 * *";
public const string EveryWeek = "0 0 * * 0";
public const string EveryMonth = "0 0 0 * *";
}
}

2
src/metadata/Features/Literals/HttpClients.cs
View File

@@ -4,4 +4,4 @@ public static class HttpClients
{
public const string ImdbDataClientName = "imdb-data";
public const string ImdbClientBaseAddress = "https://datasets.imdbws.com/";
}
}

2
src/metadata/Program.cs
View File

@@ -10,4 +10,4 @@ builder.Services
var host = builder.Build();
await host.RunAsync();
await host.RunAsync();