Run pre-commit

This commit is contained in:
purple_emily
2024-03-08 14:34:53 +00:00
parent 31e16df720
commit 79409915cf
221 changed files with 525 additions and 526 deletions

View File

@@ -6,8 +6,8 @@ public class JobConfiguration
private const string DownloadImdbDataVariable = "DOWNLOAD_IMDB_DATA_SCHEDULE";
private const string DownloadImdbDataOnceVariable = "DOWNLOAD_IMDB_DATA_ONCE";
private const string InsertBatchSizeVariable = "INSERT_BATCH_SIZE";
public int InsertBatchSize { get; init; } = Prefix.GetEnvironmentVariableAsInt(InsertBatchSizeVariable, 25_000);
public string DownloadImdbCronSchedule { get; init; } = Prefix.GetOptionalEnvironmentVariableAsString(DownloadImdbDataVariable, CronExpressions.EveryHour);
public bool DownloadImdbOnce { get; init; } = Prefix.GetEnvironmentVariableAsBool(DownloadImdbDataOnceVariable);
}
}

View File

@@ -8,13 +8,13 @@ public class MongoConfiguration
private const string DbVariable = "DB";
private const string UsernameVariable = "USER";
private const string PasswordVariable = "PASSWORD";
private string Host { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(HostVariable);
private int Port { get; init; } = Prefix.GetEnvironmentVariableAsInt(PortVariable, 27017);
private string Username { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(UsernameVariable);
private string Password { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(PasswordVariable);
public string DbName { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(DbVariable);
public string ConnectionString => $"mongodb://{Username}:{Password}@{Host}:{Port}/{DbName}?tls=false&directConnection=true&authSource=admin";
}
}

View File

@@ -1,3 +1,3 @@
namespace Metadata.Features.DeleteDownloadedImdbData;
public record DeleteDownloadedImdbDataRequest(string FilePath);
public record DeleteDownloadedImdbDataRequest(string FilePath);

View File

@@ -5,9 +5,9 @@ public class DeleteDownloadedImdbDataRequestHandler(ILogger<DeleteDownloadedImdb
public Task Handle(DeleteDownloadedImdbDataRequest request, CancellationToken _)
{
logger.LogInformation("Deleting file {FilePath}", request.FilePath);
File.Delete(request.FilePath);
logger.LogInformation("File Deleted");
if (configuration.DownloadImdbOnce)
@@ -18,4 +18,4 @@ public class DeleteDownloadedImdbDataRequestHandler(ILogger<DeleteDownloadedImdb
return Task.CompletedTask;
}
}
}

View File

@@ -5,4 +5,4 @@ public class DownloadImdbDataJob(IMessageBus messageBus, JobConfiguration config
public override bool IsScheduelable => !configuration.DownloadImdbOnce && !string.IsNullOrEmpty(configuration.DownloadImdbCronSchedule);
public override string JobName => nameof(DownloadImdbDataJob);
public override async Task Invoke() => await messageBus.SendAsync(new GetImdbDataRequest());
}
}

View File

@@ -1,3 +1,3 @@
namespace Metadata.Features.DownloadImdbData;
public record GetImdbDataRequest;
public record GetImdbDataRequest;

View File

@@ -3,28 +3,28 @@ namespace Metadata.Features.DownloadImdbData;
public class GetImdbDataRequestHandler(IHttpClientFactory clientFactory, ILogger<GetImdbDataRequestHandler> logger)
{
private const string TitleBasicsFileName = "title.basics.tsv";
public async Task<ImportImdbDataRequest> Handle(GetImdbDataRequest _, CancellationToken cancellationToken)
{
logger.LogInformation("Downloading IMDB data");
var client = clientFactory.CreateClient("imdb-data");
var response = await client.GetAsync($"{TitleBasicsFileName}.gz", cancellationToken);
var tempFile = Path.Combine(Path.GetTempPath(), TitleBasicsFileName);
response.EnsureSuccessStatusCode();
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken);
await using var gzipStream = new GZipStream(stream, CompressionMode.Decompress);
await using var fileStream = File.Create(tempFile);
await gzipStream.CopyToAsync(fileStream, cancellationToken);
logger.LogInformation("Downloaded IMDB data to {TempFile}", tempFile);
fileStream.Close();
return new(tempFile);
}
}
}

View File

@@ -12,4 +12,4 @@ public class ImdbEntry
public string? EndYear { get; set; }
public string? RuntimeMinutes { get; set; }
public string? Genres { get; set; }
}
}

View File

@@ -8,7 +8,7 @@ public class ImdbMongoDbService
public ImdbMongoDbService(MongoConfiguration configuration, ILogger<ImdbMongoDbService> logger)
{
_logger = logger;
var client = new MongoClient(configuration.ConnectionString);
var database = client.GetDatabase(configuration.DbName);
@@ -37,7 +37,7 @@ public class ImdbMongoDbService
await _imdbCollection.BulkWriteAsync(operations);
}
public bool IsDatabaseInitialized()
{
try
@@ -61,4 +61,4 @@ public class ImdbMongoDbService
return false;
}
}
}
}

View File

@@ -1,3 +1,3 @@
namespace Metadata.Features.ImportImdbData;
public record ImportImdbDataRequest(string FilePath);
public record ImportImdbDataRequest(string FilePath);

View File

@@ -21,16 +21,16 @@ public class ImportImdbDataRequestHandler(ILogger<ImportImdbDataRequestHandler>
FullMode = BoundedChannelFullMode.Wait,
});
// Skip the header row
await csv.ReadAsync();
var batchInsertTask = CreateBatchOfEntries(channel, cancellationToken);
await ReadEntries(csv, channel, cancellationToken);
channel.Writer.Complete();
await batchInsertTask;
return new(request.FilePath);
@@ -45,7 +45,7 @@ public class ImportImdbDataRequestHandler(ILogger<ImportImdbDataRequestHandler>
{
return;
}
var batch = new List<ImdbEntry>
{
movieData,
@@ -63,7 +63,7 @@ public class ImportImdbDataRequestHandler(ILogger<ImportImdbDataRequestHandler>
}
}
}, cancellationToken);
private static async Task ReadEntries(CsvReader csv, Channel<ImdbEntry, ImdbEntry> channel, CancellationToken cancellationToken)
{
while (await csv.ReadAsync())
@@ -80,13 +80,13 @@ public class ImportImdbDataRequestHandler(ILogger<ImportImdbDataRequestHandler>
RuntimeMinutes = csv.GetField(7),
Genres = csv.GetField(8),
};
if (cancellationToken.IsCancellationRequested)
{
return;
}
await channel.Writer.WriteAsync(movieData, cancellationToken);
}
}
}
}

View File

@@ -5,6 +5,6 @@ public abstract class BaseJob : IMetadataJob
public abstract bool IsScheduelable { get; }
public abstract string JobName { get; }
public abstract Task Invoke();
}
}

View File

@@ -4,4 +4,4 @@ public interface IMetadataJob : IInvocable
{
bool IsScheduelable { get; }
string JobName { get; }
}
}

View File

@@ -5,14 +5,14 @@ public class JobScheduler(IServiceProvider serviceProvider) : IHostedService
public Task StartAsync(CancellationToken cancellationToken)
{
using var scope = serviceProvider.CreateAsyncScope();
var mongoDbService = scope.ServiceProvider.GetRequiredService<ImdbMongoDbService>();
if (!mongoDbService.IsDatabaseInitialized())
{
throw new InvalidOperationException("MongoDb is not initialized");
}
var jobConfigurations = scope.ServiceProvider.GetRequiredService<JobConfiguration>();
var downloadJob = scope.ServiceProvider.GetRequiredService<DownloadImdbDataJob>();
@@ -20,15 +20,15 @@ public class JobScheduler(IServiceProvider serviceProvider) : IHostedService
{
return downloadJob.Invoke();
}
var scheduler = scope.ServiceProvider.GetRequiredService<IScheduler>();
scheduler.Schedule<DownloadImdbDataJob>()
.Cron(jobConfigurations.DownloadImdbCronSchedule)
.PreventOverlapping(nameof(downloadJob.JobName));
return Task.CompletedTask;
}
public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
}
}

View File

@@ -6,4 +6,4 @@ public static class CronExpressions
public const string EveryDay = "0 0 0 * *";
public const string EveryWeek = "0 0 * * 0";
public const string EveryMonth = "0 0 0 * *";
}
}

View File

@@ -4,4 +4,4 @@ public static class HttpClients
{
public const string ImdbDataClientName = "imdb-data";
public const string ImdbClientBaseAddress = "https://datasets.imdbws.com/";
}
}