mirror of
https://github.com/knightcrawler-stremio/knightcrawler.git
synced 2024-12-20 03:29:51 +00:00
Run pre-commit
This commit is contained in:
@@ -6,8 +6,8 @@ public class JobConfiguration
|
||||
private const string DownloadImdbDataVariable = "DOWNLOAD_IMDB_DATA_SCHEDULE";
|
||||
private const string DownloadImdbDataOnceVariable = "DOWNLOAD_IMDB_DATA_ONCE";
|
||||
private const string InsertBatchSizeVariable = "INSERT_BATCH_SIZE";
|
||||
|
||||
|
||||
public int InsertBatchSize { get; init; } = Prefix.GetEnvironmentVariableAsInt(InsertBatchSizeVariable, 25_000);
|
||||
public string DownloadImdbCronSchedule { get; init; } = Prefix.GetOptionalEnvironmentVariableAsString(DownloadImdbDataVariable, CronExpressions.EveryHour);
|
||||
public bool DownloadImdbOnce { get; init; } = Prefix.GetEnvironmentVariableAsBool(DownloadImdbDataOnceVariable);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,13 +8,13 @@ public class MongoConfiguration
|
||||
private const string DbVariable = "DB";
|
||||
private const string UsernameVariable = "USER";
|
||||
private const string PasswordVariable = "PASSWORD";
|
||||
|
||||
|
||||
|
||||
private string Host { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(HostVariable);
|
||||
private int Port { get; init; } = Prefix.GetEnvironmentVariableAsInt(PortVariable, 27017);
|
||||
private string Username { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(UsernameVariable);
|
||||
private string Password { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(PasswordVariable);
|
||||
public string DbName { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(DbVariable);
|
||||
|
||||
|
||||
public string ConnectionString => $"mongodb://{Username}:{Password}@{Host}:{Port}/{DbName}?tls=false&directConnection=true&authSource=admin";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
namespace Metadata.Features.DeleteDownloadedImdbData;
|
||||
|
||||
public record DeleteDownloadedImdbDataRequest(string FilePath);
|
||||
public record DeleteDownloadedImdbDataRequest(string FilePath);
|
||||
|
||||
@@ -5,9 +5,9 @@ public class DeleteDownloadedImdbDataRequestHandler(ILogger<DeleteDownloadedImdb
|
||||
public Task Handle(DeleteDownloadedImdbDataRequest request, CancellationToken _)
|
||||
{
|
||||
logger.LogInformation("Deleting file {FilePath}", request.FilePath);
|
||||
|
||||
|
||||
File.Delete(request.FilePath);
|
||||
|
||||
|
||||
logger.LogInformation("File Deleted");
|
||||
|
||||
if (configuration.DownloadImdbOnce)
|
||||
@@ -18,4 +18,4 @@ public class DeleteDownloadedImdbDataRequestHandler(ILogger<DeleteDownloadedImdb
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,4 +5,4 @@ public class DownloadImdbDataJob(IMessageBus messageBus, JobConfiguration config
|
||||
public override bool IsScheduelable => !configuration.DownloadImdbOnce && !string.IsNullOrEmpty(configuration.DownloadImdbCronSchedule);
|
||||
public override string JobName => nameof(DownloadImdbDataJob);
|
||||
public override async Task Invoke() => await messageBus.SendAsync(new GetImdbDataRequest());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
namespace Metadata.Features.DownloadImdbData;
|
||||
|
||||
public record GetImdbDataRequest;
|
||||
public record GetImdbDataRequest;
|
||||
|
||||
@@ -3,28 +3,28 @@ namespace Metadata.Features.DownloadImdbData;
|
||||
public class GetImdbDataRequestHandler(IHttpClientFactory clientFactory, ILogger<GetImdbDataRequestHandler> logger)
|
||||
{
|
||||
private const string TitleBasicsFileName = "title.basics.tsv";
|
||||
|
||||
|
||||
public async Task<ImportImdbDataRequest> Handle(GetImdbDataRequest _, CancellationToken cancellationToken)
|
||||
{
|
||||
logger.LogInformation("Downloading IMDB data");
|
||||
|
||||
|
||||
var client = clientFactory.CreateClient("imdb-data");
|
||||
var response = await client.GetAsync($"{TitleBasicsFileName}.gz", cancellationToken);
|
||||
|
||||
|
||||
var tempFile = Path.Combine(Path.GetTempPath(), TitleBasicsFileName);
|
||||
|
||||
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
|
||||
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken);
|
||||
await using var gzipStream = new GZipStream(stream, CompressionMode.Decompress);
|
||||
await using var fileStream = File.Create(tempFile);
|
||||
|
||||
|
||||
await gzipStream.CopyToAsync(fileStream, cancellationToken);
|
||||
|
||||
|
||||
logger.LogInformation("Downloaded IMDB data to {TempFile}", tempFile);
|
||||
|
||||
|
||||
fileStream.Close();
|
||||
|
||||
return new(tempFile);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,4 +12,4 @@ public class ImdbEntry
|
||||
public string? EndYear { get; set; }
|
||||
public string? RuntimeMinutes { get; set; }
|
||||
public string? Genres { get; set; }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@ public class ImdbMongoDbService
|
||||
public ImdbMongoDbService(MongoConfiguration configuration, ILogger<ImdbMongoDbService> logger)
|
||||
{
|
||||
_logger = logger;
|
||||
|
||||
|
||||
var client = new MongoClient(configuration.ConnectionString);
|
||||
var database = client.GetDatabase(configuration.DbName);
|
||||
|
||||
@@ -37,7 +37,7 @@ public class ImdbMongoDbService
|
||||
|
||||
await _imdbCollection.BulkWriteAsync(operations);
|
||||
}
|
||||
|
||||
|
||||
public bool IsDatabaseInitialized()
|
||||
{
|
||||
try
|
||||
@@ -61,4 +61,4 @@ public class ImdbMongoDbService
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
namespace Metadata.Features.ImportImdbData;
|
||||
|
||||
public record ImportImdbDataRequest(string FilePath);
|
||||
public record ImportImdbDataRequest(string FilePath);
|
||||
|
||||
@@ -21,16 +21,16 @@ public class ImportImdbDataRequestHandler(ILogger<ImportImdbDataRequestHandler>
|
||||
FullMode = BoundedChannelFullMode.Wait,
|
||||
});
|
||||
|
||||
|
||||
|
||||
// Skip the header row
|
||||
await csv.ReadAsync();
|
||||
|
||||
|
||||
var batchInsertTask = CreateBatchOfEntries(channel, cancellationToken);
|
||||
|
||||
await ReadEntries(csv, channel, cancellationToken);
|
||||
|
||||
channel.Writer.Complete();
|
||||
|
||||
|
||||
await batchInsertTask;
|
||||
|
||||
return new(request.FilePath);
|
||||
@@ -45,7 +45,7 @@ public class ImportImdbDataRequestHandler(ILogger<ImportImdbDataRequestHandler>
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
var batch = new List<ImdbEntry>
|
||||
{
|
||||
movieData,
|
||||
@@ -63,7 +63,7 @@ public class ImportImdbDataRequestHandler(ILogger<ImportImdbDataRequestHandler>
|
||||
}
|
||||
}
|
||||
}, cancellationToken);
|
||||
|
||||
|
||||
private static async Task ReadEntries(CsvReader csv, Channel<ImdbEntry, ImdbEntry> channel, CancellationToken cancellationToken)
|
||||
{
|
||||
while (await csv.ReadAsync())
|
||||
@@ -80,13 +80,13 @@ public class ImportImdbDataRequestHandler(ILogger<ImportImdbDataRequestHandler>
|
||||
RuntimeMinutes = csv.GetField(7),
|
||||
Genres = csv.GetField(8),
|
||||
};
|
||||
|
||||
|
||||
if (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
await channel.Writer.WriteAsync(movieData, cancellationToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,6 @@ public abstract class BaseJob : IMetadataJob
|
||||
public abstract bool IsScheduelable { get; }
|
||||
|
||||
public abstract string JobName { get; }
|
||||
|
||||
|
||||
public abstract Task Invoke();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,4 +4,4 @@ public interface IMetadataJob : IInvocable
|
||||
{
|
||||
bool IsScheduelable { get; }
|
||||
string JobName { get; }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,14 +5,14 @@ public class JobScheduler(IServiceProvider serviceProvider) : IHostedService
|
||||
public Task StartAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
using var scope = serviceProvider.CreateAsyncScope();
|
||||
|
||||
|
||||
var mongoDbService = scope.ServiceProvider.GetRequiredService<ImdbMongoDbService>();
|
||||
|
||||
|
||||
if (!mongoDbService.IsDatabaseInitialized())
|
||||
{
|
||||
throw new InvalidOperationException("MongoDb is not initialized");
|
||||
}
|
||||
|
||||
|
||||
var jobConfigurations = scope.ServiceProvider.GetRequiredService<JobConfiguration>();
|
||||
var downloadJob = scope.ServiceProvider.GetRequiredService<DownloadImdbDataJob>();
|
||||
|
||||
@@ -20,15 +20,15 @@ public class JobScheduler(IServiceProvider serviceProvider) : IHostedService
|
||||
{
|
||||
return downloadJob.Invoke();
|
||||
}
|
||||
|
||||
|
||||
var scheduler = scope.ServiceProvider.GetRequiredService<IScheduler>();
|
||||
|
||||
|
||||
scheduler.Schedule<DownloadImdbDataJob>()
|
||||
.Cron(jobConfigurations.DownloadImdbCronSchedule)
|
||||
.PreventOverlapping(nameof(downloadJob.JobName));
|
||||
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,4 +6,4 @@ public static class CronExpressions
|
||||
public const string EveryDay = "0 0 0 * *";
|
||||
public const string EveryWeek = "0 0 * * 0";
|
||||
public const string EveryMonth = "0 0 0 * *";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,4 +4,4 @@ public static class HttpClients
|
||||
{
|
||||
public const string ImdbDataClientName = "imdb-data";
|
||||
public const string ImdbClientBaseAddress = "https://datasets.imdbws.com/";
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user