Producer / Consumer / Collector rewrite (#160)

* Converted metadata service to redis

* move to postgres instead

* fix global usings

* [skip ci] optimize wolverine by prebuilding static types

* [skip ci] Stop indexing mac folder indexes

* [skip ci] producer, metadata and migrations

removed mongodb
added redis cache
imdb meta in postgres
Enable pgtrm
Create trigrams index
Add search meta postgres function

* [skip ci] get rid of node folder, replace mongo with redis in consumer

also wire up postgres metadata searches

* [skip ci] change mongo to redis in the addon

* [skip ci] jackettio to redis

* Rest of mongo removed...

* Cleaner rerunning of metadata - without conflicts

* Add akas import as well as basic metadata

* Include episodes file too

* cascade truncate pre-import

* reverse order to avoid cascadeing

* separate out clean to separate handler

* Switch producer to use metadata matching pre-preocessing dmm

* More work

* Still porting PTN

* PTN port, adding tests

* [skip ci] Codec tests

* [skip ci] Complete Collection handler tests

* [skip ci] container tests

* [skip ci] Convert handlers tests

* [skip ci] DateHandler tests

* [skip ci] Dual Audio matching tests

* [skip ci] episode code tests

* [skip ci] Extended handler tests

* [skip ci] group handler tests

* [skip ci] some broken stuff right now

* [skip ci] more ptn

* [skip ci] PTN now in a separate nuget package, rebased this on the redis changes - i need them.

* [skip ci] Wire up PTN port. Tired - will test tomorrow

* [skip ci] Needs a lot of work - too many titles being missed now

* cleaner. done?

* Handle the date in the imdb search

- add integer function to confirm its a valid integer
- use the input date as a range of -+1 year

* [skip ci] Start of collector service for RD

[skip ci] WIP

Implemented metadata saga, along with channels to process up to a maximum of 100 infohashes each time
The saga will rety for each infohas by requeuing up to three times, before just marking as complete for that infoHash - meaning no data will be updated in the db for that torrent.

[skip ci] Ready to test with queue publishing

Will provision a fanout exchange if it doesn't exist, and create and bind a queue to it. Listens to the queue with 50 prefetch count.
Still needs PTN rewrite bringing in to parse the filename response from real debrid, and extract season and episode numbers if the file is a tvshow

[skip ci] Add Debrid Collector Build Job

Debrid Collector ready for testing

New consumer, new collector, producer has meta lookup and anti porn measures

[skip ci] WIP - moving from wolverine to MassTransit.

 not happy that wolverine cannot effectively control saga concurrency. we need to really.

[skip ci] Producer and new Consumer moved to MassTransit

Just the debrid collector to go now, then to write the optional qbit collector.

Collector now switched to mass transit too

hide porn titles in logs, clean up cache name in redis for imdb titles

[skip ci] Allow control of queues

[skip ci] Update deployment

Remove old consumer, fix deployment files, fix dockerfiles for shared project import

fix base deployment

* Add collector missing env var

* edits to kick off builds

* Add optional qbit deployment which qbit collector will use

* Qbit collector done

* reorder compose, and bring both qbit and qbitcollector into the compose, with 0 replicas as default

* Clean up compose file

* Ensure debrid collector errors if no debrid api key

src/shared/Configuration/PostgresConfiguration.cs

@@ -0,0 +1,19 @@
+namespace SharedContracts.Configuration;
+
+public class PostgresConfiguration
+{
+    private const string Prefix = "POSTGRES";
+    private const string HostVariable = "HOST";
+    private const string UsernameVariable = "USER";
+    private const string PasswordVariable = "PASSWORD";
+    private const string DatabaseVariable = "DB";
+    private const string PortVariable = "PORT";
+
+    private string Host { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(HostVariable);
+    private string Username { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(UsernameVariable);
+    private string Password { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(PasswordVariable);
+    private string Database { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(DatabaseVariable);
+    private int PORT { get; init; } = Prefix.GetEnvironmentVariableAsInt(PortVariable, 5432);
+
+    public string StorageConnectionString => $"Host={Host};Port={PORT};Username={Username};Password={Password};Database={Database};";
+}

src/shared/Configuration/RabbitMqConfiguration.cs

@@ -0,0 +1,54 @@
+namespace SharedContracts.Configuration;
+
+public class RabbitMqConfiguration
+{
+    private const string Prefix = "RABBITMQ";
+    private const string HostVariable = "HOST";
+    private const string UsernameVariable = "USER";
+    private const string PasswordVariable = "PASSWORD";
+    private const string QueueNameVariable = "CONSUMER_QUEUE_NAME";
+    private const string DurableVariable = "DURABLE";
+    private const string MaxQueueSizeVariable = "MAX_QUEUE_SIZE";
+    private const string MaxPublishBatchSizeVariable = "MAX_PUBLISH_BATCH_SIZE";
+    private const string PublishIntervalInSecondsVariable = "PUBLISH_INTERVAL_IN_SECONDS";
+    private const string CollectorPrefix = "COLLECTOR";
+    private const string DebridEnabledVariable = "DEBRID_ENABLED";
+    private const string QbitEnabledVariable = "QBIT_ENABLED";
+    
+    public string? Username { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(UsernameVariable);
+    public string? Host { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(HostVariable);
+    public string? CollectorsExchange { get; init; } = "collectors";
+    public string? DebridCollectorQueueName { get; init; } = "debrid-collector";
+    public string? QbitCollectorQueueName { get; init; } = "qbit-collector";
+    public bool DebridEnabled { get; init; } = CollectorPrefix.GetEnvironmentVariableAsBool(DebridEnabledVariable, true);
+    public bool QbitEnabled { get; init; } = CollectorPrefix.GetEnvironmentVariableAsBool(QbitEnabledVariable, false);
+    public string? Password { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(PasswordVariable);
+    public string? QueueName { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(QueueNameVariable);
+    public bool Durable { get; init; } = Prefix.GetEnvironmentVariableAsBool(DurableVariable, true);
+    public int MaxQueueSize { get; init; } = Prefix.GetEnvironmentVariableAsInt(MaxQueueSizeVariable);
+    public int MaxPublishBatchSize { get; set; } = Prefix.GetEnvironmentVariableAsInt(MaxPublishBatchSizeVariable, 500);
+    public int PublishIntervalInSeconds { get; set; } = Prefix.GetEnvironmentVariableAsInt(PublishIntervalInSecondsVariable, 1000 * 10);
+
+    public void Validate()
+    {
+        if (MaxQueueSize == 0)
+        {
+            return;
+        }
+
+        if (MaxQueueSize < 0)
+        {
+            throw new InvalidOperationException("MaxQueueSize cannot be less than 0 in RabbitMqConfiguration");
+        }
+
+        if (MaxPublishBatchSize < 0)
+        {
+            throw new InvalidOperationException("MaxPublishBatchSize cannot be less than 0 in RabbitMqConfiguration");
+        }
+
+        if (MaxPublishBatchSize > MaxQueueSize)
+        {
+            throw new InvalidOperationException("MaxPublishBatchSize cannot be greater than MaxQueueSize in RabbitMqConfiguration");
+        }
+    }
+}

src/shared/Configuration/RedisConfiguration.cs

@@ -0,0 +1,9 @@
+namespace SharedContracts.Configuration;
+
+public class RedisConfiguration
+{
+    private const string Prefix = "REDIS";
+    private const string ConnectionStringVariable = "CONNECTION_STRING";
+    
+    public string? ConnectionString { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(ConnectionStringVariable) + ",abortConnect=false,allowAdmin=true";
+}

src/shared/Dapper/BaseDapperStorage.cs

@@ -0,0 +1,53 @@
+namespace SharedContracts.Dapper;
+
+public abstract class BaseDapperStorage(ILogger<IDataStorage> logger, PostgresConfiguration configuration)
+{
+    protected async Task ExecuteCommandAsync(Func<NpgsqlConnection, Task> operation, string errorMessage, CancellationToken cancellationToken = default)
+    {
+        try
+        {
+            await using var connection = new NpgsqlConnection(configuration.StorageConnectionString);
+            await connection.OpenAsync(cancellationToken);
+
+            await operation(connection);
+        }
+        catch (Exception e)
+        {
+            logger.LogError(e, errorMessage);
+        }
+    }
+    
+    protected async Task<TResult> ExecuteCommandAsync<TResult>(Func<NpgsqlConnection, Task<TResult>> operation, string errorMessage, CancellationToken cancellationToken = default)
+    {
+        try
+        {
+            await using var connection = new NpgsqlConnection(configuration.StorageConnectionString);
+            await connection.OpenAsync(cancellationToken);
+
+            var result = await operation(connection);
+            return result;
+        }
+        catch (Exception e)
+        {
+            logger.LogError(e, errorMessage);
+            throw;
+        }
+    }
+    
+    protected async Task<DapperResult<TResult, TFailure>> ExecuteCommandAsync<TResult, TFailure>(Func<NpgsqlConnection, Task<TResult>> operation, Func<Exception, TFailure> createFailureResult, CancellationToken cancellationToken = default)
+    {
+        try
+        {
+            await using var connection = new NpgsqlConnection(configuration.StorageConnectionString);
+            await connection.OpenAsync(cancellationToken);
+
+            var result = await operation(connection);
+            return DapperResult<TResult, TFailure>.Ok(result);
+        }
+        catch (Exception e)
+        {
+            var failureResult = createFailureResult(e);
+            return DapperResult<TResult, TFailure>.Fail(failureResult);
+        }
+    }
+}

src/shared/Dapper/DapperDataStorage.cs

@@ -0,0 +1,189 @@
+namespace SharedContracts.Dapper;
+
+public class DapperDataStorage(PostgresConfiguration configuration, RabbitMqConfiguration rabbitConfig, ILogger<DapperDataStorage> logger) : 
+    BaseDapperStorage(logger, configuration), IDataStorage
+{
+    public async Task<DapperResult<InsertTorrentResult, InsertTorrentResult>> InsertTorrents(IReadOnlyCollection<IngestedTorrent> torrents, CancellationToken cancellationToken = default) =>
+        await ExecuteCommandAsync(async connection =>
+        {
+            const string query = 
+                """
+                INSERT INTO ingested_torrents
+                    ("name", "source", "category", "info_hash", "size", "seeders", "leechers", "imdb", "processed", "createdAt", "updatedAt")
+                VALUES
+                    (@Name, @Source, @Category, @InfoHash, @Size, @Seeders, @Leechers, @Imdb, @Processed, @CreatedAt, @UpdatedAt)
+                ON CONFLICT (source, info_hash) DO NOTHING
+                """;
+            
+            var inserted = await connection.ExecuteAsync(query, torrents);
+            return new InsertTorrentResult(true, inserted);
+        }, _ => new InsertTorrentResult(false, 0, "Failed to insert torrents."), cancellationToken);
+
+    public async Task<DapperResult<List<IngestedTorrent>, List<IngestedTorrent>>> GetPublishableTorrents(CancellationToken cancellationToken = default) =>
+        await ExecuteCommandAsync(async connection =>
+        {
+            const string query =
+                """
+                SELECT
+                    "id" as "Id",
+                    "name" as "Name",
+                    "source" as "Source",
+                    "category" as "Category",
+                    "info_hash" as "InfoHash",
+                    "size" as "Size",
+                    "seeders" as "Seeders",
+                    "leechers" as "Leechers",
+                    "imdb" as "Imdb",
+                    "processed" as "Processed",
+                    "createdAt" as "CreatedAt",
+                    "updatedAt" as "UpdatedAt"
+                FROM ingested_torrents
+                WHERE processed = false AND category != 'xxx'
+                """;
+            
+            var torrents = await connection.QueryAsync<IngestedTorrent>(query);
+            return torrents.Take(rabbitConfig.MaxPublishBatchSize).ToList();
+        }, _ => new List<IngestedTorrent>(), cancellationToken);
+
+    public async Task<DapperResult<UpdatedTorrentResult, UpdatedTorrentResult>> SetTorrentsProcessed(IReadOnlyCollection<IngestedTorrent> torrents, CancellationToken cancellationToken = default) =>
+        await ExecuteCommandAsync(async connection =>
+        {
+            foreach (var torrent in torrents)
+            {
+                torrent.UpdatedAt = DateTime.UtcNow;
+            }
+
+            const string query =
+                """
+                UPDATE ingested_torrents
+                Set
+                    processed = true,
+                    "updatedAt" = @UpdatedAt
+                WHERE id = @Id
+                """;
+            
+            var updated = await connection.ExecuteAsync(query, torrents);
+            return new UpdatedTorrentResult(true, updated);
+        }, _ => new UpdatedTorrentResult(false, 0, "Failed to mark torrents as processed"), cancellationToken);
+
+    public async Task<bool> PageIngested(string pageId, CancellationToken cancellationToken = default) =>
+        await ExecuteCommandAsync(async connection =>
+        {
+            const string query = "SELECT EXISTS (SELECT 1 FROM ingested_pages WHERE url = @Url)";
+            return await connection.ExecuteScalarAsync<bool>(query, new { Url = pageId });
+        }, "Failed to check if page is ingested", cancellationToken);
+
+    public async Task<DapperResult<PageIngestedResult, PageIngestedResult>> MarkPageAsIngested(string pageId, CancellationToken cancellationToken = default) =>
+        await ExecuteCommandAsync(async connection =>
+        {
+            var date = DateTime.UtcNow;
+
+            const string query =
+                """
+                INSERT INTO ingested_pages
+                    (url, "createdAt", "updatedAt")
+                VALUES
+                    (@Url, @CreatedAt, @UpdatedAt)
+                """;
+            
+            await connection.ExecuteAsync(query, new
+            {
+                Url = pageId,
+                CreatedAt = date,
+                UpdatedAt = date,
+            });
+
+            return new PageIngestedResult(true, "Page successfully marked as ingested");
+            
+        }, _ => new PageIngestedResult(false, "Page successfully marked as ingested"), cancellationToken);
+
+    public async Task<DapperResult<int, int>> GetRowCountImdbMetadata(CancellationToken cancellationToken = default) =>
+        await ExecuteCommandAsync(async connection =>
+        {
+            const string query = "SELECT COUNT(*) FROM imdb_metadata";
+
+            var result = await connection.ExecuteScalarAsync<int>(query);
+
+            return result;
+        }, _ => 0, cancellationToken);
+
+    public async Task<List<ImdbEntry>> GetImdbEntriesForRequests(int year, int batchSize, string? stateLastProcessedImdbId, CancellationToken cancellationToken = default) =>
+        await ExecuteCommandAsync(async connection =>
+        {
+            const string query = @"SELECT imdb_id AS ImdbId, title as Title, category as Category, year as Year, adult as Adult FROM imdb_metadata WHERE CAST(NULLIF(Year, '\N') AS INTEGER) <= @Year AND imdb_id > @LastProcessedImdbId ORDER BY ImdbId LIMIT @BatchSize";
+            var result = await connection.QueryAsync<ImdbEntry>(query, new { Year = year, LastProcessedImdbId = stateLastProcessedImdbId, BatchSize = batchSize });
+            return result.ToList();
+        }, "Error getting imdb metadata.", cancellationToken);
+
+    public async Task<List<ImdbEntry>> FindImdbMetadata(string? parsedTorrentTitle, TorrentType torrentType, string? year, CancellationToken cancellationToken = default) =>
+        await ExecuteCommandAsync(async connection =>
+        {
+            var query = $"select \"imdb_id\" as \"ImdbId\", \"title\" as \"Title\", \"year\" as \"Year\" from search_imdb_meta('{parsedTorrentTitle.Replace("'", "").Replace("\"", "")}', '{(torrentType == TorrentType.Movie ? "movie" : "tvSeries")}'";
+            query += year is not null ? $", '{year}'" : ", NULL";
+            query += ", 15)";
+            
+            var result = await connection.QueryAsync<ImdbEntry>(query);
+
+            return result.ToList();
+        }, "Error finding imdb metadata.", cancellationToken);
+    
+    public Task InsertTorrent(Torrent torrent, CancellationToken cancellationToken = default) =>
+        ExecuteCommandAsync(
+            async connection =>
+            {
+                const string query = 
+                    """
+                    INSERT INTO "torrents"
+                        ("infoHash", "provider", "torrentId", "title", "size", "type", "uploadDate", "seeders", "trackers", "languages", "resolution", "reviewed", "opened", "createdAt", "updatedAt")
+                    VALUES
+                        (@InfoHash, @Provider, @TorrentId, @Title, 0, @Type, NOW(), @Seeders, NULL, NULL, NULL, false, false, NOW(), NOW())
+                    ON CONFLICT ("infoHash") DO NOTHING
+                    """;
+                
+                await connection.ExecuteAsync(query, torrent);
+            }, "Failed to insert torrent files into database", cancellationToken);
+    
+    public Task InsertFiles(IEnumerable<TorrentFile> files, CancellationToken cancellationToken = default) =>
+        ExecuteCommandAsync(
+            async connection =>
+            {
+                const string query = 
+                    """
+                    INSERT INTO files
+                        ("infoHash", "fileIndex", title, "size", "imdbId", "imdbSeason", "imdbEpisode", "kitsuId", "kitsuEpisode", "createdAt", "updatedAt")
+                    VALUES
+                        (@InfoHash, @FileIndex, @Title, @Size, @ImdbId, @ImdbSeason, @ImdbEpisode, @KitsuId, @KitsuEpisode, Now(), Now());
+                    """;
+                
+                await connection.ExecuteAsync(query, files);
+            }, "Failed to insert torrent files into database", cancellationToken);
+    
+    public Task InsertSubtitles(IEnumerable<SubtitleFile> subtitles, CancellationToken cancellationToken = default) =>
+        ExecuteCommandAsync(
+            async connection =>
+            {
+                const string query = 
+                    """
+                    INSERT INTO subtitles
+                        ("infoHash", "fileIndex", "fileId", "title")
+                    VALUES
+                        (@InfoHash, @FileIndex, @FileId, @Title)
+                    ON CONFLICT
+                        ("infoHash", "fileIndex")
+                    DO UPDATE SET
+                        "fileId" = COALESCE(subtitles."fileId", EXCLUDED."fileId"),
+                        "title" = COALESCE(subtitles."title", EXCLUDED."title");
+                    """;
+                
+                await connection.ExecuteAsync(query, subtitles);
+            }, "Failed to insert subtitles into database", cancellationToken);
+
+    public Task<List<TorrentFile>> GetTorrentFiles(string infoHash, CancellationToken cancellationToken = default) =>
+        ExecuteCommandAsync(
+            async connection =>
+            {
+                const string query = "SELECT * FROM files WHERE LOWER(\"infoHash\") = @InfoHash";
+                var files = await connection.QueryAsync<TorrentFile>(query, new { InfoHash = infoHash });
+                return files.ToList();
+            }, "Failed to insert subtitles into database", cancellationToken);
+}

src/shared/Dapper/DapperResult.cs

@@ -0,0 +1,21 @@
+namespace SharedContracts.Dapper;
+
+public class DapperResult<TSuccess, TFailure>
+{
+    public TSuccess Success { get; }
+    public TFailure Failure { get; }
+    public bool IsSuccess { get; }
+
+    private DapperResult(TSuccess success, TFailure failure, bool isSuccess)
+    {
+        Success = success;
+        Failure = failure;
+        IsSuccess = isSuccess;
+    }
+
+    public static DapperResult<TSuccess, TFailure> Ok(TSuccess success) => 
+        new(success, default, true);
+
+    public static DapperResult<TSuccess, TFailure> Fail(TFailure failure) => 
+        new(default, failure, false);
+}

src/shared/Dapper/IDataStorage.cs

@@ -0,0 +1,17 @@
+namespace SharedContracts.Dapper;
+
+public interface IDataStorage
+{
+    Task<DapperResult<InsertTorrentResult, InsertTorrentResult>> InsertTorrents(IReadOnlyCollection<IngestedTorrent> torrents, CancellationToken cancellationToken = default);
+    Task<DapperResult<List<IngestedTorrent>, List<IngestedTorrent>>> GetPublishableTorrents(CancellationToken cancellationToken = default);
+    Task<DapperResult<UpdatedTorrentResult, UpdatedTorrentResult>> SetTorrentsProcessed(IReadOnlyCollection<IngestedTorrent> torrents, CancellationToken cancellationToken = default);
+    Task<bool> PageIngested(string pageId, CancellationToken cancellationToken = default);
+    Task<DapperResult<PageIngestedResult, PageIngestedResult>> MarkPageAsIngested(string pageId, CancellationToken cancellationToken = default);
+    Task<DapperResult<int, int>> GetRowCountImdbMetadata(CancellationToken cancellationToken = default);
+    Task<List<ImdbEntry>> GetImdbEntriesForRequests(int year, int batchSize, string? stateLastProcessedImdbId, CancellationToken cancellationToken = default);
+    Task<List<ImdbEntry>> FindImdbMetadata(string? parsedTorrentTitle, TorrentType parsedTorrentTorrentType, string? parsedTorrentYear, CancellationToken cancellationToken = default);
+    Task InsertTorrent(Torrent torrent, CancellationToken cancellationToken = default);
+    Task InsertFiles(IEnumerable<TorrentFile> files, CancellationToken cancellationToken = default);
+    Task InsertSubtitles(IEnumerable<SubtitleFile> subtitles, CancellationToken cancellationToken = default);
+    Task<List<TorrentFile>> GetTorrentFiles(string infoHash, CancellationToken cancellationToken = default);
+}

src/shared/Dapper/Results.cs

@@ -0,0 +1,5 @@
+namespace SharedContracts.Dapper;
+
+public record InsertTorrentResult(bool Success, int InsertedCount = 0, string? ErrorMessage = null);
+public record UpdatedTorrentResult(bool Success, int UpdatedCount = 0, string? ErrorMessage = null);
+public record PageIngestedResult(bool Success, string? ErrorMessage = null);

src/shared/Extensions/ConfigurationExtensions.cs

@@ -0,0 +1,45 @@
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.DependencyInjection.Extensions;
+
+namespace SharedContracts.Extensions;
+
+public static class ConfigurationExtensions
+{
+    private const string ConfigurationFolder = "Configuration";
+    private const string LoggingConfig = "logging.json";
+
+    public static IConfigurationBuilder AddServiceConfiguration(this IConfigurationBuilder builder)
+    {
+        builder.SetBasePath(Path.Combine(AppContext.BaseDirectory, ConfigurationFolder));
+
+        builder.AddJsonFile(LoggingConfig, false, true);
+
+        builder.AddEnvironmentVariables();
+
+        return builder;
+    }
+
+    public static TConfiguration LoadConfigurationFromConfig<TConfiguration>(this IServiceCollection services, IConfiguration configuration, string sectionName)
+        where TConfiguration : class
+    {
+        var instance = configuration.GetSection(sectionName).Get<TConfiguration>();
+
+        ArgumentNullException.ThrowIfNull(instance, nameof(instance));
+
+        services.TryAddSingleton(instance);
+
+        return instance;
+    }
+
+    public static TConfiguration LoadConfigurationFromEnv<TConfiguration>(this IServiceCollection services)
+        where TConfiguration : class
+    {
+        var instance = Activator.CreateInstance<TConfiguration>();
+
+        ArgumentNullException.ThrowIfNull(instance, nameof(instance));
+
+        services.TryAddSingleton(instance);
+
+        return instance;
+    }
+}

src/shared/Extensions/EnvironmentExtensions.cs

@@ -0,0 +1,68 @@
+namespace SharedContracts.Extensions;
+
+public static class EnvironmentExtensions
+{
+    public static bool GetEnvironmentVariableAsBool(this string prefix, string varName, bool fallback = false)
+    {
+        var fullVarName = GetFullVariableName(prefix, varName);
+
+        var str = Environment.GetEnvironmentVariable(fullVarName);
+
+        if (string.IsNullOrEmpty(str))
+        {
+            return fallback;
+        }
+
+        return str.Trim().ToLower() switch
+        {
+            "true" => true,
+            "yes" => true,
+            "1" => true,
+            _ => false,
+        };
+    }
+
+    public static int GetEnvironmentVariableAsInt(this string prefix, string varName, int fallback = 0)
+    {
+        var fullVarName = GetFullVariableName(prefix, varName);
+
+        var str = Environment.GetEnvironmentVariable(fullVarName);
+
+        if (string.IsNullOrEmpty(str))
+        {
+            return fallback;
+        }
+
+        return int.TryParse(str, out var result) ? result : fallback;
+    }
+
+    public static string GetRequiredEnvironmentVariableAsString(this string prefix, string varName)
+    {
+        var fullVarName = GetFullVariableName(prefix, varName);
+
+        var str = Environment.GetEnvironmentVariable(fullVarName);
+
+        if (string.IsNullOrEmpty(str))
+        {
+            throw new InvalidOperationException($"Environment variable {fullVarName} is not set");
+        }
+
+        return str;
+    }
+
+    public static string GetOptionalEnvironmentVariableAsString(this string prefix, string varName, string? fallback = null)
+    {
+        var fullVarName = GetFullVariableName(prefix, varName);
+
+        var str = Environment.GetEnvironmentVariable(fullVarName);
+
+        if (string.IsNullOrEmpty(str))
+        {
+            return fallback;
+        }
+
+        return str;
+    }
+
+    private static string GetFullVariableName(string prefix, string varName) => $"{prefix}_{varName}";
+}

src/shared/Extensions/HostBuilderExtensions.cs

@@ -0,0 +1,8 @@
+namespace SharedContracts.Extensions;
+
+public static class HostBuilderExtensions
+{
+    public static IHostBuilder SetupSerilog(this IHostBuilder builder, IConfiguration configuration) =>
+        builder.UseSerilog((_, c) =>
+            c.ReadFrom.Configuration(configuration));
+}

src/shared/Extensions/JsonSerializerExtensions.cs

@@ -0,0 +1,6 @@
+namespace SharedContracts.Extensions;
+
+public static class JsonSerializerExtensions
+{
+    public static string ToJson<T>(this T value) => JsonSerializer.Serialize(value);
+}

src/shared/Extensions/RabbitMqBusFactoryExtensions.cs

@@ -0,0 +1,44 @@
+namespace SharedContracts.Extensions;
+
+public static class RabbitMqBusFactoryExtensions
+{
+    public static IRabbitMqBusFactoryConfigurator SetupExchangeEndpoint(this IRabbitMqBusFactoryConfigurator cfg,
+        string exchangeName,
+        bool durable = true,
+        bool autoDelete = false,
+        string type = "fanout")
+    {
+        cfg.ReceiveEndpoint(
+            exchangeName, e =>
+            {
+                e.Bind(
+                    exchangeName, config =>
+                    {
+                        config.Durable = durable;
+                        config.AutoDelete = autoDelete;
+                        config.ExchangeType = type;
+                    });
+            });
+
+        return cfg;
+    }
+
+    public static IRabbitMqMessagePublishTopologyConfigurator<TMessage> PublishToQueue<TMessage>(this IRabbitMqMessagePublishTopologyConfigurator<TMessage> cfg,
+        string exchangeName,
+        string queueName,
+        bool durable = true,
+        bool autoDelete = false,
+        string type = "fanout") where TMessage : class
+    {
+        cfg.Durable = durable;
+        cfg.AutoDelete = autoDelete;
+        cfg.ExchangeType = type;
+        cfg.BindQueue(
+            exchangeName, queueName, options =>
+            {
+                options.Durable = durable;
+            });
+        
+        return cfg;
+    }
+}

src/shared/Extensions/StringExtensions.cs

@@ -0,0 +1,41 @@
+using System.Text.RegularExpressions;
+
+namespace SharedContracts.Extensions;
+
+public static partial class StringExtensions
+{
+    [GeneratedRegex("[^a-zA-Z0-9 ]")]
+    private static partial Regex NotAlphaNumeric();
+
+    [GeneratedRegex(@"\s*\([^)]*\)|\s*\b\d{4}\b")]
+    private static partial Regex CleanTitleForImdb();
+
+    private static readonly char[] separator = [' '];
+
+    public static bool IsNullOrEmpty(this string? value) =>
+        string.IsNullOrEmpty(value);
+
+    public static string NormalizeTitle(this string title)
+    {
+        var alphanumericTitle = NotAlphaNumeric().Replace(title, " ");
+
+        var words = alphanumericTitle.Split(separator, StringSplitOptions.RemoveEmptyEntries)
+            .Select(word => word.ToLower());
+
+        var normalizedTitle = string.Join(" ", words);
+
+        return normalizedTitle;
+    }
+
+    public static string RemoveMatches(this string input, IEnumerable<Func<Regex>> regexPatterns) =>
+        regexPatterns.Aggregate(input, (current, regex) => regex().Replace(current, string.Empty));
+
+    public static string CleanTorrentTitleForImdb(this string title)
+    {
+        var cleanTitle = CleanTitleForImdb().Replace(title, "").Trim();
+
+        cleanTitle = cleanTitle.ToLower();
+
+        return cleanTitle;
+    }
+}

src/shared/Extensions/WebApplicationBuilderExtensions.cs

@@ -0,0 +1,13 @@
+namespace SharedContracts.Extensions;
+
+public static class WebApplicationBuilderExtensions
+{
+    public static void DisableIpPortBinding(this WebApplicationBuilder builder) =>
+        builder.WebHost.ConfigureKestrel(options =>
+        {
+            options.ListenAnyIP(0, listenOptions =>
+            {
+                listenOptions.Protocols = Microsoft.AspNetCore.Server.Kestrel.Core.HttpProtocols.None;
+            });
+        });
+}

src/shared/GlobalUsings.cs

@@ -0,0 +1,16 @@
+// Global using directives
+
+global using System.Text.Json;
+global using Dapper;
+global using MassTransit;
+global using Microsoft.AspNetCore.Builder;
+global using Microsoft.AspNetCore.Hosting;
+global using Microsoft.Extensions.Configuration;
+global using Microsoft.Extensions.Hosting;
+global using Microsoft.Extensions.Logging;
+global using Npgsql;
+global using PromKnight.ParseTorrentTitle;
+global using Serilog;
+global using SharedContracts.Configuration;
+global using SharedContracts.Extensions;
+global using SharedContracts.Models;

src/shared/Models/ImdbEntry.cs

@@ -0,0 +1,10 @@
+namespace SharedContracts.Models;
+
+public class ImdbEntry
+{
+    public string ImdbId { get; set; } = default!;
+    public string? Title { get; set; }
+    public string? Category { get; set; }
+    public string? Year { get; set; }
+    public bool? Adult { get; set; }
+}

src/shared/Models/IngestedTorrent.cs

@@ -0,0 +1,18 @@
+namespace SharedContracts.Models;
+
+public class IngestedTorrent
+{
+    public long? Id { get; set; }
+    public string? Name { get; set; }
+    public string? Source { get; set; }
+    public string? Category { get; set; }
+    public string? InfoHash { get; set; }
+    public string? Size { get; set; }
+    public int Seeders { get; set; }
+    public int Leechers { get; set; }
+    public string? Imdb { get; set; }
+
+    public bool Processed { get; set; } = false;
+    public DateTime CreatedAt { get; set; } = DateTime.UtcNow;
+    public DateTime UpdatedAt { get; set; } = DateTime.UtcNow;
+}

src/shared/Models/SubtitleFile.cs

@@ -0,0 +1,10 @@
+namespace SharedContracts.Models;
+
+public class SubtitleFile
+{
+    public int Id { get; set; }
+    public string? InfoHash { get; set; }
+    public int FileIndex { get; set; }
+    public int FileId { get; set; }
+    public string? Title { get; set; }
+}

src/shared/Models/Torrent.cs

@@ -0,0 +1,20 @@
+namespace SharedContracts.Models;
+
+public class Torrent
+{
+    public string? InfoHash { get; set; }
+    public string? Provider { get; set; }
+    public string? TorrentId { get; set; }
+    public string? Title { get; set; }
+    public long? Size { get; set; }
+    public string? Type { get; set; }
+    public string? UploadDate { get; set; }
+    public int? Seeders { get; set; }
+    public string? Trackers { get; set; }
+    public string? Languages { get; set; }
+    public string? Resolution { get; set; }
+    public bool? Reviewed { get; set; }
+    public bool? Opened { get; set; }
+    public string? CreatedAt { get; set; }
+    public string? UpdatedAt { get; set; }
+}

src/shared/Models/TorrentFile.cs

@@ -0,0 +1,15 @@
+namespace SharedContracts.Models;
+
+public class TorrentFile
+{
+    public int Id { get; set; }
+    public string InfoHash { get; set; } = default!;
+    public int FileIndex { get; set; }
+    public string Title { get; set; } = default!;
+    public long Size { get; set; }
+    public string? ImdbId { get; set; }
+    public int? ImdbSeason { get; set; }
+    public int? ImdbEpisode { get; set; }
+    public int? KitsuId { get; set; }
+    public int? KitsuEpisode { get; set; }
+}

src/shared/Requests/CollectMetadata.cs

@@ -0,0 +1,3 @@
+namespace SharedContracts.Requests;
+
+public record CollectMetadata(Guid CorrelationId, Torrent Torrent, string ImdbId) : CorrelatedBy<Guid>;

src/shared/Requests/IngestTorrent.cs

@@ -0,0 +1,3 @@
+namespace SharedContracts.Requests;
+
+public record IngestTorrent(Guid CorrelationId, IngestedTorrent IngestedTorrent) : CorrelatedBy<Guid>;

src/shared/SharedContracts.csproj

@@ -0,0 +1,25 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+<PropertyGroup>
+    <OutputType>Library</OutputType>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+</PropertyGroup>
+
+    <ItemGroup>
+        <FrameworkReference Include="Microsoft.AspNetCore.App" />
+    </ItemGroup>
+    
+<ItemGroup>
+  <PackageReference Include="Dapper" Version="2.1.35" />
+  <PackageReference Include="MassTransit.Abstractions" Version="8.2.0" />
+  <PackageReference Include="MassTransit.RabbitMQ" Version="8.2.0" />
+  <PackageReference Include="Npgsql" Version="8.0.2" />
+  <PackageReference Include="PromKnight.ParseTorrentTitle" Version="1.0.4" />
+  <PackageReference Include="Serilog" Version="3.1.1" />
+  <PackageReference Include="Serilog.Extensions.Hosting" Version="8.0.0" />
+  <PackageReference Include="Serilog.Settings.Configuration" Version="8.0.0" />
+</ItemGroup>
+    
+</Project>