diff --git a/deployment/docker/docker-compose.yaml b/deployment/docker/docker-compose.yaml index 334ac29..2906f54 100644 --- a/deployment/docker/docker-compose.yaml +++ b/deployment/docker/docker-compose.yaml @@ -94,7 +94,7 @@ services: condition: service_healthy env_file: stack.env hostname: knightcrawler-addon - image: gabisonfire/knightcrawler-addon:2.0.9 + image: gabisonfire/knightcrawler-addon:2.0.10 labels: logging: promtail networks: @@ -117,7 +117,7 @@ services: redis: condition: service_healthy env_file: stack.env - image: gabisonfire/knightcrawler-consumer:2.0.9 + image: gabisonfire/knightcrawler-consumer:2.0.10 labels: logging: promtail networks: @@ -138,7 +138,7 @@ services: redis: condition: service_healthy env_file: stack.env - image: gabisonfire/knightcrawler-debrid-collector:2.0.9 + image: gabisonfire/knightcrawler-debrid-collector:2.0.10 labels: logging: promtail networks: @@ -152,7 +152,7 @@ services: migrator: condition: service_completed_successfully env_file: stack.env - image: gabisonfire/knightcrawler-metadata:2.0.9 + image: gabisonfire/knightcrawler-metadata:2.0.10 networks: - knightcrawler-network restart: "no" @@ -163,7 +163,7 @@ services: postgres: condition: service_healthy env_file: stack.env - image: gabisonfire/knightcrawler-migrator:2.0.9 + image: gabisonfire/knightcrawler-migrator:2.0.10 networks: - knightcrawler-network restart: "no" @@ -182,7 +182,7 @@ services: redis: condition: service_healthy env_file: stack.env - image: gabisonfire/knightcrawler-producer:2.0.9 + image: gabisonfire/knightcrawler-producer:2.0.10 labels: logging: promtail networks: @@ -207,7 +207,7 @@ services: deploy: replicas: ${QBIT_REPLICAS:-0} env_file: stack.env - image: gabisonfire/knightcrawler-qbit-collector:2.0.9 + image: gabisonfire/knightcrawler-qbit-collector:2.0.10 labels: logging: promtail networks: diff --git a/deployment/docker/src/components/knightcrawler.yaml b/deployment/docker/src/components/knightcrawler.yaml index e06261d..830d0a7 100644 --- a/deployment/docker/src/components/knightcrawler.yaml +++ b/deployment/docker/src/components/knightcrawler.yaml @@ -20,7 +20,7 @@ x-depends: &knightcrawler-app-depends services: metadata: - image: gabisonfire/knightcrawler-metadata:2.0.9 + image: gabisonfire/knightcrawler-metadata:2.0.10 env_file: ../../.env networks: - knightcrawler-network @@ -30,7 +30,7 @@ services: condition: service_completed_successfully migrator: - image: gabisonfire/knightcrawler-migrator:2.0.9 + image: gabisonfire/knightcrawler-migrator:2.0.10 env_file: ../../.env networks: - knightcrawler-network @@ -40,7 +40,7 @@ services: condition: service_healthy addon: - image: gabisonfire/knightcrawler-addon:2.0.9 + image: gabisonfire/knightcrawler-addon:2.0.10 <<: [*knightcrawler-app, *knightcrawler-app-depends] restart: unless-stopped hostname: knightcrawler-addon @@ -48,22 +48,22 @@ services: - "7000:7000" consumer: - image: gabisonfire/knightcrawler-consumer:2.0.9 + image: gabisonfire/knightcrawler-consumer:2.0.10 <<: [*knightcrawler-app, *knightcrawler-app-depends] restart: unless-stopped debridcollector: - image: gabisonfire/knightcrawler-debrid-collector:2.0.9 + image: gabisonfire/knightcrawler-debrid-collector:2.0.10 <<: [*knightcrawler-app, *knightcrawler-app-depends] restart: unless-stopped producer: - image: gabisonfire/knightcrawler-producer:2.0.9 + image: gabisonfire/knightcrawler-producer:2.0.10 <<: [*knightcrawler-app, *knightcrawler-app-depends] restart: unless-stopped qbitcollector: - image: gabisonfire/knightcrawler-qbit-collector:2.0.9 + image: gabisonfire/knightcrawler-qbit-collector:2.0.10 <<: [*knightcrawler-app, *knightcrawler-app-depends] restart: unless-stopped depends_on: diff --git a/src/metadata/Features/Files/BasicsFile.cs b/src/metadata/Features/Files/BasicsFile.cs index c55a610..490e7d2 100644 --- a/src/metadata/Features/Files/BasicsFile.cs +++ b/src/metadata/Features/Files/BasicsFile.cs @@ -72,7 +72,7 @@ public class BasicsFile(ILogger logger, ImdbDbService dbService): IF Category = csv.GetField(1), Title = csv.GetField(2), Adult = isAdultSet && adult == 1, - Year = csv.GetField(5), + Year = csv.GetField(5) == @"\N" ? 0 : int.Parse(csv.GetField(5)), }; if (cancellationToken.IsCancellationRequested) diff --git a/src/metadata/Features/Files/ImdbBasicEntry.cs b/src/metadata/Features/Files/ImdbBasicEntry.cs index c8b2cb1..cccaec6 100644 --- a/src/metadata/Features/Files/ImdbBasicEntry.cs +++ b/src/metadata/Features/Files/ImdbBasicEntry.cs @@ -6,5 +6,5 @@ public class ImdbBasicEntry public string? Category { get; set; } public string? Title { get; set; } public bool Adult { get; set; } - public string? Year { get; set; } + public int Year { get; set; } } \ No newline at end of file diff --git a/src/metadata/Features/ImportImdbData/ImdbDbService.cs b/src/metadata/Features/ImportImdbData/ImdbDbService.cs index 88add91..4c584c0 100644 --- a/src/metadata/Features/ImportImdbData/ImdbDbService.cs +++ b/src/metadata/Features/ImportImdbData/ImdbDbService.cs @@ -17,7 +17,7 @@ public class ImdbDbService(PostgresConfiguration configuration, ILogger { - await using var command = new NpgsqlCommand($"CREATE INDEX title_gist ON {TableNames.MetadataTable} USING gist(title gist_trgm_ops)", connection); + await using var command = new NpgsqlCommand($"CREATE INDEX title_gin ON {TableNames.MetadataTable} USING gin(title gin_trgm_ops)", connection); await command.ExecuteNonQueryAsync(); }, "Error while creating index on imdb_metadata table"); @@ -125,7 +125,7 @@ public class ImdbDbService(PostgresConfiguration configuration, ILogger { logger.LogInformation("Dropping Trigrams index if it exists already"); - await using var dropCommand = new NpgsqlCommand("DROP INDEX if exists title_gist", connection); + await using var dropCommand = new NpgsqlCommand("DROP INDEX if exists title_gin", connection); await dropCommand.ExecuteNonQueryAsync(); }, $"Error while dropping index on {TableNames.MetadataTable} table"); diff --git a/src/migrator/migrations/009_imdb_year_column_int.sql b/src/migrator/migrations/009_imdb_year_column_int.sql new file mode 100644 index 0000000..2617c57 --- /dev/null +++ b/src/migrator/migrations/009_imdb_year_column_int.sql @@ -0,0 +1,35 @@ +-- Purpose: Change the year column to integer and add a search function that allows for searching by year. +ALTER TABLE imdb_metadata +ALTER COLUMN year TYPE integer USING (CASE WHEN year = '\N' THEN 0 ELSE year::integer END); + +-- Remove the old search function +DROP FUNCTION IF EXISTS search_imdb_meta(TEXT, TEXT, TEXT, INT); + +-- Add the new search function that allows for searching by year with a plus/minus one year range +CREATE OR REPLACE FUNCTION search_imdb_meta(search_term TEXT, category_param TEXT DEFAULT NULL, year_param INT DEFAULT NULL, limit_param INT DEFAULT 10) + RETURNS TABLE(imdb_id character varying(16), title character varying(1000),category character varying(50),year INT, score REAL) AS $$ +BEGIN + SET pg_trgm.similarity_threshold = 0.9; + RETURN QUERY + SELECT imdb_metadata.imdb_id, imdb_metadata.title, imdb_metadata.category, imdb_metadata.year, similarity(imdb_metadata.title, search_term) as score + FROM imdb_metadata + WHERE (imdb_metadata.title % search_term) + AND (imdb_metadata.adult = FALSE) + AND (category_param IS NULL OR imdb_metadata.category = category_param) + AND (year_param IS NULL OR imdb_metadata.year BETWEEN year_param - 1 AND year_param + 1) + ORDER BY score DESC + LIMIT limit_param; +END; $$ + LANGUAGE plpgsql; + +-- Drop the old indexes +DROP INDEX IF EXISTS idx_imdb_metadata_adult; +DROP INDEX IF EXISTS idx_imdb_metadata_category; +DROP INDEX IF EXISTS idx_imdb_metadata_year; +DROP INDEX IF EXISTS title_gist; + +-- Add indexes for the new columns +CREATE INDEX idx_imdb_metadata_adult ON imdb_metadata(adult); +CREATE INDEX idx_imdb_metadata_category ON imdb_metadata(category); +CREATE INDEX idx_imdb_metadata_year ON imdb_metadata(year); +CREATE INDEX title_gin ON imdb_metadata USING gin(title gin_trgm_ops); \ No newline at end of file diff --git a/src/migrator/migrations/010_add_rtn_response_to_ingested_and_torrent.sql b/src/migrator/migrations/010_add_rtn_response_to_ingested_and_torrent.sql new file mode 100644 index 0000000..6a1a4b9 --- /dev/null +++ b/src/migrator/migrations/010_add_rtn_response_to_ingested_and_torrent.sql @@ -0,0 +1,40 @@ +-- Purpose: Add the jsonb column to the ingested_torrents table to store the response from RTN +ALTER TABLE ingested_torrents +ADD COLUMN IF NOT EXISTS rtn_response jsonb; + +-- Purpose: Drop torrentId column from torrents table +ALTER TABLE torrents +DROP COLUMN IF EXISTS "torrentId"; + +-- Purpose: Drop Trackers column from torrents table +ALTER TABLE torrents +DROP COLUMN IF EXISTS "trackers"; + +-- Purpose: Create a foreign key relationsship if it does not already exist between torrents and the source table ingested_torrents, but do not cascade on delete. +ALTER TABLE torrents +ADD COLUMN IF NOT EXISTS "ingestedTorrentId" bigint; + +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM information_schema.table_constraints + WHERE constraint_name = 'fk_torrents_info_hash' + ) + THEN + ALTER TABLE torrents + DROP CONSTRAINT fk_torrents_info_hash; + END IF; +END $$; + +ALTER TABLE torrents +ADD CONSTRAINT fk_torrents_info_hash +FOREIGN KEY ("ingestedTorrentId") +REFERENCES ingested_torrents("id") +ON DELETE NO ACTION; + +UPDATE torrents +SET "ingestedTorrentId" = ingested_torrents."id" +FROM ingested_torrents +WHERE torrents."infoHash" = ingested_torrents."info_hash" +AND torrents."provider" = ingested_torrents."source"; \ No newline at end of file diff --git a/src/migrator/migrations/011_housekeeping_reconciliation_imdbIds_dmm.sql b/src/migrator/migrations/011_housekeeping_reconciliation_imdbIds_dmm.sql new file mode 100644 index 0000000..8ef4843 --- /dev/null +++ b/src/migrator/migrations/011_housekeeping_reconciliation_imdbIds_dmm.sql @@ -0,0 +1,55 @@ +DROP FUNCTION IF EXISTS kc_maintenance_reconcile_dmm_imdb_ids(); +CREATE OR REPLACE FUNCTION kc_maintenance_reconcile_dmm_imdb_ids() +RETURNS INTEGER AS $$ +DECLARE + rec RECORD; + imdb_rec RECORD; + rows_affected INTEGER := 0; +BEGIN + RAISE NOTICE 'Starting Reconciliation of DMM IMDB Ids...'; + FOR rec IN + SELECT + it."id" as "ingestion_id", + t."infoHash", + it."category" as "ingestion_category", + f."id" as "file_Id", + f."title" as "file_Title", + (rtn_response->>'raw_title')::text as "raw_title", + (rtn_response->>'parsed_title')::text as "parsed_title", + (rtn_response->>'year')::int as "year" + FROM torrents t + JOIN ingested_torrents it ON t."ingestedTorrentId" = it."id" + JOIN files f ON t."infoHash" = f."infoHash" + WHERE t."provider" = 'DMM' + LOOP + RAISE NOTICE 'Processing record with file_Id: %', rec."file_Id"; + FOR imdb_rec IN + SELECT * FROM search_imdb_meta( + rec."parsed_title", + CASE + WHEN rec."ingestion_category" = 'tv' THEN 'tvSeries' + WHEN rec."ingestion_category" = 'movies' THEN 'movie' + END, + CASE + WHEN rec."year" = 0 THEN NULL + ELSE rec."year" END, + 1) + LOOP + IF imdb_rec IS NOT NULL THEN + RAISE NOTICE 'Updating file_Id: % with imdbId: %, parsed title: %, imdb title: %', rec."file_Id", imdb_rec."imdb_id", rec."parsed_title", imdb_rec."title"; + UPDATE "files" + SET "imdbId" = imdb_rec."imdb_id" + WHERE "id" = rec."file_Id"; + rows_affected := rows_affected + 1; + ELSE + RAISE NOTICE 'No IMDB ID found for file_Id: %, parsed title: %, imdb title: %, setting imdbId to NULL', rec."file_Id", rec."parsed_title", imdb_rec."title"; + UPDATE "files" + SET "imdbId" = NULL + WHERE "id" = rec."file_Id"; + END IF; + END LOOP; + END LOOP; + RAISE NOTICE 'Finished reconciliation. Total rows affected: %', rows_affected; + RETURN rows_affected; +END; +$$ LANGUAGE plpgsql; \ No newline at end of file diff --git a/src/producer/src/Features/Crawlers/Dmm/DebridMediaManagerCrawler.cs b/src/producer/src/Features/Crawlers/Dmm/DebridMediaManagerCrawler.cs index 675a6c3..b8f0701 100644 --- a/src/producer/src/Features/Crawlers/Dmm/DebridMediaManagerCrawler.cs +++ b/src/producer/src/Features/Crawlers/Dmm/DebridMediaManagerCrawler.cs @@ -1,5 +1,3 @@ -using Microsoft.VisualBasic; - namespace Producer.Features.Crawlers.Dmm; public partial class DebridMediaManagerCrawler( @@ -12,7 +10,6 @@ public partial class DebridMediaManagerCrawler( { [GeneratedRegex("""""")] private static partial Regex HashCollectionMatcher(); - private LengthAwareRatioScorer _lengthAwareRatioScorer = new(); private const string DownloadBaseUrl = "https://raw.githubusercontent.com/debridmediamanager/hashlists/main"; protected override IReadOnlyDictionary Mappings => new Dictionary(); @@ -118,32 +115,27 @@ public partial class DebridMediaManagerCrawler( return null; } - var (cached, cachedResult) = await CheckIfInCacheAndReturn(parsedTorrent.ParsedTitle); + var (cached, cachedResult) = await CheckIfInCacheAndReturn(parsedTorrent.Response.ParsedTitle); if (cached) { - logger.LogInformation("[{ImdbId}] Found cached imdb result for {Title}", cachedResult.ImdbId, parsedTorrent.ParsedTitle); + logger.LogInformation("[{ImdbId}] Found cached imdb result for {Title}", cachedResult.ImdbId, parsedTorrent.Response.ParsedTitle); return MapToTorrent(cachedResult, bytesElement, hashElement, parsedTorrent); } - var year = parsedTorrent.Year != 0 ? parsedTorrent.Year.ToString() : null; - var imdbEntries = await Storage.FindImdbMetadata(parsedTorrent.ParsedTitle, parsedTorrent.IsMovie ? "movies" : "tv", year); + int? year = parsedTorrent.Response.Year != 0 ? parsedTorrent.Response.Year : null; + var imdbEntry = await Storage.FindImdbMetadata(parsedTorrent.Response.ParsedTitle, parsedTorrent.Response.IsMovie ? "movies" : "tv", year); - if (imdbEntries.Count == 0) + if (imdbEntry is null) { return null; } - var scoredTitles = await ScoreTitles(parsedTorrent, imdbEntries); + await AddToCache(parsedTorrent.Response.ParsedTitle.ToLowerInvariant(), imdbEntry); - if (!scoredTitles.Success) - { - return null; - } - - logger.LogInformation("[{ImdbId}] Found best match for {Title}: {BestMatch} with score {Score}", scoredTitles.BestMatch.Value.ImdbId, parsedTorrent.ParsedTitle, scoredTitles.BestMatch.Value.Title, scoredTitles.BestMatch.Score); + logger.LogInformation("[{ImdbId}] Found best match for {Title}: {BestMatch} with score {Score}", imdbEntry.ImdbId, parsedTorrent.Response.ParsedTitle, imdbEntry.Title, imdbEntry.Score); - return MapToTorrent(scoredTitles.BestMatch.Value, bytesElement, hashElement, parsedTorrent); + return MapToTorrent(imdbEntry, bytesElement, hashElement, parsedTorrent); } private IngestedTorrent MapToTorrent(ImdbEntry result, JsonElement bytesElement, JsonElement hashElement, ParseTorrentTitleResponse parsedTorrent) => @@ -156,40 +148,22 @@ public partial class DebridMediaManagerCrawler( InfoHash = hashElement.ToString(), Seeders = 0, Leechers = 0, - Category = parsedTorrent.IsMovie switch + Category = parsedTorrent.Response.IsMovie switch { true => "movies", false => "tv", }, + RtnResponse = parsedTorrent.Response.ToJson(), }; - private async Task<(bool Success, ExtractedResult? BestMatch)> ScoreTitles(ParseTorrentTitleResponse parsedTorrent, List imdbEntries) - { - var lowerCaseTitle = parsedTorrent.ParsedTitle.ToLowerInvariant(); - - // Scoring directly operates on the List, no need for lookup table. - var scoredResults = Process.ExtractAll(new(){Title = lowerCaseTitle}, imdbEntries, x => x.Title?.ToLowerInvariant(), scorer: _lengthAwareRatioScorer, cutoff: 90); - - var best = scoredResults.MaxBy(x => x.Score); - - if (best is null) - { - return (false, null); - } - - await AddToCache(lowerCaseTitle, best); - - return (true, best); - } - - private Task AddToCache(string lowerCaseTitle, ExtractedResult best) + private Task AddToCache(string lowerCaseTitle, ImdbEntry best) { var cacheOptions = new DistributedCacheEntryOptions { AbsoluteExpirationRelativeToNow = TimeSpan.FromDays(1), }; - return cache.SetStringAsync(lowerCaseTitle, JsonSerializer.Serialize(best.Value), cacheOptions); + return cache.SetStringAsync(lowerCaseTitle, JsonSerializer.Serialize(best), cacheOptions); } private async Task<(bool Success, ImdbEntry? Entry)> CheckIfInCacheAndReturn(string title) diff --git a/src/producer/src/requirements.txt b/src/producer/src/requirements.txt index 7d700a4..9256344 100644 --- a/src/producer/src/requirements.txt +++ b/src/producer/src/requirements.txt @@ -1 +1 @@ -rank-torrent-name==0.1.8 \ No newline at end of file +rank-torrent-name==0.1.9 \ No newline at end of file diff --git a/src/shared/Dapper/DapperDataStorage.cs b/src/shared/Dapper/DapperDataStorage.cs index 28ad700..3609b68 100644 --- a/src/shared/Dapper/DapperDataStorage.cs +++ b/src/shared/Dapper/DapperDataStorage.cs @@ -9,9 +9,9 @@ public class DapperDataStorage(PostgresConfiguration configuration, RabbitMqConf const string query = """ INSERT INTO ingested_torrents - ("name", "source", "category", "info_hash", "size", "seeders", "leechers", "imdb", "processed", "createdAt", "updatedAt") + ("name", "source", "category", "info_hash", "size", "seeders", "leechers", "imdb", "processed", "createdAt", "updatedAt", "rtn_response") VALUES - (@Name, @Source, @Category, @InfoHash, @Size, @Seeders, @Leechers, @Imdb, @Processed, @CreatedAt, @UpdatedAt) + (@Name, @Source, @Category, @InfoHash, @Size, @Seeders, @Leechers, @Imdb, @Processed, @CreatedAt, @UpdatedAt, @RtnResponse::jsonb) ON CONFLICT (source, info_hash) DO NOTHING """; @@ -110,21 +110,21 @@ public class DapperDataStorage(PostgresConfiguration configuration, RabbitMqConf public async Task> GetImdbEntriesForRequests(int year, int batchSize, string? stateLastProcessedImdbId, CancellationToken cancellationToken = default) => await ExecuteCommandAsync(async connection => { - const string query = @"SELECT imdb_id AS ImdbId, title as Title, category as Category, year as Year, adult as Adult FROM imdb_metadata WHERE CAST(NULLIF(Year, '\N') AS INTEGER) <= @Year AND imdb_id > @LastProcessedImdbId ORDER BY ImdbId LIMIT @BatchSize"; + const string query = @"SELECT imdb_id AS ImdbId, title as Title, category as Category, year as Year, adult as Adult FROM imdb_metadata WHERE Year <= @Year AND imdb_id > @LastProcessedImdbId ORDER BY ImdbId LIMIT @BatchSize"; var result = await connection.QueryAsync(query, new { Year = year, LastProcessedImdbId = stateLastProcessedImdbId, BatchSize = batchSize }); return result.ToList(); }, "Error getting imdb metadata.", cancellationToken); - public async Task> FindImdbMetadata(string? parsedTorrentTitle, string torrentType, string? year, CancellationToken cancellationToken = default) => + public async Task FindImdbMetadata(string? parsedTorrentTitle, string torrentType, int? year, CancellationToken cancellationToken = default) => await ExecuteCommandAsync(async connection => { - var query = $"select \"imdb_id\" as \"ImdbId\", \"title\" as \"Title\", \"year\" as \"Year\" from search_imdb_meta('{parsedTorrentTitle.Replace("'", "").Replace("\"", "")}', '{(torrentType.Equals("movie", StringComparison.OrdinalIgnoreCase) ? "movie" : "tvSeries")}'"; - query += year is not null ? $", '{year}'" : ", NULL"; - query += ", 15)"; + var query = $"select \"imdb_id\" as \"ImdbId\", \"title\" as \"Title\", \"year\" as \"Year\", \"score\" as Score from search_imdb_meta('{parsedTorrentTitle.Replace("'", "").Replace("\"", "")}', '{(torrentType.Equals("movie", StringComparison.OrdinalIgnoreCase) ? "movie" : "tvSeries")}'"; + query += year is not null ? $", {year}" : ", NULL"; + query += ", 1)"; var result = await connection.QueryAsync(query); - - return result.ToList(); + var results = result.ToList(); + return results.FirstOrDefault(); }, "Error finding imdb metadata.", cancellationToken); public Task InsertTorrent(Torrent torrent, CancellationToken cancellationToken = default) => @@ -134,9 +134,9 @@ public class DapperDataStorage(PostgresConfiguration configuration, RabbitMqConf const string query = """ INSERT INTO "torrents" - ("infoHash", "provider", "torrentId", "title", "size", "type", "uploadDate", "seeders", "trackers", "languages", "resolution", "reviewed", "opened", "createdAt", "updatedAt") + ("infoHash", "ingestedTorrentId", "provider", "title", "size", "type", "uploadDate", "seeders", "languages", "resolution", "reviewed", "opened", "createdAt", "updatedAt") VALUES - (@InfoHash, @Provider, @TorrentId, @Title, 0, @Type, NOW(), @Seeders, NULL, NULL, NULL, false, false, NOW(), NOW()) + (@InfoHash, @IngestedTorrentId, @Provider, @Title, 0, @Type, NOW(), @Seeders, NULL, NULL, false, false, NOW(), NOW()) ON CONFLICT ("infoHash") DO NOTHING """; diff --git a/src/shared/Dapper/IDataStorage.cs b/src/shared/Dapper/IDataStorage.cs index d9b0dfd..18ead18 100644 --- a/src/shared/Dapper/IDataStorage.cs +++ b/src/shared/Dapper/IDataStorage.cs @@ -9,7 +9,7 @@ public interface IDataStorage Task> MarkPageAsIngested(string pageId, CancellationToken cancellationToken = default); Task> GetRowCountImdbMetadata(CancellationToken cancellationToken = default); Task> GetImdbEntriesForRequests(int year, int batchSize, string? stateLastProcessedImdbId, CancellationToken cancellationToken = default); - Task> FindImdbMetadata(string? parsedTorrentTitle, string parsedTorrentTorrentType, string? parsedTorrentYear, CancellationToken cancellationToken = default); + Task FindImdbMetadata(string? parsedTorrentTitle, string parsedTorrentTorrentType, int? parsedTorrentYear, CancellationToken cancellationToken = default); Task InsertTorrent(Torrent torrent, CancellationToken cancellationToken = default); Task InsertFiles(IEnumerable files, CancellationToken cancellationToken = default); Task InsertSubtitles(IEnumerable subtitles, CancellationToken cancellationToken = default); diff --git a/src/shared/Extensions/JsonExtensions.cs b/src/shared/Extensions/JsonExtensions.cs new file mode 100644 index 0000000..ce0bb5c --- /dev/null +++ b/src/shared/Extensions/JsonExtensions.cs @@ -0,0 +1,14 @@ +namespace SharedContracts.Extensions; + +public static class JsonExtensions +{ + private static readonly JsonSerializerOptions JsonSerializerOptions = new() + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + WriteIndented = false, + ReferenceHandler = ReferenceHandler.IgnoreCycles, + NumberHandling = JsonNumberHandling.Strict, + }; + + public static string AsJson(this T obj) => JsonSerializer.Serialize(obj, JsonSerializerOptions); +} \ No newline at end of file diff --git a/src/shared/GlobalUsings.cs b/src/shared/GlobalUsings.cs index a652e84..1eb535e 100644 --- a/src/shared/GlobalUsings.cs +++ b/src/shared/GlobalUsings.cs @@ -1,9 +1,7 @@ // Global using directives -global using System.Collections.Concurrent; -global using System.Globalization; -global using System.Text; global using System.Text.Json; +global using System.Text.Json.Serialization; global using Dapper; global using MassTransit; global using Microsoft.AspNetCore.Builder; @@ -17,4 +15,4 @@ global using Python.Runtime; global using Serilog; global using SharedContracts.Configuration; global using SharedContracts.Extensions; -global using SharedContracts.Models; \ No newline at end of file +global using SharedContracts.Models; diff --git a/src/shared/Models/ImdbEntry.cs b/src/shared/Models/ImdbEntry.cs index 8ae6546..991c7d3 100644 --- a/src/shared/Models/ImdbEntry.cs +++ b/src/shared/Models/ImdbEntry.cs @@ -7,4 +7,5 @@ public class ImdbEntry public string? Category { get; set; } public string? Year { get; set; } public bool? Adult { get; set; } + public decimal? Score { get; set; } } diff --git a/src/shared/Models/IngestedTorrent.cs b/src/shared/Models/IngestedTorrent.cs index 54d2b85..23ae063 100644 --- a/src/shared/Models/IngestedTorrent.cs +++ b/src/shared/Models/IngestedTorrent.cs @@ -12,7 +12,9 @@ public class IngestedTorrent public int Leechers { get; set; } public string? Imdb { get; set; } - public bool Processed { get; set; } = false; + public bool Processed { get; set; } public DateTime CreatedAt { get; set; } = DateTime.UtcNow; public DateTime UpdatedAt { get; set; } = DateTime.UtcNow; + + public string? RtnResponse { get; set; } } diff --git a/src/shared/Models/Torrent.cs b/src/shared/Models/Torrent.cs index 550872d..8f488f1 100644 --- a/src/shared/Models/Torrent.cs +++ b/src/shared/Models/Torrent.cs @@ -3,6 +3,7 @@ namespace SharedContracts.Models; public class Torrent { public string? InfoHash { get; set; } + public long? IngestedTorrentId { get; set; } public string? Provider { get; set; } public string? TorrentId { get; set; } public string? Title { get; set; } diff --git a/src/shared/Python/IPythonEngineService.cs b/src/shared/Python/IPythonEngineService.cs index 5502e59..e14a7c1 100644 --- a/src/shared/Python/IPythonEngineService.cs +++ b/src/shared/Python/IPythonEngineService.cs @@ -7,7 +7,7 @@ public interface IPythonEngineService Task InitializePythonEngine(CancellationToken cancellationToken); T ExecuteCommandOrScript(string command, PyModule module, bool throwOnErrors); T ExecutePythonOperation(Func operation, string operationName, bool throwOnErrors); - T ExecutePythonOperationWithDefault(Func operation, T? defaultValue, string operationName, bool throwOnErrors); + T ExecutePythonOperationWithDefault(Func operation, T? defaultValue, string operationName, bool throwOnErrors, bool logErrors); Task StopPythonEngine(CancellationToken cancellationToken); dynamic? Sys { get; } } \ No newline at end of file diff --git a/src/shared/Python/PythonEngineService.cs b/src/shared/Python/PythonEngineService.cs index 9ae513c..183db73 100644 --- a/src/shared/Python/PythonEngineService.cs +++ b/src/shared/Python/PythonEngineService.cs @@ -53,10 +53,10 @@ public class PythonEngineService(ILogger logger) : IPythonE }, nameof(ExecuteCommandOrScript), throwOnErrors); public T ExecutePythonOperation(Func operation, string operationName, bool throwOnErrors) => - ExecutePythonOperationWithDefault(operation, default, operationName, throwOnErrors); + ExecutePythonOperationWithDefault(operation, default, operationName, throwOnErrors, true); - public T ExecutePythonOperationWithDefault(Func operation, T? defaultValue, string operationName, bool throwOnErrors) => - ExecutePythonOperationInternal(operation, defaultValue, operationName, throwOnErrors); + public T ExecutePythonOperationWithDefault(Func operation, T? defaultValue, string operationName, bool throwOnErrors, bool logErrors) => + ExecutePythonOperationInternal(operation, defaultValue, operationName, throwOnErrors, logErrors); public void ExecuteOnGIL(Action act, bool throwOnErrors) { @@ -95,7 +95,7 @@ public class PythonEngineService(ILogger logger) : IPythonE } // ReSharper disable once EntityNameCapturedOnly.Local - private T ExecutePythonOperationInternal(Func operation, T? defaultValue, string operationName, bool throwOnErrors) + private T ExecutePythonOperationInternal(Func operation, T? defaultValue, string operationName, bool throwOnErrors, bool logErrors) { Sys ??= LoadSys(); @@ -108,7 +108,10 @@ public class PythonEngineService(ILogger logger) : IPythonE } catch (Exception ex) { - Logger.LogError(ex, "Python Error: {Message} ({OperationName})", ex.Message, nameof(operationName)); + if (logErrors) + { + Logger.LogError(ex, "Python Error: {Message} ({OperationName})", ex.Message, nameof(operationName)); + } if (throwOnErrors) { diff --git a/src/shared/Python/RTN/ParseTorrentTitleResponse.cs b/src/shared/Python/RTN/ParseTorrentTitleResponse.cs index 3c1d905..63748b8 100644 --- a/src/shared/Python/RTN/ParseTorrentTitleResponse.cs +++ b/src/shared/Python/RTN/ParseTorrentTitleResponse.cs @@ -1,6 +1,3 @@ namespace SharedContracts.Python.RTN; -public record ParseTorrentTitleResponse(bool Success, string ParsedTitle, int Year, int[]? Season = null, int[]? Episode = null) -{ - public bool IsMovie => Season == null && Episode == null; -} \ No newline at end of file +public record ParseTorrentTitleResponse(bool Success, RtnResponse? Response); \ No newline at end of file diff --git a/src/shared/Python/RTN/RankTorrentName.cs b/src/shared/Python/RTN/RankTorrentName.cs index 83ec681..723bf1a 100644 --- a/src/shared/Python/RTN/RankTorrentName.cs +++ b/src/shared/Python/RTN/RankTorrentName.cs @@ -14,34 +14,31 @@ public class RankTorrentName : IRankTorrentName } public ParseTorrentTitleResponse Parse(string title) => - _pythonEngineService.ExecutePythonOperation( + _pythonEngineService.ExecutePythonOperationWithDefault( () => { var result = _rtn?.parse(title); return ParseResult(result); - }, nameof(Parse), throwOnErrors: false); - + }, new ParseTorrentTitleResponse(false, null), nameof(Parse), throwOnErrors: false, logErrors: false); + + private static ParseTorrentTitleResponse ParseResult(dynamic result) { if (result == null) { - return new(false, string.Empty, 0); + return new(false, null); + } + + var json = result.model_dump_json()?.As(); + + if (json is null || string.IsNullOrEmpty(json)) + { + return new(false, null); } - var parsedTitle = result.GetAttr("parsed_title")?.As() ?? string.Empty; - var year = result.GetAttr("year")?.As() ?? 0; - var seasons = GetIntArray(result, "season"); - var episodes = GetIntArray(result, "episode"); + var response = JsonSerializer.Deserialize(json); - return new ParseTorrentTitleResponse(true, parsedTitle, year, seasons, episodes); - } - - private static int[]? GetIntArray(dynamic result, string field) - { - var theList = result.GetAttr(field)?.As(); - int[]? results = theList?.Length() > 0 ? theList.As() : null; - - return results; + return new(true, response); } private void InitModules() => diff --git a/src/shared/Python/RTN/RtnResponse.cs b/src/shared/Python/RTN/RtnResponse.cs new file mode 100644 index 0000000..e81e99f --- /dev/null +++ b/src/shared/Python/RTN/RtnResponse.cs @@ -0,0 +1,83 @@ +namespace SharedContracts.Python.RTN; + +public class RtnResponse +{ + [JsonPropertyName("raw_title")] + public string? RawTitle { get; set; } + + [JsonPropertyName("parsed_title")] + public string? ParsedTitle { get; set; } + + [JsonPropertyName("fetch")] + public bool Fetch { get; set; } + + [JsonPropertyName("is_4k")] + public bool Is4K { get; set; } + + [JsonPropertyName("is_multi_audio")] + public bool IsMultiAudio { get; set; } + + [JsonPropertyName("is_multi_subtitle")] + public bool IsMultiSubtitle { get; set; } + + [JsonPropertyName("is_complete")] + public bool IsComplete { get; set; } + + [JsonPropertyName("year")] + public int Year { get; set; } + + [JsonPropertyName("resolution")] + public List? Resolution { get; set; } + + [JsonPropertyName("quality")] + public List? Quality { get; set; } + + [JsonPropertyName("season")] + public List? Season { get; set; } + + [JsonPropertyName("episode")] + public List? Episode { get; set; } + + [JsonPropertyName("codec")] + public List? Codec { get; set; } + + [JsonPropertyName("audio")] + public List? Audio { get; set; } + + [JsonPropertyName("subtitles")] + public List? Subtitles { get; set; } + + [JsonPropertyName("language")] + public List? Language { get; set; } + + [JsonPropertyName("bit_depth")] + public List? BitDepth { get; set; } + + [JsonPropertyName("hdr")] + public string? Hdr { get; set; } + + [JsonPropertyName("proper")] + public bool Proper { get; set; } + + [JsonPropertyName("repack")] + public bool Repack { get; set; } + + [JsonPropertyName("remux")] + public bool Remux { get; set; } + + [JsonPropertyName("upscaled")] + public bool Upscaled { get; set; } + + [JsonPropertyName("remastered")] + public bool Remastered { get; set; } + + [JsonPropertyName("directors_cut")] + public bool DirectorsCut { get; set; } + + [JsonPropertyName("extended")] + public bool Extended { get; set; } + + public bool IsMovie => Season == null && Episode == null; + + public string ToJson() => this.AsJson(); +} \ No newline at end of file diff --git a/src/torrent-consumer/Features/TorrentIngestion/PerformIngestionConsumer.cs b/src/torrent-consumer/Features/TorrentIngestion/PerformIngestionConsumer.cs index dc20d0e..8a361e3 100644 --- a/src/torrent-consumer/Features/TorrentIngestion/PerformIngestionConsumer.cs +++ b/src/torrent-consumer/Features/TorrentIngestion/PerformIngestionConsumer.cs @@ -11,6 +11,7 @@ public class PerformIngestionConsumer(IDataStorage dataStorage, ILogger