From c493ef3376caeb6a17ce3a76a4b894c434cd8fe2 Mon Sep 17 00:00:00 2001 From: iPromKnight <156901906+iPromKnight@users.noreply.github.com> Date: Sat, 30 Mar 2024 04:47:36 +0000 Subject: [PATCH] Hotfix category, and roll back RTN to 0.1.8 (#192) * Hotfix categories Also roll back RTN to 0.1.8 as regression introduced in 0.2 * bump version --- deployment/docker/docker-compose.yaml | 14 ++++---- .../docker/src/components/knightcrawler.yaml | 14 ++++---- .../Crawlers/Dmm/DebridMediaManagerCrawler.cs | 34 +++++++++---------- src/producer/src/requirements.txt | 2 +- src/shared/Dapper/DapperDataStorage.cs | 2 +- src/shared/Extensions/StringExtensions.cs | 2 -- src/shared/GlobalUsings.cs | 1 + src/shared/Python/RTN/RankTorrentName.cs | 3 +- src/shared/Python/RTN/RtnResponse.cs | 17 +++++++++- 9 files changed, 51 insertions(+), 38 deletions(-) diff --git a/deployment/docker/docker-compose.yaml b/deployment/docker/docker-compose.yaml index 5855258..eeae9ae 100644 --- a/deployment/docker/docker-compose.yaml +++ b/deployment/docker/docker-compose.yaml @@ -94,7 +94,7 @@ services: condition: service_healthy env_file: stack.env hostname: knightcrawler-addon - image: gabisonfire/knightcrawler-addon:2.0.15 + image: gabisonfire/knightcrawler-addon:2.0.16 labels: logging: promtail networks: @@ -117,7 +117,7 @@ services: redis: condition: service_healthy env_file: stack.env - image: gabisonfire/knightcrawler-consumer:2.0.15 + image: gabisonfire/knightcrawler-consumer:2.0.16 labels: logging: promtail networks: @@ -138,7 +138,7 @@ services: redis: condition: service_healthy env_file: stack.env - image: gabisonfire/knightcrawler-debrid-collector:2.0.15 + image: gabisonfire/knightcrawler-debrid-collector:2.0.16 labels: logging: promtail networks: @@ -152,7 +152,7 @@ services: migrator: condition: service_completed_successfully env_file: stack.env - image: gabisonfire/knightcrawler-metadata:2.0.15 + image: gabisonfire/knightcrawler-metadata:2.0.16 networks: - knightcrawler-network restart: "no" @@ -163,7 +163,7 @@ services: postgres: condition: service_healthy env_file: stack.env - image: gabisonfire/knightcrawler-migrator:2.0.15 + image: gabisonfire/knightcrawler-migrator:2.0.16 networks: - knightcrawler-network restart: "no" @@ -182,7 +182,7 @@ services: redis: condition: service_healthy env_file: stack.env - image: gabisonfire/knightcrawler-producer:2.0.15 + image: gabisonfire/knightcrawler-producer:2.0.16 labels: logging: promtail networks: @@ -207,7 +207,7 @@ services: deploy: replicas: ${QBIT_REPLICAS:-0} env_file: stack.env - image: gabisonfire/knightcrawler-qbit-collector:2.0.15 + image: gabisonfire/knightcrawler-qbit-collector:2.0.16 labels: logging: promtail networks: diff --git a/deployment/docker/src/components/knightcrawler.yaml b/deployment/docker/src/components/knightcrawler.yaml index 1eb2b4a..e0e3a66 100644 --- a/deployment/docker/src/components/knightcrawler.yaml +++ b/deployment/docker/src/components/knightcrawler.yaml @@ -20,7 +20,7 @@ x-depends: &knightcrawler-app-depends services: metadata: - image: gabisonfire/knightcrawler-metadata:2.0.15 + image: gabisonfire/knightcrawler-metadata:2.0.16 env_file: ../../.env networks: - knightcrawler-network @@ -30,7 +30,7 @@ services: condition: service_completed_successfully migrator: - image: gabisonfire/knightcrawler-migrator:2.0.15 + image: gabisonfire/knightcrawler-migrator:2.0.16 env_file: ../../.env networks: - knightcrawler-network @@ -40,7 +40,7 @@ services: condition: service_healthy addon: - image: gabisonfire/knightcrawler-addon:2.0.15 + image: gabisonfire/knightcrawler-addon:2.0.16 <<: [*knightcrawler-app, *knightcrawler-app-depends] restart: unless-stopped hostname: knightcrawler-addon @@ -48,22 +48,22 @@ services: - "7000:7000" consumer: - image: gabisonfire/knightcrawler-consumer:2.0.15 + image: gabisonfire/knightcrawler-consumer:2.0.16 <<: [*knightcrawler-app, *knightcrawler-app-depends] restart: unless-stopped debridcollector: - image: gabisonfire/knightcrawler-debrid-collector:2.0.15 + image: gabisonfire/knightcrawler-debrid-collector:2.0.16 <<: [*knightcrawler-app, *knightcrawler-app-depends] restart: unless-stopped producer: - image: gabisonfire/knightcrawler-producer:2.0.15 + image: gabisonfire/knightcrawler-producer:2.0.16 <<: [*knightcrawler-app, *knightcrawler-app-depends] restart: unless-stopped qbitcollector: - image: gabisonfire/knightcrawler-qbit-collector:2.0.15 + image: gabisonfire/knightcrawler-qbit-collector:2.0.16 <<: [*knightcrawler-app, *knightcrawler-app-depends] restart: unless-stopped depends_on: diff --git a/src/producer/src/Features/Crawlers/Dmm/DebridMediaManagerCrawler.cs b/src/producer/src/Features/Crawlers/Dmm/DebridMediaManagerCrawler.cs index 431e44b..1a6dd70 100644 --- a/src/producer/src/Features/Crawlers/Dmm/DebridMediaManagerCrawler.cs +++ b/src/producer/src/Features/Crawlers/Dmm/DebridMediaManagerCrawler.cs @@ -108,14 +108,18 @@ public partial class DebridMediaManagerCrawler( return null; } - var parsedTorrent = rankTorrentName.Parse(torrentTitle.CleanTorrentTitleForImdb()); + var parsedTorrent = rankTorrentName.Parse(torrentTitle); if (!parsedTorrent.Success) { return null; } - var (cached, cachedResult) = await CheckIfInCacheAndReturn(parsedTorrent.Response.ParsedTitle); + var torrentType = parsedTorrent.Response.IsMovie ? "movie" : "tvSeries"; + + var cacheKey = GetCacheKey(torrentType, parsedTorrent.Response.ParsedTitle, parsedTorrent.Response.Year); + + var (cached, cachedResult) = await CheckIfInCacheAndReturn(cacheKey); if (cached) { @@ -124,14 +128,14 @@ public partial class DebridMediaManagerCrawler( } int? year = parsedTorrent.Response.Year != 0 ? parsedTorrent.Response.Year : null; - var imdbEntry = await Storage.FindImdbMetadata(parsedTorrent.Response.ParsedTitle, parsedTorrent.Response.IsMovie ? "movies" : "tv", year); + var imdbEntry = await Storage.FindImdbMetadata(parsedTorrent.Response.ParsedTitle, torrentType, year); if (imdbEntry is null) { return null; } - await AddToCache(parsedTorrent.Response.ParsedTitle.ToLowerInvariant(), imdbEntry); + await AddToCache(cacheKey, imdbEntry); logger.LogInformation("[{ImdbId}] Found best match for {Title}: {BestMatch} with score {Score}", imdbEntry.ImdbId, parsedTorrent.Response.ParsedTitle, imdbEntry.Title, imdbEntry.Score); @@ -153,19 +157,19 @@ public partial class DebridMediaManagerCrawler( }; - private Task AddToCache(string lowerCaseTitle, ImdbEntry best) + private Task AddToCache(string cacheKey, ImdbEntry best) { var cacheOptions = new DistributedCacheEntryOptions { AbsoluteExpirationRelativeToNow = TimeSpan.FromDays(1), }; - return cache.SetStringAsync(lowerCaseTitle, JsonSerializer.Serialize(best), cacheOptions); + return cache.SetStringAsync(cacheKey, JsonSerializer.Serialize(best), cacheOptions); } - private async Task<(bool Success, ImdbEntry? Entry)> CheckIfInCacheAndReturn(string title) + private async Task<(bool Success, ImdbEntry? Entry)> CheckIfInCacheAndReturn(string cacheKey) { - var cachedImdbId = await cache.GetStringAsync(title.ToLowerInvariant()); + var cachedImdbId = await cache.GetStringAsync(cacheKey); if (!string.IsNullOrEmpty(cachedImdbId)) { @@ -207,16 +211,12 @@ public partial class DebridMediaManagerCrawler( } private static string AssignCategory(ImdbEntry entry) => - entry.Category switch + entry.Category.ToLower() switch { - "movie" => "movies", - "tvMovie" => "movies", - "tvSeries" => "tv", - "tvEpisode" => "tv", - "tvSpecial" => "tv", - "tvMiniSeries" => "tv", - "tv" => "tv", - "short" => "tv", + var category when string.Equals(category, "movie", StringComparison.OrdinalIgnoreCase) => "movies", + var category when string.Equals(category, "tvSeries", StringComparison.OrdinalIgnoreCase) => "tv", _ => "unknown", }; + + private static string GetCacheKey(string category, string title, int year) => $"{category.ToLowerInvariant()}:{year}:{title.ToLowerInvariant()}"; } diff --git a/src/producer/src/requirements.txt b/src/producer/src/requirements.txt index 9256344..7d700a4 100644 --- a/src/producer/src/requirements.txt +++ b/src/producer/src/requirements.txt @@ -1 +1 @@ -rank-torrent-name==0.1.9 \ No newline at end of file +rank-torrent-name==0.1.8 \ No newline at end of file diff --git a/src/shared/Dapper/DapperDataStorage.cs b/src/shared/Dapper/DapperDataStorage.cs index 3609b68..828b4ec 100644 --- a/src/shared/Dapper/DapperDataStorage.cs +++ b/src/shared/Dapper/DapperDataStorage.cs @@ -118,7 +118,7 @@ public class DapperDataStorage(PostgresConfiguration configuration, RabbitMqConf public async Task FindImdbMetadata(string? parsedTorrentTitle, string torrentType, int? year, CancellationToken cancellationToken = default) => await ExecuteCommandAsync(async connection => { - var query = $"select \"imdb_id\" as \"ImdbId\", \"title\" as \"Title\", \"year\" as \"Year\", \"score\" as Score from search_imdb_meta('{parsedTorrentTitle.Replace("'", "").Replace("\"", "")}', '{(torrentType.Equals("movie", StringComparison.OrdinalIgnoreCase) ? "movie" : "tvSeries")}'"; + var query = $"select \"imdb_id\" as \"ImdbId\", \"title\" as \"Title\", \"year\" as \"Year\", \"score\" as Score, \"category\" as Category from search_imdb_meta('{parsedTorrentTitle.Replace("'", "").Replace("\"", "")}', '{torrentType}'"; query += year is not null ? $", {year}" : ", NULL"; query += ", 1)"; diff --git a/src/shared/Extensions/StringExtensions.cs b/src/shared/Extensions/StringExtensions.cs index 374cbc7..656c120 100644 --- a/src/shared/Extensions/StringExtensions.cs +++ b/src/shared/Extensions/StringExtensions.cs @@ -1,5 +1,3 @@ -using System.Text.RegularExpressions; - namespace SharedContracts.Extensions; public static partial class StringExtensions diff --git a/src/shared/GlobalUsings.cs b/src/shared/GlobalUsings.cs index 1eb535e..efaf674 100644 --- a/src/shared/GlobalUsings.cs +++ b/src/shared/GlobalUsings.cs @@ -2,6 +2,7 @@ global using System.Text.Json; global using System.Text.Json.Serialization; +global using System.Text.RegularExpressions; global using Dapper; global using MassTransit; global using Microsoft.AspNetCore.Builder; diff --git a/src/shared/Python/RTN/RankTorrentName.cs b/src/shared/Python/RTN/RankTorrentName.cs index 723bf1a..8c78e68 100644 --- a/src/shared/Python/RTN/RankTorrentName.cs +++ b/src/shared/Python/RTN/RankTorrentName.cs @@ -20,8 +20,7 @@ public class RankTorrentName : IRankTorrentName var result = _rtn?.parse(title); return ParseResult(result); }, new ParseTorrentTitleResponse(false, null), nameof(Parse), throwOnErrors: false, logErrors: false); - - + private static ParseTorrentTitleResponse ParseResult(dynamic result) { if (result == null) diff --git a/src/shared/Python/RTN/RtnResponse.cs b/src/shared/Python/RTN/RtnResponse.cs index 9fd81d2..d2e1d85 100644 --- a/src/shared/Python/RTN/RtnResponse.cs +++ b/src/shared/Python/RTN/RtnResponse.cs @@ -77,7 +77,22 @@ public class RtnResponse [JsonPropertyName("extended")] public bool Extended { get; set; } - public bool IsMovie => (Season == null && Episode == null) || (Season?.Count == 0 && Episode?.Count == 0); + // [JsonPropertyName("is_show")] + // public bool IsTvShow { get; set; } + // + // [JsonPropertyName("is_movie")] + // public bool IsMovie { get; set; } public string ToJson() => this.AsJson(); + + public bool IsMovie => !TvRegexes.Any(regex => regex.IsMatch(RawTitle)) && Season?.Count == 0 && Episode?.Count == 0; + + private static List TvRegexes { get; set; } = + [ + new(@"[se]\d\d", RegexOptions.IgnoreCase), + new(@"\b(tv|complete)\b", RegexOptions.IgnoreCase), + new(@"\b(saisons?|stages?|seasons?).?\d", RegexOptions.IgnoreCase), + new(@"[a-z]\s?\-\s?\d{2,4}\b", RegexOptions.IgnoreCase), + new(@"\d{2,4}\s?\-\s?\d{2,4}\b", RegexOptions.IgnoreCase), + ]; } \ No newline at end of file