Hotfix category, and roll back RTN to 0.1.8 (#192)

* Hotfix categories

Also roll back RTN to 0.1.8 as regression introduced in 0.2

* bump version

deployment/docker/docker-compose.yaml

@@ -94,7 +94,7 @@ services:
         condition: service_healthy
     env_file: stack.env
     hostname: knightcrawler-addon
-    image: gabisonfire/knightcrawler-addon:2.0.13
+    image: gabisonfire/knightcrawler-addon:2.0.16
     labels:
       logging: promtail
     networks:
@@ -117,7 +117,7 @@ services:
       redis:
         condition: service_healthy
     env_file: stack.env
-    image: gabisonfire/knightcrawler-consumer:2.0.13
+    image: gabisonfire/knightcrawler-consumer:2.0.16
     labels:
       logging: promtail
     networks:
@@ -138,7 +138,7 @@ services:
       redis:
         condition: service_healthy
     env_file: stack.env
-    image: gabisonfire/knightcrawler-debrid-collector:2.0.13
+    image: gabisonfire/knightcrawler-debrid-collector:2.0.16
     labels:
       logging: promtail
     networks:
@@ -152,7 +152,7 @@ services:
       migrator:
         condition: service_completed_successfully
     env_file: stack.env
-    image: gabisonfire/knightcrawler-metadata:2.0.13
+    image: gabisonfire/knightcrawler-metadata:2.0.16
     networks:
       - knightcrawler-network
     restart: "no"
@@ -163,7 +163,7 @@ services:
       postgres:
         condition: service_healthy
     env_file: stack.env
-    image: gabisonfire/knightcrawler-migrator:2.0.13
+    image: gabisonfire/knightcrawler-migrator:2.0.16
     networks:
       - knightcrawler-network
     restart: "no"
@@ -182,7 +182,7 @@ services:
       redis:
         condition: service_healthy
     env_file: stack.env
-    image: gabisonfire/knightcrawler-producer:2.0.13
+    image: gabisonfire/knightcrawler-producer:2.0.16
     labels:
       logging: promtail
     networks:
@@ -207,7 +207,7 @@ services:
     deploy:
       replicas: ${QBIT_REPLICAS:-0}
     env_file: stack.env
-    image: gabisonfire/knightcrawler-qbit-collector:2.0.13
+    image: gabisonfire/knightcrawler-qbit-collector:2.0.16
     labels:
       logging: promtail
     networks:

deployment/docker/src/components/knightcrawler.yaml

@@ -20,7 +20,7 @@ x-depends: &knightcrawler-app-depends
 
 services:
   metadata:
-    image: gabisonfire/knightcrawler-metadata:2.0.13
+    image: gabisonfire/knightcrawler-metadata:2.0.16
     env_file: ../../.env
     networks:
       - knightcrawler-network
@@ -30,7 +30,7 @@ services:
         condition: service_completed_successfully
 
   migrator:
-    image: gabisonfire/knightcrawler-migrator:2.0.13
+    image: gabisonfire/knightcrawler-migrator:2.0.16
     env_file: ../../.env
     networks:
       - knightcrawler-network
@@ -40,7 +40,7 @@ services:
         condition: service_healthy
 
   addon:
-    image: gabisonfire/knightcrawler-addon:2.0.13
+    image: gabisonfire/knightcrawler-addon:2.0.16
     <<: [*knightcrawler-app, *knightcrawler-app-depends]
     restart: unless-stopped
     hostname: knightcrawler-addon
@@ -48,22 +48,22 @@ services:
       - "7000:7000"
 
   consumer:
-    image: gabisonfire/knightcrawler-consumer:2.0.13
+    image: gabisonfire/knightcrawler-consumer:2.0.16
     <<: [*knightcrawler-app, *knightcrawler-app-depends]
     restart: unless-stopped
 
   debridcollector:
-    image: gabisonfire/knightcrawler-debrid-collector:2.0.13
+    image: gabisonfire/knightcrawler-debrid-collector:2.0.16
     <<: [*knightcrawler-app, *knightcrawler-app-depends]
     restart: unless-stopped
         
   producer:
-    image: gabisonfire/knightcrawler-producer:2.0.13
+    image: gabisonfire/knightcrawler-producer:2.0.16
     <<: [*knightcrawler-app, *knightcrawler-app-depends]
     restart: unless-stopped
 
   qbitcollector:
-    image: gabisonfire/knightcrawler-qbit-collector:2.0.13
+    image: gabisonfire/knightcrawler-qbit-collector:2.0.16
     <<: [*knightcrawler-app, *knightcrawler-app-depends]
     restart: unless-stopped
     depends_on:

src/migrator/migrations/012_imdb_change_ratio.sql

@@ -0,0 +1,19 @@
+-- Remove the old search function
+DROP FUNCTION IF EXISTS search_imdb_meta(TEXT, TEXT, INT, INT);
+
+-- Add the new search function that allows for searching by year with a plus/minus one year range
+CREATE OR REPLACE FUNCTION search_imdb_meta(search_term TEXT, category_param TEXT DEFAULT NULL, year_param INT DEFAULT NULL, limit_param INT DEFAULT 10, similarity_threshold REAL DEFAULT 0.95)
+    RETURNS TABLE(imdb_id character varying(16), title character varying(1000),category character varying(50),year INT, score REAL) AS $$
+BEGIN
+    SET pg_trgm.similarity_threshold = similarity_threshold;
+    RETURN QUERY
+        SELECT imdb_metadata.imdb_id, imdb_metadata.title, imdb_metadata.category, imdb_metadata.year, similarity(imdb_metadata.title, search_term) as score
+        FROM imdb_metadata
+        WHERE (imdb_metadata.title % search_term)
+          AND (imdb_metadata.adult = FALSE)
+          AND (category_param IS NULL OR imdb_metadata.category = category_param)
+          AND (year_param IS NULL OR imdb_metadata.year BETWEEN year_param - 1 AND year_param + 1)
+        ORDER BY score DESC
+        LIMIT limit_param;
+END; $$
+    LANGUAGE plpgsql;

src/migrator/migrations/013_imdb_change_ratio_fix.sql

@@ -0,0 +1,19 @@
+-- Remove the old search function
+DROP FUNCTION IF EXISTS search_imdb_meta(TEXT, TEXT, INT, INT);
+
+-- Add the new search function that allows for searching by year with a plus/minus one year range
+CREATE OR REPLACE FUNCTION search_imdb_meta(search_term TEXT, category_param TEXT DEFAULT NULL, year_param INT DEFAULT NULL, limit_param INT DEFAULT 10, similarity_threshold REAL DEFAULT 0.95)
+    RETURNS TABLE(imdb_id character varying(16), title character varying(1000),category character varying(50),year INT, score REAL) AS $$
+BEGIN
+    EXECUTE format('SET pg_trgm.similarity_threshold = %L', similarity_threshold);
+    RETURN QUERY
+        SELECT imdb_metadata.imdb_id, imdb_metadata.title, imdb_metadata.category, imdb_metadata.year, similarity(imdb_metadata.title, search_term) as score
+        FROM imdb_metadata
+        WHERE (imdb_metadata.title % search_term)
+          AND (imdb_metadata.adult = FALSE)
+          AND (category_param IS NULL OR imdb_metadata.category = category_param)
+          AND (year_param IS NULL OR imdb_metadata.year BETWEEN year_param - 1 AND year_param + 1)
+        ORDER BY score DESC
+        LIMIT limit_param;
+END; $$
+    LANGUAGE plpgsql;

src/producer/src/Features/Crawlers/Dmm/DebridMediaManagerCrawler.cs

@@ -108,14 +108,18 @@ public partial class DebridMediaManagerCrawler(
             return null;
         }
         
-        var parsedTorrent = rankTorrentName.Parse(torrentTitle.CleanTorrentTitleForImdb());
+        var parsedTorrent = rankTorrentName.Parse(torrentTitle);
         
         if (!parsedTorrent.Success)
         {
             return null;
         }
         
-        var (cached, cachedResult) = await CheckIfInCacheAndReturn(parsedTorrent.Response.ParsedTitle);
+        var torrentType = parsedTorrent.Response.IsMovie ? "movie" : "tvSeries";
+        
+        var cacheKey = GetCacheKey(torrentType, parsedTorrent.Response.ParsedTitle, parsedTorrent.Response.Year);
+        
+        var (cached, cachedResult) = await CheckIfInCacheAndReturn(cacheKey);
         
         if (cached)
         {
@@ -124,14 +128,14 @@ public partial class DebridMediaManagerCrawler(
         }
 
         int? year = parsedTorrent.Response.Year != 0 ? parsedTorrent.Response.Year : null;
-        var imdbEntry = await Storage.FindImdbMetadata(parsedTorrent.Response.ParsedTitle, parsedTorrent.Response.IsMovie ? "movies" : "tv", year);
+        var imdbEntry = await Storage.FindImdbMetadata(parsedTorrent.Response.ParsedTitle, torrentType, year);
 
         if (imdbEntry is null)
         {
             return null;
         }
         
-        await AddToCache(parsedTorrent.Response.ParsedTitle.ToLowerInvariant(), imdbEntry);
+        await AddToCache(cacheKey, imdbEntry);
         
         logger.LogInformation("[{ImdbId}] Found best match for {Title}: {BestMatch} with score {Score}", imdbEntry.ImdbId, parsedTorrent.Response.ParsedTitle, imdbEntry.Title, imdbEntry.Score);
 
@@ -153,19 +157,19 @@ public partial class DebridMediaManagerCrawler(
         };
     
 
-    private Task AddToCache(string lowerCaseTitle, ImdbEntry best)
+    private Task AddToCache(string cacheKey, ImdbEntry best)
     {
         var cacheOptions = new DistributedCacheEntryOptions
         {
             AbsoluteExpirationRelativeToNow = TimeSpan.FromDays(1),
         };
         
-        return cache.SetStringAsync(lowerCaseTitle, JsonSerializer.Serialize(best), cacheOptions);
+        return cache.SetStringAsync(cacheKey, JsonSerializer.Serialize(best), cacheOptions);
     }
 
-    private async Task<(bool Success, ImdbEntry? Entry)> CheckIfInCacheAndReturn(string title)
+    private async Task<(bool Success, ImdbEntry? Entry)> CheckIfInCacheAndReturn(string cacheKey)
     {
-        var cachedImdbId = await cache.GetStringAsync(title.ToLowerInvariant());
+        var cachedImdbId = await cache.GetStringAsync(cacheKey);
         
         if (!string.IsNullOrEmpty(cachedImdbId))
         {
@@ -207,16 +211,12 @@ public partial class DebridMediaManagerCrawler(
     }
     
     private static string AssignCategory(ImdbEntry entry) =>
-        entry.Category switch
+        entry.Category.ToLower() switch
         {
-            "movie" => "movies",
-            "tvMovie" => "movies",
-            "tvSeries" => "tv",
-            "tvEpisode" => "tv",
-            "tvSpecial" => "tv",
-            "tvMiniSeries" => "tv",
-            "tv" => "tv",
-            "short" => "tv",
+            var category when string.Equals(category, "movie", StringComparison.OrdinalIgnoreCase) => "movies",
+            var category when string.Equals(category, "tvSeries", StringComparison.OrdinalIgnoreCase) => "tv",
             _ => "unknown",
         };
+    
+    private static string GetCacheKey(string category, string title, int year) => $"{category.ToLowerInvariant()}:{year}:{title.ToLowerInvariant()}";
 }

src/producer/src/requirements.txt

@@ -1 +1 @@
-rank-torrent-name==0.1.9
+rank-torrent-name==0.1.8

src/shared/Dapper/DapperDataStorage.cs

@@ -118,7 +118,7 @@ public class DapperDataStorage(PostgresConfiguration configuration, RabbitMqConf
     public async Task<ImdbEntry?> FindImdbMetadata(string? parsedTorrentTitle, string torrentType, int? year, CancellationToken cancellationToken = default) =>
         await ExecuteCommandAsync(async connection =>
         {
-            var query = $"select \"imdb_id\" as \"ImdbId\", \"title\" as \"Title\", \"year\" as \"Year\", \"score\" as Score from search_imdb_meta('{parsedTorrentTitle.Replace("'", "").Replace("\"", "")}', '{(torrentType.Equals("movie", StringComparison.OrdinalIgnoreCase) ? "movie" : "tvSeries")}'";
+            var query = $"select \"imdb_id\" as \"ImdbId\", \"title\" as \"Title\", \"year\" as \"Year\", \"score\" as Score, \"category\" as Category from search_imdb_meta('{parsedTorrentTitle.Replace("'", "").Replace("\"", "")}', '{torrentType}'";
             query += year is not null ? $", {year}" : ", NULL";
             query += ", 1)";
             

src/shared/Extensions/StringExtensions.cs

@@ -1,5 +1,3 @@
-using System.Text.RegularExpressions;
-
 namespace SharedContracts.Extensions;
 
 public static partial class StringExtensions

src/shared/GlobalUsings.cs

@@ -2,6 +2,7 @@
 
 global using System.Text.Json;
 global using System.Text.Json.Serialization;
+global using System.Text.RegularExpressions;
 global using Dapper;
 global using MassTransit;
 global using Microsoft.AspNetCore.Builder;

src/shared/Python/RTN/RankTorrentName.cs

@@ -20,8 +20,7 @@ public class RankTorrentName : IRankTorrentName
                 var result = _rtn?.parse(title);
                 return ParseResult(result);
             }, new ParseTorrentTitleResponse(false, null), nameof(Parse), throwOnErrors: false, logErrors: false);
-
-
+    
     private static ParseTorrentTitleResponse ParseResult(dynamic result)
     {
         if (result == null)

src/shared/Python/RTN/RtnResponse.cs

@@ -77,7 +77,22 @@ public class RtnResponse
     [JsonPropertyName("extended")]
     public bool Extended { get; set; }
     
-    public bool IsMovie => (Season == null && Episode == null) || (Season?.Count == 0 && Episode?.Count == 0);
+    // [JsonPropertyName("is_show")]
+    // public bool IsTvShow { get; set; }
+    //
+    // [JsonPropertyName("is_movie")]
+    // public bool IsMovie { get; set; }
     
     public string ToJson() => this.AsJson();
+    
+    public bool IsMovie => !TvRegexes.Any(regex => regex.IsMatch(RawTitle)) && Season?.Count == 0 && Episode?.Count == 0;
+    
+    private static List<Regex> TvRegexes { get; set; } =
+    [
+        new(@"[se]\d\d", RegexOptions.IgnoreCase),
+        new(@"\b(tv|complete)\b", RegexOptions.IgnoreCase),
+        new(@"\b(saisons?|stages?|seasons?).?\d", RegexOptions.IgnoreCase),
+        new(@"[a-z]\s?\-\s?\d{2,4}\b", RegexOptions.IgnoreCase),
+        new(@"\d{2,4}\s?\-\s?\d{2,4}\b", RegexOptions.IgnoreCase),
+    ];
 }