From 3c8ffd5082ddb2b48b568b731e61dfdb03217546 Mon Sep 17 00:00:00 2001
From: iPromKnight <156901906+iPromKnight@users.noreply.github.com>
Date: Tue, 2 Apr 2024 20:31:22 +0100
Subject: [PATCH] Fix Duplicates (#199)

* Fix Duplicates

* Version
---
 deployment/docker/docker-compose.yaml         | 14 +++---
 .../docker/src/components/knightcrawler.yaml  | 14 +++---
 ...4_file_subtitle_infohash_fileidx_index.sql | 43 +++++++++++++++++++
 src/shared/Dapper/DapperDataStorage.cs        |  6 ++-
 4 files changed, 61 insertions(+), 16 deletions(-)
 create mode 100644 src/migrator/migrations/014_file_subtitle_infohash_fileidx_index.sql

diff --git a/deployment/docker/docker-compose.yaml b/deployment/docker/docker-compose.yaml
index 26f6942..b20ab08 100644
--- a/deployment/docker/docker-compose.yaml
+++ b/deployment/docker/docker-compose.yaml
@@ -94,7 +94,7 @@ services:
         condition: service_healthy
     env_file: stack.env
     hostname: knightcrawler-addon
-    image: gabisonfire/knightcrawler-addon:2.0.20
+    image: gabisonfire/knightcrawler-addon:2.0.21
     labels:
       logging: promtail
     networks:
@@ -117,7 +117,7 @@ services:
       redis:
         condition: service_healthy
     env_file: stack.env
-    image: gabisonfire/knightcrawler-consumer:2.0.20
+    image: gabisonfire/knightcrawler-consumer:2.0.21
     labels:
       logging: promtail
     networks:
@@ -138,7 +138,7 @@ services:
       redis:
         condition: service_healthy
     env_file: stack.env
-    image: gabisonfire/knightcrawler-debrid-collector:2.0.20
+    image: gabisonfire/knightcrawler-debrid-collector:2.0.21
     labels:
       logging: promtail
     networks:
@@ -152,7 +152,7 @@ services:
       migrator:
         condition: service_completed_successfully
     env_file: stack.env
-    image: gabisonfire/knightcrawler-metadata:2.0.20
+    image: gabisonfire/knightcrawler-metadata:2.0.21
     networks:
       - knightcrawler-network
     restart: "no"
@@ -163,7 +163,7 @@ services:
       postgres:
         condition: service_healthy
     env_file: stack.env
-    image: gabisonfire/knightcrawler-migrator:2.0.20
+    image: gabisonfire/knightcrawler-migrator:2.0.21
     networks:
       - knightcrawler-network
     restart: "no"
@@ -182,7 +182,7 @@ services:
       redis:
         condition: service_healthy
     env_file: stack.env
-    image: gabisonfire/knightcrawler-producer:2.0.20
+    image: gabisonfire/knightcrawler-producer:2.0.21
     labels:
       logging: promtail
     networks:
@@ -207,7 +207,7 @@ services:
     deploy:
       replicas: ${QBIT_REPLICAS:-0}
     env_file: stack.env
-    image: gabisonfire/knightcrawler-qbit-collector:2.0.20
+    image: gabisonfire/knightcrawler-qbit-collector:2.0.21
     labels:
       logging: promtail
     networks:
diff --git a/deployment/docker/src/components/knightcrawler.yaml b/deployment/docker/src/components/knightcrawler.yaml
index e4164b1..eab5326 100644
--- a/deployment/docker/src/components/knightcrawler.yaml
+++ b/deployment/docker/src/components/knightcrawler.yaml
@@ -20,7 +20,7 @@ x-depends: &knightcrawler-app-depends
 
 services:
   metadata:
-    image: gabisonfire/knightcrawler-metadata:2.0.20
+    image: gabisonfire/knightcrawler-metadata:2.0.21
     env_file: ../../.env
     networks:
       - knightcrawler-network
@@ -30,7 +30,7 @@ services:
         condition: service_completed_successfully
 
   migrator:
-    image: gabisonfire/knightcrawler-migrator:2.0.20
+    image: gabisonfire/knightcrawler-migrator:2.0.21
     env_file: ../../.env
     networks:
       - knightcrawler-network
@@ -40,7 +40,7 @@ services:
         condition: service_healthy
 
   addon:
-    image: gabisonfire/knightcrawler-addon:2.0.20
+    image: gabisonfire/knightcrawler-addon:2.0.21
     <<: [*knightcrawler-app, *knightcrawler-app-depends]
     restart: unless-stopped
     hostname: knightcrawler-addon
@@ -48,22 +48,22 @@ services:
       - "7000:7000"
 
   consumer:
-    image: gabisonfire/knightcrawler-consumer:2.0.20
+    image: gabisonfire/knightcrawler-consumer:2.0.21
     <<: [*knightcrawler-app, *knightcrawler-app-depends]
     restart: unless-stopped
 
   debridcollector:
-    image: gabisonfire/knightcrawler-debrid-collector:2.0.20
+    image: gabisonfire/knightcrawler-debrid-collector:2.0.21
     <<: [*knightcrawler-app, *knightcrawler-app-depends]
     restart: unless-stopped
         
   producer:
-    image: gabisonfire/knightcrawler-producer:2.0.20
+    image: gabisonfire/knightcrawler-producer:2.0.21
     <<: [*knightcrawler-app, *knightcrawler-app-depends]
     restart: unless-stopped
 
   qbitcollector:
-    image: gabisonfire/knightcrawler-qbit-collector:2.0.20
+    image: gabisonfire/knightcrawler-qbit-collector:2.0.21
     <<: [*knightcrawler-app, *knightcrawler-app-depends]
     restart: unless-stopped
     depends_on:
diff --git a/src/migrator/migrations/014_file_subtitle_infohash_fileidx_index.sql b/src/migrator/migrations/014_file_subtitle_infohash_fileidx_index.sql
new file mode 100644
index 0000000..e69c873
--- /dev/null
+++ b/src/migrator/migrations/014_file_subtitle_infohash_fileidx_index.sql
@@ -0,0 +1,43 @@
+-- Drop Duplicate Files in Files Table
+DELETE FROM public.files
+WHERE id NOT IN (
+    SELECT MAX(id)
+    FROM public.files
+    GROUP BY "infoHash", "fileIndex"
+);
+
+-- Add Index to files table
+DO $$
+BEGIN
+    IF NOT EXISTS (
+        SELECT 1
+        FROM pg_constraint
+        WHERE conname = 'files_unique_infohash_fileindex'
+    ) THEN
+        ALTER TABLE public.files
+            ADD CONSTRAINT files_unique_infohash_fileindex UNIQUE ("infoHash", "fileIndex");
+    END IF;
+END $$;
+
+
+-- Drop Duplicate subtitles in Subtitles Table
+DELETE FROM public.subtitles
+WHERE id NOT IN (
+    SELECT MAX(id)
+    FROM public.subtitles
+    GROUP BY "infoHash", "fileIndex"
+);
+
+-- Add Index to subtitles table
+DO $$
+    BEGIN
+        IF NOT EXISTS (
+            SELECT 1
+            FROM pg_constraint
+            WHERE conname = 'subtitles_unique_infohash_fileindex'
+        ) THEN
+            ALTER TABLE public.subtitles
+                ADD CONSTRAINT subtitles_unique_infohash_fileindex UNIQUE ("infoHash", "fileIndex");
+        END IF;
+    END $$;
+
diff --git a/src/shared/Dapper/DapperDataStorage.cs b/src/shared/Dapper/DapperDataStorage.cs
index 5cc710c..0d3fd63 100644
--- a/src/shared/Dapper/DapperDataStorage.cs
+++ b/src/shared/Dapper/DapperDataStorage.cs
@@ -152,7 +152,8 @@ public class DapperDataStorage(PostgresConfiguration configuration, RabbitMqConf
                     INSERT INTO files
                         ("infoHash", "fileIndex", title, "size", "imdbId", "imdbSeason", "imdbEpisode", "kitsuId", "kitsuEpisode", "createdAt", "updatedAt")
                     VALUES
-                        (@InfoHash, @FileIndex, @Title, @Size, @ImdbId, @ImdbSeason, @ImdbEpisode, @KitsuId, @KitsuEpisode, Now(), Now());
+                        (@InfoHash, @FileIndex, @Title, @Size, @ImdbId, @ImdbSeason, @ImdbEpisode, @KitsuId, @KitsuEpisode, Now(), Now())
+                    ON CONFLICT ("infoHash", "fileIndex") DO NOTHING;
                     """;
                 
                 await connection.ExecuteAsync(query, files);
@@ -167,7 +168,8 @@ public class DapperDataStorage(PostgresConfiguration configuration, RabbitMqConf
                     INSERT INTO subtitles
                         ("infoHash", "fileIndex", "fileId", "title")
                     VALUES
-                        (@InfoHash, @FileIndex, @FileId, @Title);
+                        (@InfoHash, @FileIndex, @FileId, @Title)
+                    ON CONFLICT ("infoHash", "fileIndex") DO NOTHING;
                     """;
                 
                 await connection.ExecuteAsync(query, subtitles);