Fix DMM parser and a couple of other minor issues. (#226)

src/debrid-collector/Dockerfile

@@ -15,7 +15,7 @@ WORKDIR /app
 
 ENV PYTHONUNBUFFERED=1
 
-RUN apk add --update --no-cache python3=~3.11.9-r0 py3-pip && ln -sf python3 /usr/bin/python
+RUN apk add --update --no-cache python3=~3.11 py3-pip && ln -sf python3 /usr/bin/python
 
 COPY --from=build /src/out .
 

src/metadata/Features/Configuration/PostgresConfiguration.cs

@@ -8,12 +8,14 @@ public class PostgresConfiguration
     private const string PasswordVariable = "PASSWORD";
     private const string DatabaseVariable = "DB";
     private const string PortVariable = "PORT";
+    private const string CommandTimeoutVariable = "COMMAND_TIMEOUT_SEC"; // Seconds
 
     private string Host { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(HostVariable);
     private string Username { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(UsernameVariable);
     private string Password { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(PasswordVariable);
     private string Database { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(DatabaseVariable);
     private int PORT { get; init; } = Prefix.GetEnvironmentVariableAsInt(PortVariable, 5432);
+    private int CommandTimeout { get; init; } = Prefix.GetEnvironmentVariableAsInt(CommandTimeoutVariable, 300);
 
-    public string StorageConnectionString => $"Host={Host};Port={PORT};Username={Username};Password={Password};Database={Database};";
+    public string StorageConnectionString => $"Host={Host};Port={PORT};Username={Username};Password={Password};Database={Database};CommandTimeout={CommandTimeout}";
 }

src/metadata/Features/ImportImdbData/ImdbDbService.cs

@@ -134,7 +134,7 @@ public class ImdbDbService(PostgresConfiguration configuration, ILogger<ImdbDbSe
     {
         try
         {
-            await using var connection = CreateNpgsqlConnection();
+            await using var connection = new NpgsqlConnection(configuration.StorageConnectionString);
             await connection.OpenAsync();
 
             await operation(connection);
@@ -145,16 +145,6 @@ public class ImdbDbService(PostgresConfiguration configuration, ILogger<ImdbDbSe
         }
     }
 
-    private NpgsqlConnection CreateNpgsqlConnection()
-    {
-        var connectionStringBuilder = new NpgsqlConnectionStringBuilder(configuration.StorageConnectionString)
-        {
-            CommandTimeout = 3000,
-        };
-
-        return new(connectionStringBuilder.ConnectionString);
-    }
-
     private async Task ExecuteCommandWithTransactionAsync(Func<NpgsqlConnection, NpgsqlTransaction, Task> operation, NpgsqlTransaction transaction, string errorMessage)
     {
         try

src/metadata/Metadata.csproj

@@ -13,7 +13,7 @@
       <PackageReference Include="Dapper" Version="2.1.35" />
       <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
       <PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
-      <PackageReference Include="Npgsql" Version="8.0.2" />
+      <PackageReference Include="Npgsql" Version="8.0.3" />
       <PackageReference Include="Serilog" Version="3.1.1" />
       <PackageReference Include="Serilog.AspNetCore" Version="8.0.1" />
       <PackageReference Include="Serilog.Sinks.Console" Version="5.0.1" />

src/producer/src/Dockerfile

@@ -14,7 +14,7 @@ WORKDIR /app
 
 ENV PYTHONUNBUFFERED=1
 
-RUN apk add --update --no-cache python3=~3.11.9-r0 py3-pip && ln -sf python3 /usr/bin/python
+RUN apk add --update --no-cache python3=~3.11 py3-pip && ln -sf python3 /usr/bin/python
 
 COPY --from=build /src/out .
 

src/producer/src/Features/Crawlers/Dmm/DebridMediaManagerCrawler.cs

@@ -69,7 +69,7 @@ public partial class DebridMediaManagerCrawler(
             if (page.TryGetValue(infoHash, out var dmmContent) &&
                 successfulResponses.TryGetValue(dmmContent.Filename, out var parsedResponse))
             {
-                page[infoHash] = dmmContent with {ParseResponse = parsedResponse};
+                page[infoHash] = dmmContent with { ParseResponse = parsedResponse };
             }
 
             return ValueTask.CompletedTask;
@@ -106,9 +106,34 @@ public partial class DebridMediaManagerCrawler(
 
         var decodedJson = LZString.DecompressFromEncodedURIComponent(encodedJson.Value);
 
+        JsonElement arrayToProcess;
+        try
+        {
             var json = JsonDocument.Parse(decodedJson);
 
-        var torrents = await json.RootElement.EnumerateArray()
+            if (json.RootElement.ValueKind == JsonValueKind.Object &&
+                json.RootElement.TryGetProperty("torrents", out var torrentsProperty) &&
+                torrentsProperty.ValueKind == JsonValueKind.Array)
+            {
+                arrayToProcess = torrentsProperty;
+            }
+            else if (json.RootElement.ValueKind == JsonValueKind.Array)
+            {
+                arrayToProcess = json.RootElement;
+            }
+            else
+            {
+                logger.LogWarning("Unexpected JSON format in {Name}", name);
+                return [];
+            }
+        }
+        catch (Exception ex)
+        {
+            logger.LogError("Failed to parse JSON {decodedJson} for {Name}: {Exception}", decodedJson, name, ex);
+            return [];
+        }
+
+        var torrents = await arrayToProcess.EnumerateArray()
             .ToAsyncEnumerable()
             .Select(ParsePageContent)
             .Where(t => t is not null)
@@ -141,7 +166,7 @@ public partial class DebridMediaManagerCrawler(
         {
             var (infoHash, dmmContent) = kvp;
             var parsedTorrent = dmmContent.ParseResponse;
-            if (parsedTorrent is not {Success: true})
+            if (parsedTorrent is not { Success: true })
             {
                 return;
             }

src/qbit-collector/Dockerfile

@@ -15,7 +15,7 @@ WORKDIR /app
 
 ENV PYTHONUNBUFFERED=1
 
-RUN apk add --update --no-cache python3=~3.11.9-r0 py3-pip && ln -sf python3 /usr/bin/python
+RUN apk add --update --no-cache python3=~3.11 py3-pip && ln -sf python3 /usr/bin/python
 
 COPY --from=build /src/out .
 

src/shared/Configuration/PostgresConfiguration.cs

@@ -8,12 +8,14 @@ public class PostgresConfiguration
     private const string PasswordVariable = "PASSWORD";
     private const string DatabaseVariable = "DB";
     private const string PortVariable = "PORT";
+    private const string CommandTimeoutVariable = "COMMAND_TIMEOUT_SEC"; // Seconds
 
     private string Host { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(HostVariable);
     private string Username { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(UsernameVariable);
     private string Password { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(PasswordVariable);
     private string Database { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(DatabaseVariable);
     private int PORT { get; init; } = Prefix.GetEnvironmentVariableAsInt(PortVariable, 5432);
+    private int CommandTimeout { get; init; } = Prefix.GetEnvironmentVariableAsInt(CommandTimeoutVariable, 300);
 
-    public string StorageConnectionString => $"Host={Host};Port={PORT};Username={Username};Password={Password};Database={Database};";
+    public string StorageConnectionString => $"Host={Host};Port={PORT};Username={Username};Password={Password};Database={Database};CommandTimeout={CommandTimeout}";
 }

src/shared/SharedContracts.csproj

@@ -15,7 +15,7 @@
   <PackageReference Include="Dapper" Version="2.1.35" />
   <PackageReference Include="MassTransit.Abstractions" Version="8.2.0" />
   <PackageReference Include="MassTransit.RabbitMQ" Version="8.2.0" />
-  <PackageReference Include="Npgsql" Version="8.0.2" />
+  <PackageReference Include="Npgsql" Version="8.0.3" />
   <PackageReference Include="pythonnet" Version="3.0.3" />
   <PackageReference Include="Serilog" Version="3.1.1" />
   <PackageReference Include="Serilog.Extensions.Hosting" Version="8.0.0" />

src/tissue/Features/DataProcessing/PostgresConfiguration.cs

@@ -8,12 +8,14 @@ public class PostgresConfiguration
     private const string PasswordVariable = "PASSWORD";
     private const string DatabaseVariable = "DB";
     private const string PortVariable = "PORT";
+    private const string CommandTimeoutVariable = "COMMAND_TIMEOUT_SEC"; // Seconds
 
     private string Host { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(HostVariable);
     private string Username { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(UsernameVariable);
     private string Password { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(PasswordVariable);
     private string Database { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(DatabaseVariable);
     private int PORT { get; init; } = Prefix.GetEnvironmentVariableAsInt(PortVariable, 5432);
+    private int CommandTimeout { get; init; } = Prefix.GetEnvironmentVariableAsInt(CommandTimeoutVariable, 300);
 
-    public string StorageConnectionString => $"Host={Host};Port={PORT};Username={Username};Password={Password};Database={Database};";
+    public string StorageConnectionString => $"Host={Host};Port={PORT};Username={Username};Password={Password};Database={Database};CommandTimeout={CommandTimeout}";
 }

src/tissue/Tissue.csproj

@@ -12,7 +12,7 @@
       <PackageReference Include="Dapper" Version="2.1.28" />
       <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
       <PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
-      <PackageReference Include="Npgsql" Version="8.0.1" />
+      <PackageReference Include="Npgsql" Version="8.0.3" />
       <PackageReference Include="Serilog" Version="3.1.1" />
       <PackageReference Include="Serilog.AspNetCore" Version="8.0.1" />
       <PackageReference Include="Serilog.Sinks.Console" Version="5.0.1" />