Allow qbit concurrency to be configurable (#200)

deployment/docker/config/qbit/qbittorrent.conf

@@ -14,12 +14,14 @@ program=
 [BitTorrent]
 Session\AnonymousModeEnabled=true
 Session\BTProtocol=TCP
+Session\ConnectionSpeed=150
 Session\DefaultSavePath=/downloads/
 Session\ExcludedFileNames=
-Session\MaxActiveCheckingTorrents=5
+Session\MaxActiveCheckingTorrents=20
-Session\MaxActiveDownloads=10
+Session\MaxActiveDownloads=20
 Session\MaxActiveTorrents=50
 Session\MaxActiveUploads=50
+Session\MaxConcurrentHTTPAnnounces=1000
 Session\MaxConnections=2000
 Session\Port=6881
 Session\QueueingSystemEnabled=true

deployment/docker/docker-compose.yaml

@@ -94,7 +94,7 @@ services:
         condition: service_healthy
     env_file: stack.env
     hostname: knightcrawler-addon
-    image: gabisonfire/knightcrawler-addon:2.0.21
+    image: gabisonfire/knightcrawler-addon:2.0.22
     labels:
       logging: promtail
     networks:
@@ -117,7 +117,7 @@ services:
       redis:
         condition: service_healthy
     env_file: stack.env
-    image: gabisonfire/knightcrawler-consumer:2.0.21
+    image: gabisonfire/knightcrawler-consumer:2.0.22
     labels:
       logging: promtail
     networks:
@@ -138,7 +138,7 @@ services:
       redis:
         condition: service_healthy
     env_file: stack.env
-    image: gabisonfire/knightcrawler-debrid-collector:2.0.21
+    image: gabisonfire/knightcrawler-debrid-collector:2.0.22
     labels:
       logging: promtail
     networks:
@@ -152,7 +152,7 @@ services:
       migrator:
         condition: service_completed_successfully
     env_file: stack.env
-    image: gabisonfire/knightcrawler-metadata:2.0.21
+    image: gabisonfire/knightcrawler-metadata:2.0.22
     networks:
       - knightcrawler-network
     restart: "no"
@@ -163,7 +163,7 @@ services:
       postgres:
         condition: service_healthy
     env_file: stack.env
-    image: gabisonfire/knightcrawler-migrator:2.0.21
+    image: gabisonfire/knightcrawler-migrator:2.0.22
     networks:
       - knightcrawler-network
     restart: "no"
@@ -182,7 +182,7 @@ services:
       redis:
         condition: service_healthy
     env_file: stack.env
-    image: gabisonfire/knightcrawler-producer:2.0.21
+    image: gabisonfire/knightcrawler-producer:2.0.22
     labels:
       logging: promtail
     networks:
@@ -207,7 +207,7 @@ services:
     deploy:
       replicas: ${QBIT_REPLICAS:-0}
     env_file: stack.env
-    image: gabisonfire/knightcrawler-qbit-collector:2.0.21
+    image: gabisonfire/knightcrawler-qbit-collector:2.0.22
     labels:
       logging: promtail
     networks:

deployment/docker/src/components/knightcrawler.yaml

@@ -20,7 +20,7 @@ x-depends: &knightcrawler-app-depends
 
 services:
   metadata:
-    image: gabisonfire/knightcrawler-metadata:2.0.21
+    image: gabisonfire/knightcrawler-metadata:2.0.22
     env_file: ../../.env
     networks:
       - knightcrawler-network
@@ -30,7 +30,7 @@ services:
         condition: service_completed_successfully
 
   migrator:
-    image: gabisonfire/knightcrawler-migrator:2.0.21
+    image: gabisonfire/knightcrawler-migrator:2.0.22
     env_file: ../../.env
     networks:
       - knightcrawler-network
@@ -40,7 +40,7 @@ services:
         condition: service_healthy
 
   addon:
-    image: gabisonfire/knightcrawler-addon:2.0.21
+    image: gabisonfire/knightcrawler-addon:2.0.22
     <<: [*knightcrawler-app, *knightcrawler-app-depends]
     restart: unless-stopped
     hostname: knightcrawler-addon
@@ -48,22 +48,22 @@ services:
       - "7000:7000"
 
   consumer:
-    image: gabisonfire/knightcrawler-consumer:2.0.21
+    image: gabisonfire/knightcrawler-consumer:2.0.22
     <<: [*knightcrawler-app, *knightcrawler-app-depends]
     restart: unless-stopped
 
   debridcollector:
-    image: gabisonfire/knightcrawler-debrid-collector:2.0.21
+    image: gabisonfire/knightcrawler-debrid-collector:2.0.22
     <<: [*knightcrawler-app, *knightcrawler-app-depends]
     restart: unless-stopped
         
   producer:
-    image: gabisonfire/knightcrawler-producer:2.0.21
+    image: gabisonfire/knightcrawler-producer:2.0.22
     <<: [*knightcrawler-app, *knightcrawler-app-depends]
     restart: unless-stopped
 
   qbitcollector:
-    image: gabisonfire/knightcrawler-qbit-collector:2.0.21
+    image: gabisonfire/knightcrawler-qbit-collector:2.0.22
     <<: [*knightcrawler-app, *knightcrawler-app-depends]
     restart: unless-stopped
     depends_on:

deployment/docker/stack.env

@@ -32,6 +32,7 @@ COLLECTOR_DEBRID_ENABLED=true
 COLLECTOR_REAL_DEBRID_API_KEY=
 QBIT_HOST=http://qbittorrent:8080
 QBIT_TRACKERS_URL=https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_http.txt
+QBIT_CONCURRENCY=8
 
 # Number of replicas for the qBittorrent collector and qBitTorrent client. Should be 0 or 1.
 QBIT_REPLICAS=0

src/qbit-collector/Extensions/ServiceCollectionExtensions.cs

@@ -44,6 +44,7 @@ public static class ServiceCollectionExtensions
     {
         var rabbitConfiguration = services.LoadConfigurationFromEnv<RabbitMqConfiguration>();
         var redisConfiguration = services.LoadConfigurationFromEnv<RedisConfiguration>();
+        var qbitConfiguration = services.LoadConfigurationFromEnv<QbitConfiguration>();
 
         services.AddStackExchangeRedisCache(
             option =>
@@ -80,8 +81,8 @@ public static class ServiceCollectionExtensions
                     e.ConfigureConsumer<WriteQbitMetadataConsumer>(context);
                     e.ConfigureConsumer<PerformQbitMetadataRequestConsumer>(context);
                     e.ConfigureSaga<QbitMetadataSagaState>(context);
-                    e.ConcurrentMessageLimit = 5;
+                    e.ConcurrentMessageLimit = qbitConfiguration.Concurrency;
-                    e.PrefetchCount = 5;
+                    e.PrefetchCount = qbitConfiguration.Concurrency;
                 });
             });
         });
@@ -98,7 +99,7 @@ public static class ServiceCollectionExtensions
                     cfg.UseTimeout(
                         timeout =>
                         {
-                            timeout.Timeout = TimeSpan.FromMinutes(1);
+                            timeout.Timeout = TimeSpan.FromMinutes(3);
                         });
                 })
             .RedisRepository(redisConfiguration.ConnectionString, options =>
@@ -110,7 +111,7 @@ public static class ServiceCollectionExtensions
     {
         var qbitConfiguration = services.LoadConfigurationFromEnv<QbitConfiguration>();
         var client = new QBittorrentClient(new(qbitConfiguration.Host));
-        client.Timeout = TimeSpan.FromSeconds(10);
+        client.Timeout = TimeSpan.FromSeconds(20);
 
         services.AddSingleton<IQBittorrentClient>(client);
     }

src/qbit-collector/Features/Qbit/QBitRequestProcessor.cs

@@ -1,6 +1,6 @@
 namespace QBitCollector.Features.Qbit;
 
-public class QbitRequestProcessor(IQBittorrentClient client, ITrackersService trackersService, ILogger<QbitRequestProcessor> logger)
+public class QbitRequestProcessor(IQBittorrentClient client, ITrackersService trackersService, ILogger<QbitRequestProcessor> logger, QbitConfiguration configuration)
 {
     public async Task<IReadOnlyList<TorrentContent>?> ProcessAsync(string infoHash, CancellationToken cancellationToken = default)
     {
@@ -14,7 +14,7 @@ public class QbitRequestProcessor(IQBittorrentClient client, ITrackersService tr
 
         using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
 
-        timeoutCts.CancelAfter(TimeSpan.FromSeconds(30));
+        timeoutCts.CancelAfter(TimeSpan.FromSeconds(60));
 
         try
         {
@@ -30,7 +30,7 @@ public class QbitRequestProcessor(IQBittorrentClient client, ITrackersService tr
                     break;
                 }
 
-                await Task.Delay(TimeSpan.FromSeconds(1), timeoutCts.Token);
+                await Task.Delay(TimeSpan.FromMilliseconds(200), timeoutCts.Token);
             }
         }
         catch (OperationCanceledException) when (timeoutCts.IsCancellationRequested)

src/qbit-collector/Features/Qbit/QbitConfiguration.cs

@@ -5,7 +5,10 @@ public class QbitConfiguration
     private const string Prefix = "QBIT";
     private const string HOST_VARIABLE = "HOST";
     private const string TRACKERS_URL_VARIABLE = "TRACKERS_URL";
+    private const string CONCURRENCY_VARIABLE = "CONCURRENCY";
     
     public string? Host { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(HOST_VARIABLE);
     public string? TrackersUrl { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(TRACKERS_URL_VARIABLE);
+    
+    public int Concurrency { get; init; } = Prefix.GetEnvironmentVariableAsInt(CONCURRENCY_VARIABLE, 8);
 }

src/qbit-collector/Features/Worker/WriteQbitMetadataConsumer.cs

@@ -6,6 +6,12 @@ public class WriteQbitMetadataConsumer(IRankTorrentName rankTorrentName, IDataSt
     {
         var request = context.Message;
         
+        if (request.Metadata.Metadata.Count == 0)
+        {
+            await context.Publish(new QbitMetadataWritten(request.Metadata, false));
+            return;
+        }
+
         var torrentFiles = QbitMetaToTorrentMeta.MapMetadataToFilesCollection(
             rankTorrentName, request.Torrent, request.ImdbId, request.Metadata.Metadata, logger);