From 02101ac50a3c325302fd172ea78478341c1da983 Mon Sep 17 00:00:00 2001 From: iPromKnight <156901906+iPromKnight@users.noreply.github.com> Date: Thu, 11 Apr 2024 18:02:29 +0100 Subject: [PATCH] Allow qbit concurrency to be configurable (#200) --- deployment/docker/config/qbit/qbittorrent.conf | 6 ++++-- deployment/docker/docker-compose.yaml | 14 +++++++------- .../docker/src/components/knightcrawler.yaml | 14 +++++++------- deployment/docker/stack.env | 1 + .../Extensions/ServiceCollectionExtensions.cs | 9 +++++---- .../Features/Qbit/QBitRequestProcessor.cs | 6 +++--- .../Features/Qbit/QbitConfiguration.cs | 3 +++ .../Features/Worker/WriteQbitMetadataConsumer.cs | 6 ++++++ 8 files changed, 36 insertions(+), 23 deletions(-) diff --git a/deployment/docker/config/qbit/qbittorrent.conf b/deployment/docker/config/qbit/qbittorrent.conf index 8dcfd0e..e45f05c 100755 --- a/deployment/docker/config/qbit/qbittorrent.conf +++ b/deployment/docker/config/qbit/qbittorrent.conf @@ -14,12 +14,14 @@ program= [BitTorrent] Session\AnonymousModeEnabled=true Session\BTProtocol=TCP +Session\ConnectionSpeed=150 Session\DefaultSavePath=/downloads/ Session\ExcludedFileNames= -Session\MaxActiveCheckingTorrents=5 -Session\MaxActiveDownloads=10 +Session\MaxActiveCheckingTorrents=20 +Session\MaxActiveDownloads=20 Session\MaxActiveTorrents=50 Session\MaxActiveUploads=50 +Session\MaxConcurrentHTTPAnnounces=1000 Session\MaxConnections=2000 Session\Port=6881 Session\QueueingSystemEnabled=true diff --git a/deployment/docker/docker-compose.yaml b/deployment/docker/docker-compose.yaml index b20ab08..29ef4cd 100644 --- a/deployment/docker/docker-compose.yaml +++ b/deployment/docker/docker-compose.yaml @@ -94,7 +94,7 @@ services: condition: service_healthy env_file: stack.env hostname: knightcrawler-addon - image: gabisonfire/knightcrawler-addon:2.0.21 + image: gabisonfire/knightcrawler-addon:2.0.22 labels: logging: promtail networks: @@ -117,7 +117,7 @@ services: redis: condition: service_healthy env_file: stack.env - image: gabisonfire/knightcrawler-consumer:2.0.21 + image: gabisonfire/knightcrawler-consumer:2.0.22 labels: logging: promtail networks: @@ -138,7 +138,7 @@ services: redis: condition: service_healthy env_file: stack.env - image: gabisonfire/knightcrawler-debrid-collector:2.0.21 + image: gabisonfire/knightcrawler-debrid-collector:2.0.22 labels: logging: promtail networks: @@ -152,7 +152,7 @@ services: migrator: condition: service_completed_successfully env_file: stack.env - image: gabisonfire/knightcrawler-metadata:2.0.21 + image: gabisonfire/knightcrawler-metadata:2.0.22 networks: - knightcrawler-network restart: "no" @@ -163,7 +163,7 @@ services: postgres: condition: service_healthy env_file: stack.env - image: gabisonfire/knightcrawler-migrator:2.0.21 + image: gabisonfire/knightcrawler-migrator:2.0.22 networks: - knightcrawler-network restart: "no" @@ -182,7 +182,7 @@ services: redis: condition: service_healthy env_file: stack.env - image: gabisonfire/knightcrawler-producer:2.0.21 + image: gabisonfire/knightcrawler-producer:2.0.22 labels: logging: promtail networks: @@ -207,7 +207,7 @@ services: deploy: replicas: ${QBIT_REPLICAS:-0} env_file: stack.env - image: gabisonfire/knightcrawler-qbit-collector:2.0.21 + image: gabisonfire/knightcrawler-qbit-collector:2.0.22 labels: logging: promtail networks: diff --git a/deployment/docker/src/components/knightcrawler.yaml b/deployment/docker/src/components/knightcrawler.yaml index eab5326..ff8d71d 100644 --- a/deployment/docker/src/components/knightcrawler.yaml +++ b/deployment/docker/src/components/knightcrawler.yaml @@ -20,7 +20,7 @@ x-depends: &knightcrawler-app-depends services: metadata: - image: gabisonfire/knightcrawler-metadata:2.0.21 + image: gabisonfire/knightcrawler-metadata:2.0.22 env_file: ../../.env networks: - knightcrawler-network @@ -30,7 +30,7 @@ services: condition: service_completed_successfully migrator: - image: gabisonfire/knightcrawler-migrator:2.0.21 + image: gabisonfire/knightcrawler-migrator:2.0.22 env_file: ../../.env networks: - knightcrawler-network @@ -40,7 +40,7 @@ services: condition: service_healthy addon: - image: gabisonfire/knightcrawler-addon:2.0.21 + image: gabisonfire/knightcrawler-addon:2.0.22 <<: [*knightcrawler-app, *knightcrawler-app-depends] restart: unless-stopped hostname: knightcrawler-addon @@ -48,22 +48,22 @@ services: - "7000:7000" consumer: - image: gabisonfire/knightcrawler-consumer:2.0.21 + image: gabisonfire/knightcrawler-consumer:2.0.22 <<: [*knightcrawler-app, *knightcrawler-app-depends] restart: unless-stopped debridcollector: - image: gabisonfire/knightcrawler-debrid-collector:2.0.21 + image: gabisonfire/knightcrawler-debrid-collector:2.0.22 <<: [*knightcrawler-app, *knightcrawler-app-depends] restart: unless-stopped producer: - image: gabisonfire/knightcrawler-producer:2.0.21 + image: gabisonfire/knightcrawler-producer:2.0.22 <<: [*knightcrawler-app, *knightcrawler-app-depends] restart: unless-stopped qbitcollector: - image: gabisonfire/knightcrawler-qbit-collector:2.0.21 + image: gabisonfire/knightcrawler-qbit-collector:2.0.22 <<: [*knightcrawler-app, *knightcrawler-app-depends] restart: unless-stopped depends_on: diff --git a/deployment/docker/stack.env b/deployment/docker/stack.env index 4901898..93f2184 100644 --- a/deployment/docker/stack.env +++ b/deployment/docker/stack.env @@ -32,6 +32,7 @@ COLLECTOR_DEBRID_ENABLED=true COLLECTOR_REAL_DEBRID_API_KEY= QBIT_HOST=http://qbittorrent:8080 QBIT_TRACKERS_URL=https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_http.txt +QBIT_CONCURRENCY=8 # Number of replicas for the qBittorrent collector and qBitTorrent client. Should be 0 or 1. QBIT_REPLICAS=0 diff --git a/src/qbit-collector/Extensions/ServiceCollectionExtensions.cs b/src/qbit-collector/Extensions/ServiceCollectionExtensions.cs index 5836567..330363c 100644 --- a/src/qbit-collector/Extensions/ServiceCollectionExtensions.cs +++ b/src/qbit-collector/Extensions/ServiceCollectionExtensions.cs @@ -44,6 +44,7 @@ public static class ServiceCollectionExtensions { var rabbitConfiguration = services.LoadConfigurationFromEnv(); var redisConfiguration = services.LoadConfigurationFromEnv(); + var qbitConfiguration = services.LoadConfigurationFromEnv(); services.AddStackExchangeRedisCache( option => @@ -80,8 +81,8 @@ public static class ServiceCollectionExtensions e.ConfigureConsumer(context); e.ConfigureConsumer(context); e.ConfigureSaga(context); - e.ConcurrentMessageLimit = 5; - e.PrefetchCount = 5; + e.ConcurrentMessageLimit = qbitConfiguration.Concurrency; + e.PrefetchCount = qbitConfiguration.Concurrency; }); }); }); @@ -98,7 +99,7 @@ public static class ServiceCollectionExtensions cfg.UseTimeout( timeout => { - timeout.Timeout = TimeSpan.FromMinutes(1); + timeout.Timeout = TimeSpan.FromMinutes(3); }); }) .RedisRepository(redisConfiguration.ConnectionString, options => @@ -110,7 +111,7 @@ public static class ServiceCollectionExtensions { var qbitConfiguration = services.LoadConfigurationFromEnv(); var client = new QBittorrentClient(new(qbitConfiguration.Host)); - client.Timeout = TimeSpan.FromSeconds(10); + client.Timeout = TimeSpan.FromSeconds(20); services.AddSingleton(client); } diff --git a/src/qbit-collector/Features/Qbit/QBitRequestProcessor.cs b/src/qbit-collector/Features/Qbit/QBitRequestProcessor.cs index 6d30b35..ea88d4c 100644 --- a/src/qbit-collector/Features/Qbit/QBitRequestProcessor.cs +++ b/src/qbit-collector/Features/Qbit/QBitRequestProcessor.cs @@ -1,6 +1,6 @@ namespace QBitCollector.Features.Qbit; -public class QbitRequestProcessor(IQBittorrentClient client, ITrackersService trackersService, ILogger logger) +public class QbitRequestProcessor(IQBittorrentClient client, ITrackersService trackersService, ILogger logger, QbitConfiguration configuration) { public async Task?> ProcessAsync(string infoHash, CancellationToken cancellationToken = default) { @@ -14,7 +14,7 @@ public class QbitRequestProcessor(IQBittorrentClient client, ITrackersService tr using var timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - timeoutCts.CancelAfter(TimeSpan.FromSeconds(30)); + timeoutCts.CancelAfter(TimeSpan.FromSeconds(60)); try { @@ -30,7 +30,7 @@ public class QbitRequestProcessor(IQBittorrentClient client, ITrackersService tr break; } - await Task.Delay(TimeSpan.FromSeconds(1), timeoutCts.Token); + await Task.Delay(TimeSpan.FromMilliseconds(200), timeoutCts.Token); } } catch (OperationCanceledException) when (timeoutCts.IsCancellationRequested) diff --git a/src/qbit-collector/Features/Qbit/QbitConfiguration.cs b/src/qbit-collector/Features/Qbit/QbitConfiguration.cs index 3130184..f9ed5a4 100644 --- a/src/qbit-collector/Features/Qbit/QbitConfiguration.cs +++ b/src/qbit-collector/Features/Qbit/QbitConfiguration.cs @@ -5,7 +5,10 @@ public class QbitConfiguration private const string Prefix = "QBIT"; private const string HOST_VARIABLE = "HOST"; private const string TRACKERS_URL_VARIABLE = "TRACKERS_URL"; + private const string CONCURRENCY_VARIABLE = "CONCURRENCY"; public string? Host { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(HOST_VARIABLE); public string? TrackersUrl { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(TRACKERS_URL_VARIABLE); + + public int Concurrency { get; init; } = Prefix.GetEnvironmentVariableAsInt(CONCURRENCY_VARIABLE, 8); } \ No newline at end of file diff --git a/src/qbit-collector/Features/Worker/WriteQbitMetadataConsumer.cs b/src/qbit-collector/Features/Worker/WriteQbitMetadataConsumer.cs index b2e0b4d..926f42b 100644 --- a/src/qbit-collector/Features/Worker/WriteQbitMetadataConsumer.cs +++ b/src/qbit-collector/Features/Worker/WriteQbitMetadataConsumer.cs @@ -5,6 +5,12 @@ public class WriteQbitMetadataConsumer(IRankTorrentName rankTorrentName, IDataSt public async Task Consume(ConsumeContext context) { var request = context.Message; + + if (request.Metadata.Metadata.Count == 0) + { + await context.Publish(new QbitMetadataWritten(request.Metadata, false)); + return; + } var torrentFiles = QbitMetaToTorrentMeta.MapMetadataToFilesCollection( rankTorrentName, request.Torrent, request.ImdbId, request.Metadata.Metadata, logger);