tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Introduce max batch size, and configurable publish window

Browse Source 
Still need to implement queue size limit
Also fixes env var consistency between addon and consumer
This commit is contained in:
iPromKnight
2024-02-02 13:49:42 +00:00
parent 0644fe7720
commit 68edaba308
41 changed files with 88 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/producer/Services/DapperDataStorage.cs
View File

@@ -1,6 +1,6 @@
namespace Scraper.Services;
namespace Producer.Services;
public class DapperDataStorage(ScrapeConfiguration configuration, ILogger<DapperDataStorage> logger) : IDataStorage
public class DapperDataStorage(ScrapeConfiguration configuration, RabbitMqConfiguration rabbitConfig, ILogger<DapperDataStorage> logger) : IDataStorage
{
private const string InsertTorrentSql =
"""
@@ -65,7 +65,7 @@ public class DapperDataStorage(ScrapeConfiguration configuration, ILogger<Dapper
await using var connection = new NpgsqlConnection(configuration.StorageConnectionString);
await connection.OpenAsync(cancellationToken);
var torrents = await connection.QueryAsync<Torrent>(GetMovieAndSeriesTorrentsNotProcessedSql);
return torrents.ToList();
return torrents.Take(rabbitConfig.MaxPublishBatchSize).ToList();
}
catch (Exception e)
{

2
src/producer/Services/IpService.cs
View File

@@ -1,4 +1,4 @@
namespace Scraper.Services;
namespace Producer.Services;
public class IpService(ILogger<IpService> logger, IHttpClientFactory httpClientFactory) : IIpService
{

2
src/producer/Services/TorrentPublisher.cs
View File

@@ -1,4 +1,4 @@
namespace Scraper.Services;
namespace Producer.Services;
public class TorrentPublisher(ISendEndpointProvider sendEndpointProvider, RabbitMqConfiguration configuration) : IMessagePublisher
{