Introduce max batch size, and configurable publish window

Still need to implement queue size limit
Also fixes env var consistency between addon and consumer
This commit is contained in:
iPromKnight
2024-02-02 13:49:42 +00:00
parent 0644fe7720
commit 68edaba308
41 changed files with 88 additions and 51 deletions

View File

@@ -1,4 +1,4 @@
namespace Scraper.Models;
namespace Producer.Models;
public class GithubConfiguration
{

View File

@@ -1,4 +1,4 @@
namespace Scraper.Models;
namespace Producer.Models;
public class RabbitMqConfiguration
{
@@ -10,4 +10,7 @@ public class RabbitMqConfiguration
public string? Password { get; set; }
public string? QueueName { get; set; }
public bool Durable { get; set; }
public int MaxQueueSize { get; set; } = 1000;
public int MaxPublishBatchSize { get; set; } = 100;
public int PublishIntervalInSeconds { get; set; } = 1000 * 10;
}

View File

@@ -1,4 +1,4 @@
namespace Scraper.Models;
namespace Producer.Models;
public record InsertTorrentResult(bool Success, int InsertedCount = 0, string? ErrorMessage = null);
public record UpdatedTorrentResult(bool Success, int UpdatedCount = 0, string? ErrorMessage = null);

View File

@@ -1,4 +1,4 @@
namespace Scraper.Models;
namespace Producer.Models;
public class ScrapeConfiguration
{

View File

@@ -1,4 +1,4 @@
namespace Scraper.Models;
namespace Producer.Models;
public class Scraper
{

View File

@@ -1,4 +1,4 @@
namespace Scraper.Models;
namespace Producer.Models;
// Torrent represents a crawled torrent from one of our
// supported sources.