Introduce max batch size, and configurable publish window

Still need to implement queue size limit
Also fixes env var consistency between addon and consumer
This commit is contained in:
iPromKnight
2024-02-02 13:49:42 +00:00
parent 0644fe7720
commit 68edaba308
41 changed files with 88 additions and 51 deletions

View File

@@ -1,4 +1,4 @@
namespace Scraper.Jobs;
namespace Producer.Jobs;
public abstract class BaseJob(ICrawlerProvider crawlerProvider) : IJob
{

View File

@@ -1,4 +1,4 @@
namespace Scraper.Jobs;
namespace Producer.Jobs;
public interface ICrawlerJob<out TCrawler> : IJob
where TCrawler : ICrawler

View File

@@ -1,4 +1,4 @@
namespace Scraper.Jobs;
namespace Producer.Jobs;
[DisallowConcurrentExecution]
public class IPJob(IIpService ipService) : IJob

View File

@@ -1,4 +1,4 @@
namespace Scraper.Jobs;
namespace Producer.Jobs;
[DisallowConcurrentExecution]
public class PublisherJob(IMessagePublisher publisher, IDataStorage storage, ILogger<PublisherJob> logger) : IJob

View File

@@ -1,4 +1,6 @@
namespace Scraper.Jobs;
using DebridMediaManagerCrawler = Producer.Crawlers.Sites.DebridMediaManagerCrawler;
namespace Producer.Jobs;
[DisallowConcurrentExecution]
public class SyncDmmJob(ICrawlerProvider crawlerProvider) : BaseJob(crawlerProvider)

View File

@@ -1,4 +1,6 @@
namespace Scraper.Jobs;
using Producer.Crawlers.Sites;
namespace Producer.Jobs;
[DisallowConcurrentExecution]
public class SyncEzTvJob(ICrawlerProvider crawlerProvider) : BaseJob(crawlerProvider)

View File

@@ -1,4 +1,6 @@
namespace Scraper.Jobs;
using TgxCrawler = Producer.Crawlers.Sites.TgxCrawler;
namespace Producer.Jobs;
[DisallowConcurrentExecution]
public class SyncTgxJob(ICrawlerProvider crawlerProvider) : BaseJob(crawlerProvider)

View File

@@ -1,4 +1,6 @@
namespace Scraper.Jobs;
using Producer.Crawlers.Sites;
namespace Producer.Jobs;
[DisallowConcurrentExecution]
public class SyncTpbJob(ICrawlerProvider crawlerProvider) : BaseJob(crawlerProvider)

View File

@@ -1,4 +1,6 @@
namespace Scraper.Jobs;
using Producer.Crawlers.Sites;
namespace Producer.Jobs;
[DisallowConcurrentExecution]
public class SyncYtsJob(ICrawlerProvider crawlerProvider) : BaseJob(crawlerProvider)