tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Introduce max batch size, and configurable publish window

Browse Source 
Still need to implement queue size limit
Also fixes env var consistency between addon and consumer
This commit is contained in:
iPromKnight
2024-02-02 13:49:42 +00:00
parent 0644fe7720
commit 68edaba308
41 changed files with 88 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
src/producer/Extensions/ServiceCollectionExtensions.cs
View File

@@ -1,4 +1,4 @@
namespace Scraper.Extensions;
namespace Producer.Extensions;
public static class ServiceCollectionExtensions
{
@@ -53,6 +53,7 @@ public static class ServiceCollectionExtensions
{
var scrapeConfiguration = LoadScrapeConfiguration(services, configuration);
var githubConfiguration = LoadGithubConfiguration(services, configuration);
var rabbitConfig = LoadRabbitMQConfiguration(services, configuration);
services
.AddTransient<SyncEzTvJob>()
@@ -75,7 +76,7 @@ public static class ServiceCollectionExtensions
AddJobWithTrigger<SyncYtsJob>(quartz, SyncYtsJob.Key, SyncYtsJob.Trigger, scrapeConfiguration);
AddJobWithTrigger<SyncTgxJob>(quartz, SyncTgxJob.Key, SyncTgxJob.Trigger, scrapeConfiguration);
AddJobWithTrigger<IPJob>(quartz, IPJob.Key, IPJob.Trigger, 60 * 5);
AddJobWithTrigger<PublisherJob>(quartz, PublisherJob.Key, PublisherJob.Trigger, 10);
AddJobWithTrigger<PublisherJob>(quartz, PublisherJob.Key, PublisherJob.Trigger, rabbitConfig.PublishIntervalInSeconds);
if (!string.IsNullOrEmpty(githubConfiguration.PAT))
{
@@ -98,10 +99,21 @@ public static class ServiceCollectionExtensions
ArgumentNullException.ThrowIfNull(githubConfiguration, nameof(githubConfiguration));
services.AddSingleton(githubConfiguration);
services.TryAddSingleton(githubConfiguration);
return githubConfiguration;
}
private static RabbitMqConfiguration LoadRabbitMQConfiguration(IServiceCollection services, IConfiguration configuration)
{
var rabbitConfiguration = configuration.GetSection(RabbitMqConfiguration.SectionName).Get<RabbitMqConfiguration>();
ArgumentNullException.ThrowIfNull(rabbitConfiguration, nameof(rabbitConfiguration));
services.TryAddSingleton(rabbitConfiguration);
return rabbitConfiguration;
}
private static ScrapeConfiguration LoadScrapeConfiguration(IServiceCollection services, IConfiguration configuration)
{
@@ -109,7 +121,7 @@ public static class ServiceCollectionExtensions
ArgumentNullException.ThrowIfNull(scrapeConfiguration, nameof(scrapeConfiguration));
services.AddSingleton(scrapeConfiguration);
services.TryAddSingleton(scrapeConfiguration);
return scrapeConfiguration;
}