Merge pull request #105 from iPromKnight/community-powered-idea

Woke up to see a discussion and comments such as: powered by community tagline

.gitignore

@@ -405,3 +405,5 @@ FodyWeavers.xsd
 
 dist/
 deployment/docker/docker-compose-dev.yaml
+
+src/producer/.run/

src/producer/Configuration/scrapers.json

@@ -30,6 +30,11 @@
         "Name": "SyncDmmJob",
         "IntervalSeconds": 1800,
         "Enabled": true
+      },
+      {
+        "Name": "SyncTorrentioJob",
+        "IntervalSeconds": 604800,
+        "Enabled": true
       }
     ]
   }

src/producer/Configuration/torrentio.json

@@ -0,0 +1,14 @@
+{
+  "TorrentioConfiguration": {
+    "Instances": [
+      {
+        "Name": "Official",
+        "Url": "https://torrentio.strem.fun",
+        "RateLimit": {
+          "RequestLimit": 300,
+          "IntervalInSeconds": 3600
+        }
+      }
+    ]
+  }
+}

src/producer/Extensions/ConfigurationExtensions.cs

@@ -11,6 +11,7 @@ public static class ConfigurationExtensions
         
         configuration.AddJsonFile(LoggingConfig, false, true);
         configuration.AddJsonFile(ScrapeConfiguration.Filename, false, true);
+        configuration.AddJsonFile(TorrentioConfiguration.Filename, false, true);
         
         configuration.AddEnvironmentVariables();
 
@@ -18,4 +19,28 @@ public static class ConfigurationExtensions
         
         return configuration;
     }
+
+    public static TConfiguration LoadConfigurationFromConfig<TConfiguration>(this IServiceCollection services, IConfiguration configuration, string sectionName)
+        where TConfiguration : class
+    {
+        var instance = configuration.GetSection(sectionName).Get<TConfiguration>();
+        
+        ArgumentNullException.ThrowIfNull(instance, nameof(instance));
+
+        services.TryAddSingleton(instance);
+
+        return instance;
+    }
+
+    public static TConfiguration LoadConfigurationFromEnv<TConfiguration>(this IServiceCollection services)
+        where TConfiguration : class
+    {
+        var instance = Activator.CreateInstance<TConfiguration>();
+        
+        ArgumentNullException.ThrowIfNull(instance, nameof(instance));
+
+        services.TryAddSingleton(instance);
+
+        return instance;
+    }
 }

src/producer/Extensions/ServiceCollectionExtensions.cs

@@ -1,160 +0,0 @@
-namespace Producer.Extensions;
-
-public static class ServiceCollectionExtensions
-{
-    internal static IServiceCollection AddCrawlers(this IServiceCollection services)
-    {
-        services.AddHttpClient("Scraper");
-
-        services
-            .AddKeyedTransient<ICrawler, EzTvCrawler>(nameof(EzTvCrawler))
-            .AddKeyedTransient<ICrawler, NyaaCrawler>(nameof(NyaaCrawler))
-            .AddKeyedTransient<ICrawler, YtsCrawler>(nameof(YtsCrawler))
-            .AddKeyedTransient<ICrawler, TpbCrawler>(nameof(TpbCrawler))
-            .AddKeyedTransient<ICrawler, TgxCrawler>(nameof(TgxCrawler))
-            .AddKeyedTransient<ICrawler, DebridMediaManagerCrawler>(nameof(DebridMediaManagerCrawler))
-            .AddSingleton<ICrawlerProvider, CrawlerProvider>()
-            .AddTransient<IIpService, IpService>();
-
-        return services;
-    }
-    
-    internal static IServiceCollection AddDataStorage(this IServiceCollection services)
-    {
-        services.LoadConfigurationFromEnv<PostgresConfiguration>();
-        services.AddTransient<IDataStorage, DapperDataStorage>();
-        services.AddTransient<IMessagePublisher, TorrentPublisher>();
-        return services;
-    }
-
-    internal static IServiceCollection RegisterMassTransit(this IServiceCollection services)
-    {
-        var rabbitConfig = services.LoadConfigurationFromEnv<RabbitMqConfiguration>();
-        
-        services.AddMassTransit(busConfigurator =>
-        {
-            busConfigurator.SetKebabCaseEndpointNameFormatter();
-            busConfigurator.UsingRabbitMq((_, busFactoryConfigurator) =>
-            {
-                busFactoryConfigurator.Host(rabbitConfig.Host, hostConfigurator =>
-                {
-                    hostConfigurator.Username(rabbitConfig.Username);
-                    hostConfigurator.Password(rabbitConfig.Password);
-                });
-            });
-        });
-
-        return services;
-    }
-    
-    internal static IServiceCollection AddQuartz(this IServiceCollection services, IConfiguration configuration)
-    {
-        var scrapeConfiguration = services.LoadConfigurationFromConfig<ScrapeConfiguration>(configuration, ScrapeConfiguration.SectionName);
-        var githubConfiguration = services.LoadConfigurationFromEnv<GithubConfiguration>();
-        var rabbitConfig = services.LoadConfigurationFromEnv<RabbitMqConfiguration>();
-
-        services
-            .AddTransient<SyncEzTvJob>()
-            .AddTransient<SyncNyaaJob>()
-            .AddTransient<SyncTpbJob>()
-            .AddTransient<SyncYtsJob>()
-            .AddTransient<SyncTgxJob>()
-            .AddTransient<IPJob>()
-            .AddTransient<PublisherJob>();
-
-        if (!string.IsNullOrEmpty(githubConfiguration.PAT))
-        {
-            services.AddTransient<SyncDmmJob>();
-        }
-
-        services.AddQuartz(
-            quartz =>
-            {
-               AddJobWithTrigger<SyncEzTvJob>(quartz, SyncEzTvJob.Key, SyncEzTvJob.Trigger, scrapeConfiguration);
-               AddJobWithTrigger<SyncNyaaJob>(quartz, SyncNyaaJob.Key, SyncNyaaJob.Trigger, scrapeConfiguration);
-               AddJobWithTrigger<SyncTpbJob>(quartz, SyncTpbJob.Key, SyncTpbJob.Trigger, scrapeConfiguration);
-               AddJobWithTrigger<SyncYtsJob>(quartz, SyncYtsJob.Key, SyncYtsJob.Trigger, scrapeConfiguration);
-               AddJobWithTrigger<SyncTgxJob>(quartz, SyncTgxJob.Key, SyncTgxJob.Trigger, scrapeConfiguration);
-               AddJobWithTrigger<IPJob>(quartz, IPJob.Key, IPJob.Trigger, 60 * 5);
-               AddJobWithTrigger<PublisherJob>(quartz, PublisherJob.Key, PublisherJob.Trigger, rabbitConfig.PublishIntervalInSeconds);
-
-                if (!string.IsNullOrEmpty(githubConfiguration.PAT))
-                {
-                     AddJobWithTrigger<SyncDmmJob>(quartz, SyncDmmJob.Key, SyncDmmJob.Trigger, scrapeConfiguration);
-                }
-            });
-
-        services.AddQuartzHostedService(
-            options =>
-            {
-                options.WaitForJobsToComplete = true;
-            });
-
-        return services;
-    }
-    
-    private static TConfiguration LoadConfigurationFromConfig<TConfiguration>(this IServiceCollection services, IConfiguration configuration, string sectionName)
-        where TConfiguration : class
-    {
-        var instance = configuration.GetSection(sectionName).Get<TConfiguration>();
-        
-        ArgumentNullException.ThrowIfNull(instance, nameof(instance));
-
-        services.TryAddSingleton(instance);
-
-        return instance;
-    }
-    
-    private static TConfiguration LoadConfigurationFromEnv<TConfiguration>(this IServiceCollection services)
-        where TConfiguration : class
-    {
-        var instance = Activator.CreateInstance<TConfiguration>();
-        
-        ArgumentNullException.ThrowIfNull(instance, nameof(instance));
-
-        services.TryAddSingleton(instance);
-
-        return instance;
-    }
-
-    private static void AddJobWithTrigger<TJobType>(
-        IServiceCollectionQuartzConfigurator quartz,
-        JobKey key,
-        TriggerKey trigger,
-        ScrapeConfiguration scrapeConfiguration) where TJobType : IJob
-    {
-        var scraper = scrapeConfiguration.Scrapers
-            .FirstOrDefault(x => x.Name != null && 
-                                 x.Name.Equals(typeof(TJobType).Name, StringComparison.OrdinalIgnoreCase));
-
-        if (scraper is null || !scraper.Enabled)
-        {
-            return;
-        }
-        
-        quartz.AddJob<TJobType>(opts => opts.WithIdentity(key).StoreDurably());
-
-        quartz.AddTrigger(
-            opts => opts
-                .ForJob(key)
-                .WithIdentity(trigger)
-                .StartAt(DateTimeOffset.Now.AddSeconds(20))
-                .WithSimpleSchedule(x => x.WithInterval(TimeSpan.FromSeconds(scraper.IntervalSeconds)).RepeatForever()));
-    }
-    
-    private static void AddJobWithTrigger<TJobType>(
-        IServiceCollectionQuartzConfigurator quartz,
-        JobKey key,
-        TriggerKey trigger,
-        int interval) where TJobType : IJob
-    {
-        quartz.AddJob<TJobType>(opts => opts.WithIdentity(key).StoreDurably());
-
-        quartz.AddTrigger(
-            opts => opts
-                .ForJob(key)
-                .WithIdentity(trigger)
-                .StartAt(DateTimeOffset.Now.AddSeconds(20))
-                .WithSimpleSchedule(x => x.WithInterval(TimeSpan.FromSeconds(interval)).RepeatForever()));
-    }
-}

src/producer/Features/Amqp/IMessagePublisher.cs

@@ -1,4 +1,4 @@
-namespace Producer.Interfaces;
+namespace Producer.Features.Amqp;
 
 public interface IMessagePublisher
 {

src/producer/Features/Amqp/PublisherJob.cs

@@ -1,11 +1,14 @@
-namespace Producer.Jobs;
+using Literals = Producer.Features.JobSupport.Literals;
+
+namespace Producer.Features.Amqp;
 
 [DisallowConcurrentExecution]
+[ManualJobRegistration]
 public class PublisherJob(IMessagePublisher publisher, IDataStorage storage, ILogger<PublisherJob> logger) : IJob
 {
     private const string JobName = nameof(PublisherJob);
-    public static readonly JobKey Key = new(JobName, nameof(Jobs));
+    public static readonly JobKey Key = new(JobName, nameof(Literals.PublishingJobs));
-    public static readonly TriggerKey Trigger = new($"{JobName}-trigger", nameof(Jobs));
+    public static readonly TriggerKey Trigger = new($"{JobName}-trigger", nameof(Literals.PublishingJobs));
     
     public async Task Execute(IJobExecutionContext context)
     {

src/producer/Features/Amqp/RabbitMqConfiguration.cs

@@ -1,4 +1,4 @@
-namespace Producer.Models.Configuration;
+namespace Producer.Features.Amqp;
 
 public class RabbitMqConfiguration
 {

src/producer/Features/Amqp/ServiceCollectionExtensions.cs

@@ -0,0 +1,24 @@
+namespace Producer.Features.Amqp;
+
+internal static class ServiceCollectionExtensions
+{
+    internal static IServiceCollection RegisterMassTransit(this IServiceCollection services)
+    {
+        var rabbitConfig = services.LoadConfigurationFromEnv<RabbitMqConfiguration>();
+        
+        services.AddMassTransit(busConfigurator =>
+        {
+            busConfigurator.SetKebabCaseEndpointNameFormatter();
+            busConfigurator.UsingRabbitMq((_, busFactoryConfigurator) =>
+            {
+                busFactoryConfigurator.Host(rabbitConfig.Host, hostConfigurator =>
+                {
+                    hostConfigurator.Username(rabbitConfig.Username);
+                    hostConfigurator.Password(rabbitConfig.Password);
+                });
+            });
+        });
+
+        return services;
+    }
+}

src/producer/Features/Amqp/TorrentPublisher.cs

@@ -1,4 +1,4 @@
-namespace Producer.Services;
+namespace Producer.Features.Amqp;
 
 public class TorrentPublisher(
     ISendEndpointProvider sendEndpointProvider,

src/producer/Features/CrawlerSupport/BaseCrawler.cs

@@ -1,4 +1,4 @@
-namespace Producer.Crawlers;
+namespace Producer.Features.CrawlerSupport;
 
 public abstract class BaseCrawler(ILogger<BaseCrawler> logger, IDataStorage storage) : ICrawler
 {

src/producer/Features/CrawlerSupport/BaseJsonCrawler.cs

@@ -1,8 +1,8 @@
-namespace Producer.Crawlers;
+namespace Producer.Features.CrawlerSupport;
 
 public abstract class BaseJsonCrawler(IHttpClientFactory httpClientFactory, ILogger<BaseJsonCrawler> logger, IDataStorage storage) : BaseCrawler(logger, storage)
 {
-    private readonly HttpClient _client = httpClientFactory.CreateClient("Scraper");
+    private readonly HttpClient _client = httpClientFactory.CreateClient(Literals.CrawlerClient);
     
     protected virtual async Task Execute(string collectionName)
     {

src/producer/Features/CrawlerSupport/BaseXmlCrawler.cs

@@ -1,4 +1,4 @@
-namespace Producer.Crawlers;
+namespace Producer.Features.CrawlerSupport;
 
 public abstract class BaseXmlCrawler(IHttpClientFactory httpClientFactory, ILogger<BaseXmlCrawler> logger, IDataStorage storage) : BaseCrawler(logger, storage)
 {
@@ -6,7 +6,7 @@ public abstract class BaseXmlCrawler(IHttpClientFactory httpClientFactory, ILogg
     {
         logger.LogInformation("Starting {Source} crawl", Source);
         
-        using var client = httpClientFactory.CreateClient("Scraper");
+        using var client = httpClientFactory.CreateClient(Literals.CrawlerClient);
         var xml = await client.GetStringAsync(Url);
         var xmlRoot = XElement.Parse(xml);
         

src/producer/Features/CrawlerSupport/CrawlerProvider.cs

@@ -1,4 +1,4 @@
-namespace Producer.Crawlers;
+namespace Producer.Features.CrawlerSupport;
 
 public class CrawlerProvider(IServiceProvider serviceProvider) : ICrawlerProvider
 {

src/producer/Features/CrawlerSupport/ICrawler.cs

@@ -1,4 +1,4 @@
-namespace Producer.Interfaces;
+namespace Producer.Features.CrawlerSupport;
 
 public interface ICrawler
 {

src/producer/Features/CrawlerSupport/ICrawlerProvider.cs

@@ -1,4 +1,4 @@
-namespace Producer.Interfaces;
+namespace Producer.Features.CrawlerSupport;
 
 public interface ICrawlerProvider
 {

src/producer/Features/CrawlerSupport/Literals.cs

@@ -0,0 +1,6 @@
+namespace Producer.Features.CrawlerSupport;
+
+public static class Literals
+{
+    public const string CrawlerClient = "Scraper";
+}

src/producer/Features/CrawlerSupport/Results.cs

@@ -1,4 +1,4 @@
-namespace Producer.Models;
+namespace Producer.Features.CrawlerSupport;
 
 public record InsertTorrentResult(bool Success, int InsertedCount = 0, string? ErrorMessage = null);
 public record UpdatedTorrentResult(bool Success, int UpdatedCount = 0, string? ErrorMessage = null);

src/producer/Features/CrawlerSupport/Scraper.cs

@@ -1,4 +1,4 @@
-namespace Producer.Models;
+namespace Producer.Features.CrawlerSupport;
 
 public class Scraper
 {

src/producer/Features/CrawlerSupport/ServiceCollectionExtensions.cs

@@ -0,0 +1,23 @@
+namespace Producer.Features.CrawlerSupport;
+
+internal static class ServiceCollectionExtensions
+{
+    internal static IServiceCollection AddCrawlers(this IServiceCollection services)
+    {
+        services.AddHttpClient(Literals.CrawlerClient);
+        
+        var crawlerTypes = Assembly.GetAssembly(typeof(ICrawler))
+            .GetTypes()
+            .Where(t => t is {IsClass: true, IsAbstract: false} && typeof(ICrawler).IsAssignableFrom(t));
+        
+        foreach (var type in crawlerTypes)
+        {
+            services.AddKeyedTransient(typeof(ICrawler), type.Name, type);
+        }
+
+        services
+            .AddSingleton<ICrawlerProvider, CrawlerProvider>();
+
+        return services;
+    }
+}

src/producer/Features/CrawlerSupport/Torrent.cs

@@ -1,4 +1,4 @@
-namespace Producer.Models;
+namespace Producer.Features.CrawlerSupport;
 
 // Torrent represents a crawled torrent from one of our
 // supported sources.

src/producer/Features/Crawlers/Dmm/DebridMediaManagerCrawler.cs

@@ -1,4 +1,4 @@
-namespace Producer.Crawlers.Sites;
+namespace Producer.Features.Crawlers.Dmm;
 
 public partial class DebridMediaManagerCrawler(
     IHttpClientFactory httpClientFactory,

src/producer/Features/Crawlers/Dmm/GithubConfiguration.cs

@@ -1,4 +1,4 @@
-namespace Producer.Models.Configuration;
+namespace Producer.Features.Crawlers.Dmm;
 
 public class GithubConfiguration
 {

src/producer/Features/Crawlers/Dmm/SyncDmmJob.cs

@@ -1,12 +1,13 @@
-using DebridMediaManagerCrawler = Producer.Crawlers.Sites.DebridMediaManagerCrawler;
+using Literals = Producer.Features.JobSupport.Literals;
 
-namespace Producer.Jobs;
+namespace Producer.Features.Crawlers.Dmm;
 
 [DisallowConcurrentExecution]
+[ManualJobRegistration]
 public class SyncDmmJob(ICrawlerProvider crawlerProvider) : BaseJob(crawlerProvider)
 {
     private const string JobName = nameof(DebridMediaManagerCrawler);
-    public static readonly JobKey Key = new(JobName, nameof(Crawlers));
+    public static readonly JobKey Key = new(JobName, nameof(Literals.CrawlersJobs));
-    public static readonly TriggerKey Trigger = new($"{JobName}-trigger", nameof(Crawlers));
+    public static readonly TriggerKey Trigger = new($"{JobName}-trigger", nameof(Literals.CrawlersJobs));
     protected override string Crawler => nameof(DebridMediaManagerCrawler);
 }

src/producer/Features/Crawlers/EzTv/EzTvCrawler.cs

@@ -1,4 +1,4 @@
-namespace Producer.Crawlers.Sites;
+namespace Producer.Features.Crawlers.EzTv;
 
 public class EzTvCrawler(IHttpClientFactory httpClientFactory, ILogger<EzTvCrawler> logger, IDataStorage storage) : BaseXmlCrawler(httpClientFactory, logger, storage)
 {

src/producer/Features/Crawlers/EzTv/SyncEzTvJob.cs

@@ -1,12 +1,12 @@
-using Producer.Crawlers.Sites;
+using Literals = Producer.Features.JobSupport.Literals;
 
-namespace Producer.Jobs;
+namespace Producer.Features.Crawlers.EzTv;
 
 [DisallowConcurrentExecution]
 public class SyncEzTvJob(ICrawlerProvider crawlerProvider) : BaseJob(crawlerProvider)
 {
     private const string JobName = nameof(EzTvCrawler);
-    public static readonly JobKey Key = new(JobName, nameof(Crawlers));
+    public static readonly JobKey Key = new(JobName, nameof(Literals.CrawlersJobs));
-    public static readonly TriggerKey Trigger = new($"{JobName}-trigger", nameof(Crawlers));
+    public static readonly TriggerKey Trigger = new($"{JobName}-trigger", nameof(Literals.CrawlersJobs));
     protected override string Crawler => nameof(EzTvCrawler);
 }

src/producer/Features/Crawlers/Nyaa/NyaaCrawler.cs

@@ -1,4 +1,4 @@
-namespace Producer.Crawlers.Sites;
+namespace Producer.Features.Crawlers.Nyaa;
 
 public class NyaaCrawler(IHttpClientFactory httpClientFactory, ILogger<NyaaCrawler> logger, IDataStorage storage) : BaseXmlCrawler(httpClientFactory, logger, storage)
 {

src/producer/Features/Crawlers/Nyaa/SyncNyaaJob.cs

@@ -1,12 +1,12 @@
-using Producer.Crawlers.Sites;
+using Literals = Producer.Features.JobSupport.Literals;
 
-namespace Producer.Jobs;
+namespace Producer.Features.Crawlers.Nyaa;
 
 [DisallowConcurrentExecution]
 public class SyncNyaaJob(ICrawlerProvider crawlerProvider) : BaseJob(crawlerProvider)
 {
     private const string JobName = nameof(NyaaCrawler);
-    public static readonly JobKey Key = new(JobName, nameof(Crawlers));
+    public static readonly JobKey Key = new(JobName, nameof(Literals.CrawlersJobs));
-    public static readonly TriggerKey Trigger = new($"{JobName}-trigger", nameof(Crawlers));
+    public static readonly TriggerKey Trigger = new($"{JobName}-trigger", nameof(Literals.CrawlersJobs));
     protected override string Crawler => nameof(NyaaCrawler);
 }

src/producer/Features/Crawlers/Tgx/SyncTgxJob.cs

@@ -1,12 +1,12 @@
-using TgxCrawler = Producer.Crawlers.Sites.TgxCrawler;
+using Literals = Producer.Features.JobSupport.Literals;
 
-namespace Producer.Jobs;
+namespace Producer.Features.Crawlers.Tgx;
 
 [DisallowConcurrentExecution]
 public class SyncTgxJob(ICrawlerProvider crawlerProvider) : BaseJob(crawlerProvider)
 {
     private const string JobName = nameof(TgxCrawler);
-    public static readonly JobKey Key = new(JobName, nameof(Crawlers));
+    public static readonly JobKey Key = new(JobName, nameof(Literals.CrawlersJobs));
-    public static readonly TriggerKey Trigger = new($"{JobName}-trigger", nameof(Crawlers));
+    public static readonly TriggerKey Trigger = new($"{JobName}-trigger", nameof(Literals.CrawlersJobs));
     protected override string Crawler => nameof(TgxCrawler);
 }

src/producer/Features/Crawlers/Tgx/TgxCrawler.cs

@@ -1,4 +1,4 @@
-namespace Producer.Crawlers.Sites;
+namespace Producer.Features.Crawlers.Tgx;
 
 public partial class TgxCrawler(IHttpClientFactory httpClientFactory, ILogger<TgxCrawler> logger, IDataStorage storage) : BaseXmlCrawler(httpClientFactory, logger, storage)
 {

src/producer/Features/Crawlers/Torrentio/SyncTorrentioJob.cs

@@ -0,0 +1,13 @@
+using Literals = Producer.Features.JobSupport.Literals;
+
+namespace Producer.Features.Crawlers.Torrentio;
+
+[DisallowConcurrentExecution]
+[ManualJobRegistration]
+public class SyncTorrentioJob(ICrawlerProvider crawlerProvider) : BaseJob(crawlerProvider)
+{
+    private const string JobName = nameof(TorrentioCrawler);
+    public static readonly JobKey Key = new(JobName, nameof(Literals.CrawlersJobs));
+    public static readonly TriggerKey Trigger = new($"{JobName}-trigger", nameof(Literals.CrawlersJobs));
+    protected override string Crawler => nameof(TorrentioCrawler);
+}

src/producer/Features/Crawlers/Torrentio/TorrentioConfiguration.cs

@@ -0,0 +1,9 @@
+namespace Producer.Features.Crawlers.Torrentio;
+
+public class TorrentioConfiguration
+{
+    public const string SectionName = "TorrentioConfiguration";
+    public const string Filename = "torrentio.json";
+    
+    public List<TorrentioInstance> Instances { get; set; } = [];
+}

src/producer/Features/Crawlers/Torrentio/TorrentioCrawler.cs

@@ -0,0 +1,197 @@
+using Literals = Producer.Features.CrawlerSupport.Literals;
+
+namespace Producer.Features.Crawlers.Torrentio;
+
+public partial class TorrentioCrawler(
+    IHttpClientFactory httpClientFactory,
+    ILogger<TorrentioCrawler> logger,
+    IDataStorage storage,
+    TorrentioConfiguration configuration,
+    ImdbMongoDbService imdbDataService) : BaseCrawler(logger, storage)
+{
+    [GeneratedRegex(@"(\d+(\.\d+)?) (GB|MB)")]
+    private static partial Regex SizeMatcher();
+    
+    private const string MovieSlug = "movie/{0}.json";
+    protected override string Url => "sort=size%7Cqualityfilter=other,scr,cam,unknown/stream/{0}";
+    protected override IReadOnlyDictionary<string, string> Mappings { get; } = new Dictionary<string, string>();
+    protected override string Source  => "Torrentio";
+    private readonly Dictionary<string, TorrentioScrapeInstance> _instanceStates = [];
+    public override async Task Execute()
+    {
+        var client = httpClientFactory.CreateClient(Literals.CrawlerClient);
+        var instances = configuration.Instances;
+        var totalRecordCount = await imdbDataService.GetTotalCountAsync();
+        logger.LogInformation("Total IMDB records to process: {TotalRecordCount}", totalRecordCount);
+        var tasks = instances.Select(x => ProcessForInstanceAsync(x, client, totalRecordCount)).ToArray();
+        await Task.WhenAll(tasks);
+    }
+
+    private Task ProcessForInstanceAsync(TorrentioInstance instance, HttpClient client, long totalRecordCount) =>
+        Task.Run(
+            async () =>
+            {
+                while (instance.TotalProcessedRequests(_instanceStates) < totalRecordCount)
+                {
+                    logger.LogInformation("Processing {TorrentioInstance}", instance.Name);
+                    logger.LogInformation("Current processed requests: {ProcessedRequests}", instance.TotalProcessedRequests(_instanceStates));
+                    
+                    var items = await imdbDataService.GetImdbEntriesForRequests(
+                        DateTime.UtcNow.Year.ToString(),
+                        instance.RateLimit.RequestLimit,
+                        instance.LastProcessedImdbId(_instanceStates));
+                    
+                    if (items.Count == 0)
+                    {
+                        logger.LogInformation("No items to process for {TorrentioInstance}", instance.Name);
+                        break;
+                    }
+                    
+                    var newTorrents = new List<Torrent>();
+                    var processedItemsCount = 0;
+
+                    foreach (var item in items)
+                    {
+                        processedItemsCount++;
+                        
+                        var waitTime = instance.CalculateWaitTime(_instanceStates);
+
+                        if (waitTime > TimeSpan.Zero)
+                        {
+                            logger.LogInformation("Rate limit reached for {TorrentioInstance}", instance.Name);
+                            logger.LogInformation("Waiting for {TorrentioInstance}: {WaitTime}", instance.Name, waitTime);
+                            await Task.Delay(waitTime);
+                        }
+                        
+                        if (processedItemsCount % 2 == 0)
+                        {
+                            var randomWait = new Random().Next(1000, 5000);
+                            logger.LogInformation("Waiting for {TorrentioInstance}: {WaitTime}", instance.Name, randomWait);
+                            await Task.Delay(randomWait);
+                        }
+                        
+                        try
+                        {
+                            var torrentInfo = await ScrapeInstance(instance, item.ImdbId, client);
+                            if (torrentInfo is not null)
+                            {
+                                newTorrents.AddRange(torrentInfo.Where(x => x != null).Select(x => x!));
+                            }
+                        }
+                        catch (Exception error)
+                        {
+                            logger.LogError(error, "page processing error in TorrentioCrawler");
+                        }
+                    }
+                    
+                    if (newTorrents.Count > 0)
+                    {
+                        await InsertTorrents(newTorrents);
+                        
+                        var currentState = _instanceStates[instance.Name];
+                        _instanceStates[instance.Name] = currentState with
+                        {
+                            LastProcessedImdbId = items[^1].ImdbId,
+                        };
+                    }
+                }
+            });
+
+    private async Task<List<Torrent?>?> ScrapeInstance(TorrentioInstance instance, string imdbId, HttpClient client)
+    {
+        logger.LogInformation("Searching Torrentio {TorrentioInstance}: {ImdbId}", instance.Name, imdbId);
+        try
+        {
+            var movieSlug = string.Format(MovieSlug, imdbId);
+            var urlSlug = string.Format(Url, movieSlug);
+            return await RunRequest(instance, urlSlug, imdbId, client);
+        }
+        catch (Exception error)
+        {
+            logger.LogError(error, "page processing error {TorrentioInstance}: {ImdbId}", instance.Name, imdbId);
+            logger.LogWarning("Setting {TorrentioInstance} to possibly rate limited for Five minutes", instance.Name);
+            instance.SetPossiblyRateLimited(_instanceStates);
+        }
+        return null;
+    }
+
+    private async Task<List<Torrent?>?> RunRequest(TorrentioInstance instance, string urlSlug, string imdbId, HttpClient client)
+    {
+        var requestUrl = $"{instance.Url}/{urlSlug}";
+        var response = await client.GetAsync(requestUrl);
+
+        if (!response.IsSuccessStatusCode)
+        {
+            logger.LogError("Failed to fetch {Url}", requestUrl);
+            return null;
+        }
+        
+        var json = JsonDocument.Parse(await response.Content.ReadAsStringAsync());
+        var streams = json.RootElement.GetProperty("streams").EnumerateArray();
+        return streams.Select(x => ParseTorrent(instance, x, imdbId)).Where(x => x != null).ToList();
+    }
+    
+    private Torrent? ParseTorrent(TorrentioInstance instance, JsonElement item, string imdId)
+    {
+        var title = item.GetProperty("title").GetString();
+        var infoHash = item.GetProperty("infoHash").GetString();
+        
+        if (string.IsNullOrEmpty(title) || string.IsNullOrEmpty(infoHash))
+        {
+            return null;
+        }
+        
+        var torrent = ParseTorrentDetails(title, instance, infoHash, imdId);
+        
+        if (torrent is null || string.IsNullOrEmpty(torrent.Name))
+        {
+            return null;
+        }
+        
+        return torrent;
+    }
+
+    private Torrent? ParseTorrentDetails(string title, TorrentioInstance instance, string infoHash, string imdbId)
+    {
+        try
+        {
+            var torrent = new Torrent
+            {
+                Source = $"{Source}_{instance.Name}",
+                InfoHash = infoHash,
+                Category = "movies", // we only handle movies for now...
+                Imdb = imdbId,
+            };
+
+            var span = title.AsSpan();
+            var titleEnd = span.IndexOf('\n');
+            var titlePart = titleEnd >= 0 ? span[..titleEnd].ToString() : title;
+
+            torrent.Name = titlePart.Replace('.', ' ').TrimEnd('.');
+
+            var sizeMatch = SizeMatcher().Match(title);
+
+            if (sizeMatch.Success)
+            {
+                var size = double.Parse(sizeMatch.Groups[1].Value); // Size Value
+                var sizeUnit = sizeMatch.Groups[3].Value; // Size Unit (GB/MB)
+
+                var sizeInBytes = sizeUnit switch
+                {
+                    "GB" => (long) (size * 1073741824),
+                    "MB" => (long) (size * 1048576),
+                    _ => 0,
+                };
+
+                torrent.Size = sizeInBytes.ToString();
+            }
+
+            return torrent;
+        }
+        catch (Exception e)
+        {
+            logger.LogError(e, "Error parsing torrent details");
+            return null;
+        }
+    }
+}

src/producer/Features/Crawlers/Torrentio/TorrentioInstance.cs

@@ -0,0 +1,10 @@
+namespace Producer.Features.Crawlers.Torrentio;
+
+public class TorrentioInstance
+{
+    public string Name { get; init; } = default!;
+
+    public string Url { get; init; } = default!;
+
+    public TorrentioRateLimit RateLimit { get; init; } = default!;
+}

src/producer/Features/Crawlers/Torrentio/TorrentioInstanceExtensions.cs

@@ -0,0 +1,53 @@
+namespace Producer.Features.Crawlers.Torrentio;
+
+public static class TorrentioInstancesExtensions
+{
+    public static TimeSpan CalculateWaitTime(this TorrentioInstance instance, Dictionary<string, TorrentioScrapeInstance> scraperState)
+    {
+        if (!scraperState.TryGetValue(instance.Name, out var state))
+        {
+            state = new (DateTime.UtcNow, 0, 0, null);
+            scraperState[instance.Name] = state;
+        }
+
+        var (startedAt, requestCount, totalProcessed, lastProcessedImdbId) = state;
+        
+        if (requestCount < instance.RateLimit.RequestLimit)
+        {
+            scraperState[instance.Name] = new (startedAt, requestCount + 1, totalProcessed + 1, lastProcessedImdbId);
+            return TimeSpan.Zero;
+        }
+
+        var elapsed = DateTime.UtcNow - startedAt;
+        var interval = TimeSpan.FromSeconds(instance.RateLimit.IntervalInSeconds);
+        var remaining = interval - elapsed;
+
+        // reset the state for the next interval
+        scraperState[instance.Name] = new (DateTime.UtcNow, 0, totalProcessed, lastProcessedImdbId);
+
+        return remaining > TimeSpan.Zero ? remaining : TimeSpan.Zero;
+    }
+    
+    public static void SetPossiblyRateLimited(this TorrentioInstance instance, Dictionary<string, TorrentioScrapeInstance> scraperState, int minutesToWait = 5)
+    {
+        if (!scraperState.TryGetValue(instance.Name, out var state))
+        {
+            state = new (DateTime.UtcNow, 0, 0, null);
+        }
+
+        var (_, _, totalProcessed, lastProcessedImdbId) = state;
+
+        // Set the start time to 15 minutes in the past so that the next check will result in a rate limit period of 15 minutes
+        var startedAt = DateTime.UtcNow.AddMinutes(-minutesToWait);
+        var requestCount = instance.RateLimit.RequestLimit;
+
+        // Update the scraper state for the instance
+        scraperState[instance.Name] = new (startedAt, requestCount, totalProcessed, lastProcessedImdbId);
+    }
+    
+    public static long TotalProcessedRequests(this TorrentioInstance instance, Dictionary<string, TorrentioScrapeInstance> scraperState) => 
+        !scraperState.TryGetValue(instance.Name, out var state) ? 0 : state.TotalProcessed;
+    
+    public static string? LastProcessedImdbId(this TorrentioInstance instance, Dictionary<string, TorrentioScrapeInstance> scraperState) => 
+        !scraperState.TryGetValue(instance.Name, out var state) ? null : state.LastProcessedImdbId;
+}

src/producer/Features/Crawlers/Torrentio/TorrentioRateLimit.cs

@@ -0,0 +1,7 @@
+namespace Producer.Features.Crawlers.Torrentio;
+
+public class TorrentioRateLimit
+{
+    public int RequestLimit { get; set; }
+    public int IntervalInSeconds { get; set; }
+}

src/producer/Features/Crawlers/Torrentio/TorrentioScrapeInstance.cs

@@ -0,0 +1,3 @@
+namespace Producer.Features.Crawlers.Torrentio;
+
+public record TorrentioScrapeInstance(DateTime StartedAt, int RequestCount, int TotalProcessed, string? LastProcessedImdbId);

src/producer/Features/Crawlers/Tpb/SyncTpbJob.cs

@@ -1,12 +1,12 @@
-using Producer.Crawlers.Sites;
+using Literals = Producer.Features.JobSupport.Literals;
 
-namespace Producer.Jobs;
+namespace Producer.Features.Crawlers.Tpb;
 
 [DisallowConcurrentExecution]
 public class SyncTpbJob(ICrawlerProvider crawlerProvider) : BaseJob(crawlerProvider)
 {
     private const string JobName = nameof(TpbCrawler);
-    public static readonly JobKey Key = new(JobName, nameof(Crawlers));
+    public static readonly JobKey Key = new(JobName, nameof(Literals.CrawlersJobs));
-    public static readonly TriggerKey Trigger = new($"{JobName}-trigger", nameof(Crawlers));
+    public static readonly TriggerKey Trigger = new($"{JobName}-trigger", nameof(Literals.CrawlersJobs));
     protected override string Crawler => nameof(TpbCrawler);
 }

src/producer/Features/Crawlers/Tpb/TpbCrawler.cs

@@ -1,4 +1,4 @@
-namespace Producer.Crawlers.Sites;
+namespace Producer.Features.Crawlers.Tpb;
 
 public class TpbCrawler(IHttpClientFactory httpClientFactory, ILogger<TpbCrawler> logger, IDataStorage storage) : BaseJsonCrawler(httpClientFactory, logger, storage)
 {

src/producer/Features/Crawlers/Yts/SyncYtsJob.cs

@@ -1,12 +1,12 @@
-using Producer.Crawlers.Sites;
+using Literals = Producer.Features.JobSupport.Literals;
 
-namespace Producer.Jobs;
+namespace Producer.Features.Crawlers.Yts;
 
 [DisallowConcurrentExecution]
 public class SyncYtsJob(ICrawlerProvider crawlerProvider) : BaseJob(crawlerProvider)
 {
     private const string JobName = nameof(YtsCrawler);
-    public static readonly JobKey Key = new(JobName, nameof(Crawlers));
+    public static readonly JobKey Key = new(JobName, nameof(Literals.CrawlersJobs));
-    public static readonly TriggerKey Trigger = new($"{JobName}-trigger", nameof(Crawlers));
+    public static readonly TriggerKey Trigger = new($"{JobName}-trigger", nameof(Literals.CrawlersJobs));
     protected override string Crawler => nameof(YtsCrawler);
 }

src/producer/Features/Crawlers/Yts/YtsCrawler.cs

@@ -1,4 +1,4 @@
-namespace Producer.Crawlers.Sites;
+namespace Producer.Features.Crawlers.Yts;
 
 public class YtsCrawler(IHttpClientFactory httpClientFactory, ILogger<YtsCrawler> logger, IDataStorage storage) : BaseXmlCrawler(httpClientFactory, logger, storage)
 {

src/producer/Features/DataProcessing/DapperDataStorage.cs

@@ -1,4 +1,4 @@
-namespace Producer.Services;
+namespace Producer.Features.DataProcessing;
 
 public class DapperDataStorage(PostgresConfiguration configuration, RabbitMqConfiguration rabbitConfig, ILogger<DapperDataStorage> logger) : IDataStorage
 {

src/producer/Features/DataProcessing/IDataStorage.cs

@@ -1,4 +1,4 @@
-namespace Producer.Interfaces;
+namespace Producer.Features.DataProcessing;
 
 public interface IDataStorage
 {

src/producer/Features/DataProcessing/ImdbEntry.cs

@@ -0,0 +1,15 @@
+namespace Producer.Features.DataProcessing;
+
+public class ImdbEntry
+{
+    [BsonId]
+    public string ImdbId { get; set; } = default!;
+    public string? TitleType { get; set; }
+    public string? PrimaryTitle { get; set; }
+    public string? OriginalTitle { get; set; }
+    public string? IsAdult { get; set; }
+    public string? StartYear { get; set; }
+    public string? EndYear { get; set; }
+    public string? RuntimeMinutes { get; set; }
+    public string? Genres { get; set; }
+}

src/producer/Features/DataProcessing/ImdbMongoDbService.cs

@@ -0,0 +1,78 @@
+namespace Producer.Features.DataProcessing;
+
+public class ImdbMongoDbService
+{
+    private readonly ILogger<ImdbMongoDbService> _logger;
+    private readonly IMongoCollection<ImdbEntry> _imdbCollection;
+
+    public ImdbMongoDbService(MongoConfiguration configuration, ILogger<ImdbMongoDbService> logger)
+    {
+        _logger = logger;
+        
+        var client = new MongoClient(configuration.ConnectionString);
+        var database = client.GetDatabase(configuration.DbName);
+
+        _imdbCollection = database.GetCollection<ImdbEntry>("imdb-entries");
+    }
+    
+    public async Task<IReadOnlyList<ImdbEntry>> GetImdbEntriesForRequests(string startYear, int requestLimit, string? startingId = null)
+    {
+        var sort = Builders<ImdbEntry>.Sort
+            .Descending(e => e.StartYear)
+            .Descending(e => e.ImdbId);
+        
+        var filter = Builders<ImdbEntry>.Filter
+            .And(
+                Builders<ImdbEntry>.Filter.Eq(e => e.TitleType, "movie"),
+                Builders<ImdbEntry>.Filter.Lte(e => e.StartYear, startYear)
+            );
+        
+        if (!string.IsNullOrWhiteSpace(startingId))
+        {
+            filter = Builders<ImdbEntry>.Filter.And(filter, Builders<ImdbEntry>.Filter.Lt(e => e.ImdbId, startingId));
+        }
+
+        return await _imdbCollection.Find(filter).Limit(requestLimit).Sort(sort).ToListAsync();
+    }
+    
+    public async Task<long> GetTotalCountAsync()
+    {
+        var filter = Builders<ImdbEntry>.Filter.Eq(x => x.TitleType, "movie");
+        return await _imdbCollection.CountDocumentsAsync(filter);
+    }
+
+    public bool IsDatabaseInitialized()
+    {
+        try
+        {
+            // Compound index for PrimaryTitle, TitleType, and StartYear
+            var index1KeysDefinition = Builders<ImdbEntry>.IndexKeys
+                .Text(e => e.PrimaryTitle)
+                .Ascending(e => e.TitleType)
+                .Ascending(e => e.StartYear);
+            
+            CreateIndex(index1KeysDefinition);
+
+            // Compound index for StartYear and _id in descending order
+            var index2KeysDefinition = Builders<ImdbEntry>.IndexKeys
+                .Descending(e => e.StartYear)
+                .Descending(e => e.ImdbId);
+            
+            CreateIndex(index2KeysDefinition);
+
+            return true;
+        }
+        catch (Exception e)
+        {
+            _logger.LogError(e, "Error initializing database");
+            return false;
+        }
+    }
+    
+    private void CreateIndex(IndexKeysDefinition<ImdbEntry> keysDefinition)
+    {
+        var createIndexOptions = new CreateIndexOptions { Background = true };
+        var indexModel = new CreateIndexModel<ImdbEntry>(keysDefinition, createIndexOptions);
+        _imdbCollection.Indexes.CreateOne(indexModel);
+    }
+}

src/producer/Features/DataProcessing/MongoConfiguration.cs

@@ -0,0 +1,20 @@
+namespace Producer.Features.DataProcessing;
+
+public class MongoConfiguration
+{
+    private const string Prefix = "MONGODB";
+    private const string HostVariable = "HOST";
+    private const string PortVariable = "PORT";
+    private const string DbVariable = "DB";
+    private const string UsernameVariable = "USER";
+    private const string PasswordVariable = "PASSWORD";
+    
+
+    private string Host { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(HostVariable);
+    private int Port { get; init; } = Prefix.GetEnvironmentVariableAsInt(PortVariable, 27017);
+    private string Username { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(UsernameVariable);
+    private string Password { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(PasswordVariable);
+    public string DbName { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(DbVariable);
+    
+    public string ConnectionString => $"mongodb://{Username}:{Password}@{Host}:{Port}/{DbName}?tls=false&directConnection=true&authSource=admin";
+}

src/producer/Features/DataProcessing/PostgresConfiguration.cs

@@ -1,4 +1,4 @@
-namespace Producer.Models.Configuration;
+namespace Producer.Features.DataProcessing;
 
 public class PostgresConfiguration
 {

src/producer/Features/DataProcessing/ServiceCollectionExtensions.cs

@@ -0,0 +1,14 @@
+namespace Producer.Features.DataProcessing;
+
+internal static class ServiceCollectionExtensions
+{
+    internal static IServiceCollection AddDataStorage(this IServiceCollection services)
+    {
+        services.LoadConfigurationFromEnv<PostgresConfiguration>();
+        services.LoadConfigurationFromEnv<MongoConfiguration>();
+        services.AddTransient<IDataStorage, DapperDataStorage>();
+        services.AddTransient<IMessagePublisher, TorrentPublisher>();
+        services.AddSingleton<ImdbMongoDbService>();
+        return services;
+    }
+}

src/producer/Features/JobSupport/BaseJob.cs

@@ -1,4 +1,4 @@
-namespace Producer.Jobs;
+namespace Producer.Features.JobSupport;
 
 public abstract class BaseJob(ICrawlerProvider crawlerProvider) : IJob
 {

src/producer/Features/JobSupport/ICrawlerJob.cs

@@ -1,4 +1,4 @@
-namespace Producer.Jobs;
+namespace Producer.Features.JobSupport;
 
 public interface ICrawlerJob<out TCrawler> : IJob
     where TCrawler : ICrawler

src/producer/Features/JobSupport/Literals.cs

@@ -0,0 +1,7 @@
+namespace Producer.Features.JobSupport;
+
+public static class Literals
+{
+    public const string CrawlersJobs = "CrawlersJobs";
+    public const string PublishingJobs = "PublishingJobs";
+}

src/producer/Features/JobSupport/ManualJobRegistrationAttribute.cs

@@ -0,0 +1,6 @@
+namespace Producer.Features.JobSupport;
+
+[AttributeUsage(AttributeTargets.Class)]
+public class ManualJobRegistrationAttribute : Attribute
+{
+}

src/producer/Features/JobSupport/ScrapeConfiguration.cs

@@ -1,4 +1,4 @@
-namespace Producer.Models.Configuration;
+namespace Producer.Features.JobSupport;
 
 public class ScrapeConfiguration
 {

src/producer/Features/JobSupport/ServiceCollectionExtensions.cs

@@ -0,0 +1,131 @@
+namespace Producer.Features.JobSupport;
+
+internal static class ServiceCollectionExtensions
+{
+    internal static IServiceCollection AddQuartz(this IServiceCollection services, IConfiguration configuration)
+    {
+        var scrapeConfiguration = services.LoadConfigurationFromConfig<ScrapeConfiguration>(configuration, ScrapeConfiguration.SectionName);
+        var githubConfiguration = services.LoadConfigurationFromEnv<GithubConfiguration>();
+        var rabbitConfiguration = services.LoadConfigurationFromEnv<RabbitMqConfiguration>();
+        
+        var jobTypes = Assembly.GetAssembly(typeof(BaseJob))
+            .GetTypes()
+            .Where(t => t is {IsClass: true, IsAbstract: false} && typeof(IJob).IsAssignableFrom(t) &&
+            !Attribute.IsDefined(t, typeof(ManualJobRegistrationAttribute)))
+            .ToList();
+
+        foreach (var type in jobTypes)
+        {
+            services.AddTransient(type);
+        }
+
+        if (!string.IsNullOrEmpty(githubConfiguration.PAT))
+        {
+            services.AddTransient<SyncDmmJob>();
+        }
+        
+        var openMethod = typeof(ServiceCollectionExtensions).GetMethod(nameof(AddJobWithTrigger), BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Instance);
+
+        services.AddQuartz(
+            quartz =>
+            {
+                RegisterAutomaticRegistrationJobs(jobTypes, openMethod, quartz, scrapeConfiguration);
+                RegisterDmmJob(githubConfiguration, quartz, scrapeConfiguration);
+                RegisterTorrentioJob(services, quartz, configuration, scrapeConfiguration);
+                RegisterPublisher(quartz, rabbitConfiguration);
+            });
+
+        services.AddQuartzHostedService(
+            options =>
+            {
+                options.WaitForJobsToComplete = true;
+            });
+
+        return services;
+    }
+
+    private static void RegisterAutomaticRegistrationJobs(List<Type> jobTypes, MethodInfo? openMethod, IServiceCollectionQuartzConfigurator quartz,
+        ScrapeConfiguration scrapeConfiguration)
+    {
+        foreach (var jobType in jobTypes)
+        {
+            var key = jobType.GetField("Key")?.GetValue(jobType);
+            var trigger = jobType.GetField("Trigger")?.GetValue(jobType);
+                    
+            if (key is null || trigger is null)
+            {
+                Console.WriteLine($"Job {jobType.Name} does not have a JobKey or TriggerKey property");
+                continue;
+            }
+            
+            var method = openMethod.MakeGenericMethod(jobType);
+            method.Invoke(null, [quartz, key, trigger, scrapeConfiguration]);
+        }
+    }
+    
+    private static void RegisterDmmJob(GithubConfiguration githubConfiguration, IServiceCollectionQuartzConfigurator quartz, ScrapeConfiguration scrapeConfiguration)
+    {
+        if (!string.IsNullOrEmpty(githubConfiguration.PAT))
+        {
+            AddJobWithTrigger<SyncDmmJob>(quartz, SyncDmmJob.Key, SyncDmmJob.Trigger, scrapeConfiguration);
+        }
+    }
+    
+    private static void RegisterTorrentioJob(
+        IServiceCollection services,
+        IServiceCollectionQuartzConfigurator quartz,
+        IConfiguration configuration,
+        ScrapeConfiguration scrapeConfiguration)
+    {
+       var torrentioConfiguration = services.LoadConfigurationFromConfig<TorrentioConfiguration>(configuration, TorrentioConfiguration.SectionName);
+       
+       if (torrentioConfiguration.Instances.Count != 0)
+       {
+           AddJobWithTrigger<SyncTorrentioJob>(quartz, SyncTorrentioJob.Key, SyncTorrentioJob.Trigger, scrapeConfiguration);
+       }
+    }
+
+    private static void RegisterPublisher(IServiceCollectionQuartzConfigurator quartz, RabbitMqConfiguration rabbitConfig) => 
+        AddJobWithTriggerAndInterval<PublisherJob>(quartz, PublisherJob.Key, PublisherJob.Trigger, rabbitConfig.PublishIntervalInSeconds);
+
+    private static void AddJobWithTrigger<TJobType>(
+        IServiceCollectionQuartzConfigurator quartz,
+        JobKey key,
+        TriggerKey trigger,
+        ScrapeConfiguration scrapeConfiguration) where TJobType : IJob
+    {
+        var scraper = scrapeConfiguration.Scrapers
+            .FirstOrDefault(x => x.Name != null && 
+                                 x.Name.Equals(typeof(TJobType).Name, StringComparison.OrdinalIgnoreCase));
+
+        if (scraper is null || !scraper.Enabled)
+        {
+            return;
+        }
+        
+        quartz.AddJob<TJobType>(opts => opts.WithIdentity(key).StoreDurably());
+
+        quartz.AddTrigger(
+            opts => opts
+                .ForJob(key)
+                .WithIdentity(trigger)
+                .StartAt(DateTimeOffset.Now.AddSeconds(20))
+                .WithSimpleSchedule(x => x.WithInterval(TimeSpan.FromSeconds(scraper.IntervalSeconds)).RepeatForever()));
+    }
+    
+    private static void AddJobWithTriggerAndInterval<TJobType>(
+        IServiceCollectionQuartzConfigurator quartz,
+        JobKey key,
+        TriggerKey trigger,
+        int interval) where TJobType : IJob
+    {
+        quartz.AddJob<TJobType>(opts => opts.WithIdentity(key).StoreDurably());
+    
+        quartz.AddTrigger(
+            opts => opts
+                .ForJob(key)
+                .WithIdentity(trigger)
+                .StartAt(DateTimeOffset.Now.AddSeconds(20))
+                .WithSimpleSchedule(x => x.WithInterval(TimeSpan.FromSeconds(interval)).RepeatForever()));
+    }
+}

src/producer/GlobalUsings.cs

@@ -1,9 +1,10 @@
 // Global using directives
 
+global using System.Reflection;
 global using System.Text;
 global using System.Text.Json;
-global using System.Text.Json.Serialization;
 global using System.Text.RegularExpressions;
+global using System.Threading.Channels;
 global using System.Xml.Linq;
 global using Dapper;
 global using LZStringCSharp;
@@ -12,14 +13,15 @@ global using Microsoft.AspNetCore.Builder;
 global using Microsoft.Extensions.DependencyInjection;
 global using Microsoft.Extensions.DependencyInjection.Extensions;
 global using Microsoft.Extensions.Logging;
+global using MongoDB.Bson.Serialization.Attributes;
+global using MongoDB.Driver;
 global using Npgsql;
 global using Quartz;
-global using Producer.Crawlers;
-global using Producer.Crawlers.Sites;
 global using Producer.Extensions;
-global using Producer.Interfaces;
+global using Producer.Features.Amqp;
-global using Producer.Jobs;
+global using Producer.Features.Crawlers.Dmm;
-global using Producer.Models;
+global using Producer.Features.Crawlers.Torrentio;
-global using Producer.Models.Configuration;
+global using Producer.Features.CrawlerSupport;
-global using Producer.Services;
+global using Producer.Features.DataProcessing;
+global using Producer.Features.JobSupport;
 global using Serilog;

src/producer/Interfaces/IIpService.cs

@@ -1,6 +0,0 @@
-namespace Producer.Interfaces;
-
-public interface IIpService
-{
-    Task GetPublicIpAddress();
-}

src/producer/Jobs/IPJob.cs

@@ -1,14 +0,0 @@
-namespace Producer.Jobs;
-
-[DisallowConcurrentExecution]
-public class IPJob(IIpService ipService) : IJob
-{
-    private const string JobName = nameof(IPJob);
-    public static readonly JobKey Key = new(JobName, nameof(Jobs));
-    public static readonly TriggerKey Trigger = new($"{JobName}-trigger", nameof(Jobs));
-    
-    public Task Execute(IJobExecutionContext context)
-    {
-        return ipService.GetPublicIpAddress();
-    }
-}

src/producer/Producer.csproj

@@ -15,6 +15,7 @@
       <PackageReference Include="MassTransit.RabbitMQ" Version="8.1.3" />
       <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
       <PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
+      <PackageReference Include="MongoDB.Driver" Version="2.24.0" />
       <PackageReference Include="Npgsql" Version="8.0.1" />
       <PackageReference Include="Quartz.Extensions.DependencyInjection" Version="3.8.0" />
       <PackageReference Include="Quartz.Extensions.Hosting" Version="3.8.0" />
@@ -24,12 +25,8 @@
     </ItemGroup>
 
     <ItemGroup>
-      <Content Remove="Configuration\scrapers.json" />
+      <Content Remove="Configuration\*.json" />
-      <None Include="Configuration\scrapers.json">
+      <None Include="Configuration\*.json">
-        <CopyToOutputDirectory>Always</CopyToOutputDirectory>
-      </None>
-      <Content Remove="Configuration\logging.json" />
-      <None Include="Configuration\logging.json">
         <CopyToOutputDirectory>Always</CopyToOutputDirectory>
       </None>
     </ItemGroup>

src/producer/Services/IpService.cs

@@ -1,13 +0,0 @@
-namespace Producer.Services;
-
-public class IpService(ILogger<IpService> logger, IHttpClientFactory httpClientFactory) : IIpService
-{
-    public async Task GetPublicIpAddress()
-    {
-        var client = httpClientFactory.CreateClient("Scraper");
-        client.DefaultRequestHeaders.UserAgent.ParseAdd("curl");
-        var request = await client.GetStringAsync("http://ifconfig.me");
-
-        logger.LogInformation("Public IP Address: {PublicIPAddress}", request);
-    }
-}