Big rewrite - distributed consumers for ingestion / scraping(scalable) - single producer written in c#.
Changed from page scraping to rss xml scraping Includes RealDebridManager hashlist decoding (requires a github readonly PAT as requests must be authenticated) - This allows ingestion of 200k+ entries in a few hours. Simplifies a lot of torrentio to deal with new data
This commit is contained in:
23
src/producer/Extensions/ConfigurationExtensions.cs
Normal file
23
src/producer/Extensions/ConfigurationExtensions.cs
Normal file
@@ -0,0 +1,23 @@
|
||||
namespace Scraper.Extensions;
|
||||
|
||||
public static class ConfigurationExtensions
|
||||
{
|
||||
private const string ConfigurationFolder = "Configuration";
|
||||
private const string LoggingConfig = "logging.json";
|
||||
|
||||
public static IConfigurationBuilder AddScrapeConfiguration(this IConfigurationBuilder configuration)
|
||||
{
|
||||
configuration.SetBasePath(Path.Combine(AppContext.BaseDirectory, ConfigurationFolder));
|
||||
|
||||
configuration.AddJsonFile(LoggingConfig, false, true);
|
||||
configuration.AddJsonFile(ScrapeConfiguration.Filename, false, true);
|
||||
configuration.AddJsonFile(RabbitMqConfiguration.Filename, false, true);
|
||||
configuration.AddJsonFile(GithubConfiguration.Filename, false, true);
|
||||
|
||||
configuration.AddEnvironmentVariables();
|
||||
|
||||
configuration.AddUserSecrets<Program>();
|
||||
|
||||
return configuration;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user