mirror of
https://github.com/knightcrawler-stremio/knightcrawler.git
synced 2024-12-20 03:29:51 +00:00
Merge pull request #102 from iPromKnight/rabbit-envvars
BREAKING: Cleanup RabbitMQ env vars, and Github Pat
This commit is contained in:
@@ -15,18 +15,28 @@ MONGODB_DB=knightcrawler
|
|||||||
MONGODB_USER=mongo
|
MONGODB_USER=mongo
|
||||||
MONGODB_PASSWORD=mongo
|
MONGODB_PASSWORD=mongo
|
||||||
|
|
||||||
|
# RabbitMQ
|
||||||
|
RABBITMQ_HOST=rabbitmq
|
||||||
|
RABBITMQ_USER=guest
|
||||||
|
RABBITMQ_PASSWORD=guest
|
||||||
|
RABBITMQ_QUEUE_NAME=ingested
|
||||||
|
RABBITMQ_DURABLE=true
|
||||||
|
RABBITMQ_MAX_QUEUE_SIZE=0
|
||||||
|
RABBITMQ_MAX_PUBLISH_BATCH_SIZE=500
|
||||||
|
RABBITMQ_PUBLISH_INTERVAL_IN_SECONDS=10
|
||||||
|
|
||||||
# Metadata
|
# Metadata
|
||||||
## Only used if DATA_ONCE is set to false. If true, the schedule is ignored
|
## Only used if DATA_ONCE is set to false. If true, the schedule is ignored
|
||||||
METADATA_DOWNLOAD_IMDB_DATA_SCHEDULE=0 0 1 * * *
|
METADATA_DOWNLOAD_IMDB_DATA_SCHEDULE="0 0 1 * *"
|
||||||
## If true, the metadata will be downloaded once and then the schedule will be ignored
|
## If true, the metadata will be downloaded once and then the schedule will be ignored
|
||||||
METADATA_DOWNLOAD_IMDB_DATA_ONCE=true
|
METADATA_DOWNLOAD_IMDB_DATA_ONCE=true
|
||||||
|
## Controls the amount of records processed in memory at any given time during import, higher values will consume more memory
|
||||||
|
METADATA_INSERT_BATCH_SIZE=25000
|
||||||
|
|
||||||
# Addon
|
# Addon
|
||||||
DEBUG_MODE=false
|
DEBUG_MODE=false
|
||||||
|
|
||||||
# Consumer
|
# Consumer
|
||||||
RABBIT_URI=amqp://guest:guest@rabbitmq:5672/?heartbeat=30
|
|
||||||
QUEUE_NAME=ingested
|
|
||||||
JOB_CONCURRENCY=5
|
JOB_CONCURRENCY=5
|
||||||
JOBS_ENABLED=true
|
JOBS_ENABLED=true
|
||||||
## can be debug for extra verbosity (a lot more verbosity - useful for development)
|
## can be debug for extra verbosity (a lot more verbosity - useful for development)
|
||||||
@@ -40,12 +50,4 @@ CONSUMER_REPLICAS=3
|
|||||||
AUTO_CREATE_AND_APPLY_MIGRATIONS=false
|
AUTO_CREATE_AND_APPLY_MIGRATIONS=false
|
||||||
|
|
||||||
# Producer
|
# Producer
|
||||||
RabbitMqConfiguration__Host=rabbitmq
|
GITHUB_PAT=
|
||||||
RabbitMqConfiguration__QueueName=ingested
|
|
||||||
RabbitMqConfiguration__Username=guest
|
|
||||||
RabbitMqConfiguration__Password=guest
|
|
||||||
RabbitMqConfiguration__Durable=true
|
|
||||||
RabbitMqConfiguration__MaxQueueSize=0
|
|
||||||
RabbitMqConfiguration__MaxPublishBatchSize=500
|
|
||||||
RabbitMqConfiguration__PublishIntervalInSeconds=10
|
|
||||||
GithubSettings__PAT=
|
|
||||||
|
|||||||
@@ -5,7 +5,9 @@ public class JobConfiguration
|
|||||||
private const string Prefix = "METADATA";
|
private const string Prefix = "METADATA";
|
||||||
private const string DownloadImdbDataVariable = "DOWNLOAD_IMDB_DATA_SCHEDULE";
|
private const string DownloadImdbDataVariable = "DOWNLOAD_IMDB_DATA_SCHEDULE";
|
||||||
private const string DownloadImdbDataOnceVariable = "DOWNLOAD_IMDB_DATA_ONCE";
|
private const string DownloadImdbDataOnceVariable = "DOWNLOAD_IMDB_DATA_ONCE";
|
||||||
|
private const string InsertBatchSizeVariable = "INSERT_BATCH_SIZE";
|
||||||
|
|
||||||
|
public int InsertBatchSize { get; init; } = Prefix.GetEnvironmentVariableAsInt(InsertBatchSizeVariable, 25_000);
|
||||||
public string DownloadImdbCronSchedule { get; init; } = Prefix.GetOptionalEnvironmentVariableAsString(DownloadImdbDataVariable, CronExpressions.EveryHour);
|
public string DownloadImdbCronSchedule { get; init; } = Prefix.GetOptionalEnvironmentVariableAsString(DownloadImdbDataVariable, CronExpressions.EveryHour);
|
||||||
public bool DownloadImdbOnce { get; init; } = Prefix.GetEnvironmentVariableAsBool(DownloadImdbDataOnceVariable);
|
public bool DownloadImdbOnce { get; init; } = Prefix.GetEnvironmentVariableAsBool(DownloadImdbDataOnceVariable);
|
||||||
}
|
}
|
||||||
@@ -8,6 +8,7 @@ public class MongoConfiguration
|
|||||||
private const string DbVariable = "DB";
|
private const string DbVariable = "DB";
|
||||||
private const string UsernameVariable = "USER";
|
private const string UsernameVariable = "USER";
|
||||||
private const string PasswordVariable = "PASSWORD";
|
private const string PasswordVariable = "PASSWORD";
|
||||||
|
|
||||||
|
|
||||||
private string Host { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(HostVariable);
|
private string Host { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(HostVariable);
|
||||||
private int Port { get; init; } = Prefix.GetEnvironmentVariableAsInt(PortVariable, 27017);
|
private int Port { get; init; } = Prefix.GetEnvironmentVariableAsInt(PortVariable, 27017);
|
||||||
|
|||||||
@@ -42,11 +42,16 @@ public class ImdbMongoDbService
|
|||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
|
// Create compound index for PrimaryTitle, TitleType, and StartYear
|
||||||
// Create index for PrimaryTitle
|
var indexKeysDefinition = Builders<ImdbEntry>.IndexKeys
|
||||||
var indexPrimaryTitle = Builders<ImdbEntry>.IndexKeys.Ascending(e => e.PrimaryTitle);
|
.Text(e => e.PrimaryTitle)
|
||||||
var modelPrimaryTitle = new CreateIndexModel<ImdbEntry>(indexPrimaryTitle);
|
.Ascending(e => e.TitleType)
|
||||||
_imdbCollection.Indexes.CreateOne(modelPrimaryTitle);
|
.Ascending(e => e.StartYear);
|
||||||
|
|
||||||
|
var createIndexOptions = new CreateIndexOptions { Background = true };
|
||||||
|
var indexModel = new CreateIndexModel<ImdbEntry>(indexKeysDefinition, createIndexOptions);
|
||||||
|
|
||||||
|
_imdbCollection.Indexes.CreateOne(indexModel);
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,9 +1,7 @@
|
|||||||
namespace Metadata.Features.ImportImdbData;
|
namespace Metadata.Features.ImportImdbData;
|
||||||
|
|
||||||
public class ImportImdbDataRequestHandler(ILogger<ImportImdbDataRequestHandler> logger, ImdbMongoDbService mongoDbService)
|
public class ImportImdbDataRequestHandler(ILogger<ImportImdbDataRequestHandler> logger, ImdbMongoDbService mongoDbService, JobConfiguration configuration)
|
||||||
{
|
{
|
||||||
private const int BatchSize = 50_000;
|
|
||||||
|
|
||||||
public async Task<DeleteDownloadedImdbDataRequest> Handle(ImportImdbDataRequest request, CancellationToken cancellationToken)
|
public async Task<DeleteDownloadedImdbDataRequest> Handle(ImportImdbDataRequest request, CancellationToken cancellationToken)
|
||||||
{
|
{
|
||||||
logger.LogInformation("Importing Downloaded IMDB data from {FilePath}", request.FilePath);
|
logger.LogInformation("Importing Downloaded IMDB data from {FilePath}", request.FilePath);
|
||||||
@@ -18,7 +16,7 @@ public class ImportImdbDataRequestHandler(ILogger<ImportImdbDataRequestHandler>
|
|||||||
using var reader = new StreamReader(request.FilePath);
|
using var reader = new StreamReader(request.FilePath);
|
||||||
using var csv = new CsvReader(reader, config);
|
using var csv = new CsvReader(reader, config);
|
||||||
|
|
||||||
var channel = Channel.CreateBounded<ImdbEntry>(new BoundedChannelOptions(BatchSize)
|
var channel = Channel.CreateBounded<ImdbEntry>(new BoundedChannelOptions(configuration.InsertBatchSize)
|
||||||
{
|
{
|
||||||
FullMode = BoundedChannelFullMode.Wait,
|
FullMode = BoundedChannelFullMode.Wait,
|
||||||
});
|
});
|
||||||
@@ -53,7 +51,7 @@ public class ImportImdbDataRequestHandler(ILogger<ImportImdbDataRequestHandler>
|
|||||||
movieData,
|
movieData,
|
||||||
};
|
};
|
||||||
|
|
||||||
while (batch.Count < BatchSize && channel.Reader.TryRead(out var nextMovieData))
|
while (batch.Count < configuration.InsertBatchSize && channel.Reader.TryRead(out var nextMovieData))
|
||||||
{
|
{
|
||||||
batch.Add(nextMovieData);
|
batch.Add(nextMovieData);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,4 +1,13 @@
|
|||||||
|
import {BooleanHelpers} from "@helpers/boolean_helpers";
|
||||||
|
|
||||||
export const rabbitConfig = {
|
export const rabbitConfig = {
|
||||||
RABBIT_URI: process.env.RABBIT_URI || 'amqp://localhost',
|
HOST: process.env.RABBITMQ_HOST || 'rabbitmq',
|
||||||
QUEUE_NAME: process.env.QUEUE_NAME || 'test-queue'
|
USER: process.env.RABBITMQ_USER || 'guest',
|
||||||
|
PASSWORD: process.env.RABBITMQ_PASSWORD || 'guest',
|
||||||
|
QUEUE_NAME: process.env.RABBITMQ_QUEUE_NAME || 'ingested',
|
||||||
|
DURABLE: BooleanHelpers.parseBool(process.env.RABBITMQ_DURABLE, true),
|
||||||
|
|
||||||
|
get RABBIT_URI(): string {
|
||||||
|
return `amqp://${this.USER}:${this.PASSWORD}@${this.HOST}?heartbeat=30`;
|
||||||
|
}
|
||||||
};
|
};
|
||||||
@@ -13,4 +13,6 @@ const ImdbEntriesSchema: Schema = new Schema({
|
|||||||
TitleType: { type: String, default: "" },
|
TitleType: { type: String, default: "" },
|
||||||
});
|
});
|
||||||
|
|
||||||
|
ImdbEntriesSchema.index({ PrimaryTitle: 'text', TitleType: 1, StartYear: 1 }, { background: true });
|
||||||
|
|
||||||
export const ImdbEntryModel = mongoose.model<IImdbEntry>('ImdbEntry', ImdbEntriesSchema, 'imdb-entries');
|
export const ImdbEntryModel = mongoose.model<IImdbEntry>('ImdbEntry', ImdbEntriesSchema, 'imdb-entries');
|
||||||
@@ -1,17 +1,28 @@
|
|||||||
import {TorrentType} from "@enums/torrent_types";
|
import {TorrentType} from "@enums/torrent_types";
|
||||||
|
import {ILoggingService} from "@interfaces/logging_service";
|
||||||
import {IMongoMetadataQuery} from "@mongo/interfaces/mongo_metadata_query";
|
import {IMongoMetadataQuery} from "@mongo/interfaces/mongo_metadata_query";
|
||||||
import {IMongoRepository} from "@mongo/interfaces/mongo_repository";
|
import {IMongoRepository} from "@mongo/interfaces/mongo_repository";
|
||||||
import {ImdbEntryModel} from "@mongo/models/imdb_entries_model";
|
import {ImdbEntryModel} from "@mongo/models/imdb_entries_model";
|
||||||
import {configurationService} from '@services/configuration_service';
|
import {configurationService} from '@services/configuration_service';
|
||||||
import {injectable} from "inversify";
|
import {IocTypes} from "@setup/ioc_types";
|
||||||
|
import {inject, injectable} from "inversify";
|
||||||
import mongoose from 'mongoose';
|
import mongoose from 'mongoose';
|
||||||
|
|
||||||
@injectable()
|
@injectable()
|
||||||
export class MongoRepository implements IMongoRepository {
|
export class MongoRepository implements IMongoRepository {
|
||||||
|
@inject(IocTypes.ILoggingService) private logger: ILoggingService;
|
||||||
private db: typeof mongoose = mongoose;
|
private db: typeof mongoose = mongoose;
|
||||||
|
|
||||||
async connect() : Promise<void> {
|
async connect() : Promise<void> {
|
||||||
await this.db.connect(configurationService.cacheConfig.MONGO_URI, {directConnection: true});
|
try {
|
||||||
|
await this.db.connect(configurationService.cacheConfig.MONGO_URI, {directConnection: true});
|
||||||
|
this.logger.info('Successfully connected to mongo db');
|
||||||
|
}
|
||||||
|
catch (error) {
|
||||||
|
this.logger.debug('Failed to connect to mongo db', error);
|
||||||
|
this.logger.error('Failed to connect to mongo db');
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async getImdbId(title: string, category: string, year?: string | number) : Promise<string | null> {
|
async getImdbId(title: string, category: string, year?: string | number) : Promise<string | null> {
|
||||||
@@ -35,7 +46,12 @@ export class MongoRepository implements IMongoRepository {
|
|||||||
query.StartYear = year.toString();
|
query.StartYear = year.toString();
|
||||||
}
|
}
|
||||||
|
|
||||||
const result = await ImdbEntryModel.findOne(query);
|
try {
|
||||||
return result ? result._id : null;
|
const result = await ImdbEntryModel.findOne(query, '_id').maxTimeMS(30000);
|
||||||
|
return result ? result._id : null;
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error('Query exceeded the 30 seconds time limit', error);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -110,12 +110,19 @@ describe('Configuration Tests', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('should populate rabbitConfig correctly', async () => {
|
it('should populate rabbitConfig correctly', async () => {
|
||||||
process.env.RABBIT_URI = 'amqp://localhost';
|
process.env.RABBITMQ_HOST = 'rabbitmq';
|
||||||
process.env.QUEUE_NAME = 'test-queue';
|
process.env.RABBITMQ_USER = 'guest';
|
||||||
|
process.env.RABBITMQ_PASSWORD = 'guest';
|
||||||
|
process.env.RABBITMQ_QUEUE_NAME = 'ingested';
|
||||||
|
process.env.RABBITMQ_DURABLE = 'true';
|
||||||
const {configurationService} = await import("@services/configuration_service");
|
const {configurationService} = await import("@services/configuration_service");
|
||||||
const {rabbitConfig} = configurationService;
|
const {rabbitConfig} = configurationService;
|
||||||
expect(rabbitConfig.RABBIT_URI).toBe('amqp://localhost');
|
expect(rabbitConfig.HOST).toBe('rabbitmq');
|
||||||
expect(rabbitConfig.QUEUE_NAME).toBe('test-queue');
|
expect(rabbitConfig.USER).toBe('guest');
|
||||||
|
expect(rabbitConfig.PASSWORD).toBe('guest');
|
||||||
|
expect(rabbitConfig.QUEUE_NAME).toBe('ingested');
|
||||||
|
expect(rabbitConfig.DURABLE).toBe(true);
|
||||||
|
expect(rabbitConfig.RABBIT_URI).toBe('amqp://guest:guest@rabbitmq?heartbeat=30');
|
||||||
});
|
});
|
||||||
|
|
||||||
it('should populate torrentConfig correctly', async () => {
|
it('should populate torrentConfig correctly', async () => {
|
||||||
|
|||||||
@@ -1,7 +1,9 @@
|
|||||||
import "reflect-metadata"; // required
|
import "reflect-metadata"; // required
|
||||||
import {TorrentType} from "@enums/torrent_types";
|
import {TorrentType} from "@enums/torrent_types";
|
||||||
|
import {ILoggingService} from "@interfaces/logging_service";
|
||||||
import {MongoRepository} from "@mongo/mongo_repository";
|
import {MongoRepository} from "@mongo/mongo_repository";
|
||||||
import {Container} from "inversify";
|
import {IocTypes} from "@setup/ioc_types";
|
||||||
|
import {Container, inject} from "inversify";
|
||||||
|
|
||||||
jest.mock('@services/configuration_service', () => {
|
jest.mock('@services/configuration_service', () => {
|
||||||
return {
|
return {
|
||||||
@@ -20,13 +22,24 @@ jest.mock('@services/configuration_service', () => {
|
|||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
jest.mock('@services/logging_service', () => {
|
||||||
|
return {
|
||||||
|
error: jest.fn(),
|
||||||
|
info: jest.fn(),
|
||||||
|
debug: jest.fn(),
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
xdescribe('MongoRepository Tests - Manual Tests against real cluster. Skipped by default.', () => {
|
xdescribe('MongoRepository Tests - Manual Tests against real cluster. Skipped by default.', () => {
|
||||||
let mongoRepository: MongoRepository;
|
let mongoRepository: MongoRepository,
|
||||||
|
mockLogger: ILoggingService;
|
||||||
|
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
jest.clearAllMocks();
|
jest.clearAllMocks();
|
||||||
process.env.LOG_LEVEL = 'debug';
|
process.env.LOG_LEVEL = 'debug';
|
||||||
|
mockLogger = jest.requireMock<ILoggingService>('@services/logging_service');
|
||||||
const container = new Container();
|
const container = new Container();
|
||||||
|
container.bind<ILoggingService>(IocTypes.ILoggingService).toConstantValue(mockLogger);
|
||||||
container.bind<MongoRepository>(MongoRepository).toSelf();
|
container.bind<MongoRepository>(MongoRepository).toSelf();
|
||||||
mongoRepository = container.get(MongoRepository);
|
mongoRepository = container.get(MongoRepository);
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,5 +0,0 @@
|
|||||||
{
|
|
||||||
"GithubSettings": {
|
|
||||||
"PAT": ""
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,12 +0,0 @@
|
|||||||
{
|
|
||||||
"RabbitMqConfiguration": {
|
|
||||||
"Host": "localhost",
|
|
||||||
"Username": "guest",
|
|
||||||
"Password": "guest",
|
|
||||||
"QueueName": "test-queue",
|
|
||||||
"Durable": true,
|
|
||||||
"MaxQueueSize": 0,
|
|
||||||
"MaxPublishBatchSize": 1,
|
|
||||||
"PublishIntervalInSeconds": 10
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,5 +1,3 @@
|
|||||||
using Producer.Models.Configuration;
|
|
||||||
|
|
||||||
namespace Producer.Crawlers.Sites;
|
namespace Producer.Crawlers.Sites;
|
||||||
|
|
||||||
public partial class DebridMediaManagerCrawler(
|
public partial class DebridMediaManagerCrawler(
|
||||||
|
|||||||
@@ -1,5 +1,3 @@
|
|||||||
using Producer.Models.Configuration;
|
|
||||||
|
|
||||||
namespace Producer.Extensions;
|
namespace Producer.Extensions;
|
||||||
|
|
||||||
public static class ConfigurationExtensions
|
public static class ConfigurationExtensions
|
||||||
@@ -13,8 +11,6 @@ public static class ConfigurationExtensions
|
|||||||
|
|
||||||
configuration.AddJsonFile(LoggingConfig, false, true);
|
configuration.AddJsonFile(LoggingConfig, false, true);
|
||||||
configuration.AddJsonFile(ScrapeConfiguration.Filename, false, true);
|
configuration.AddJsonFile(ScrapeConfiguration.Filename, false, true);
|
||||||
configuration.AddJsonFile(RabbitMqConfiguration.Filename, false, true);
|
|
||||||
configuration.AddJsonFile(GithubConfiguration.Filename, false, true);
|
|
||||||
|
|
||||||
configuration.AddEnvironmentVariables();
|
configuration.AddEnvironmentVariables();
|
||||||
|
|
||||||
|
|||||||
68
src/producer/Extensions/EnvironmentExtensions.cs
Normal file
68
src/producer/Extensions/EnvironmentExtensions.cs
Normal file
@@ -0,0 +1,68 @@
|
|||||||
|
namespace Producer.Extensions;
|
||||||
|
|
||||||
|
public static class EnvironmentExtensions
|
||||||
|
{
|
||||||
|
public static bool GetEnvironmentVariableAsBool(this string prefix, string varName, bool fallback = false)
|
||||||
|
{
|
||||||
|
var fullVarName = GetFullVariableName(prefix, varName);
|
||||||
|
|
||||||
|
var str = Environment.GetEnvironmentVariable(fullVarName);
|
||||||
|
|
||||||
|
if (string.IsNullOrEmpty(str))
|
||||||
|
{
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
return str.Trim().ToLower() switch
|
||||||
|
{
|
||||||
|
"true" => true,
|
||||||
|
"yes" => true,
|
||||||
|
"1" => true,
|
||||||
|
_ => false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
public static int GetEnvironmentVariableAsInt(this string prefix, string varName, int fallback = 0)
|
||||||
|
{
|
||||||
|
var fullVarName = GetFullVariableName(prefix, varName);
|
||||||
|
|
||||||
|
var str = Environment.GetEnvironmentVariable(fullVarName);
|
||||||
|
|
||||||
|
if (string.IsNullOrEmpty(str))
|
||||||
|
{
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
return int.TryParse(str, out var result) ? result : fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static string GetRequiredEnvironmentVariableAsString(this string prefix, string varName)
|
||||||
|
{
|
||||||
|
var fullVarName = GetFullVariableName(prefix, varName);
|
||||||
|
|
||||||
|
var str = Environment.GetEnvironmentVariable(fullVarName);
|
||||||
|
|
||||||
|
if (string.IsNullOrEmpty(str))
|
||||||
|
{
|
||||||
|
throw new InvalidOperationException($"Environment variable {fullVarName} is not set");
|
||||||
|
}
|
||||||
|
|
||||||
|
return str;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static string GetOptionalEnvironmentVariableAsString(this string prefix, string varName, string? fallback = null)
|
||||||
|
{
|
||||||
|
var fullVarName = GetFullVariableName(prefix, varName);
|
||||||
|
|
||||||
|
var str = Environment.GetEnvironmentVariable(fullVarName);
|
||||||
|
|
||||||
|
if (string.IsNullOrEmpty(str))
|
||||||
|
{
|
||||||
|
return fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
return str;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string GetFullVariableName(string prefix, string varName) => $"{prefix}_{varName}";
|
||||||
|
}
|
||||||
@@ -1,5 +1,3 @@
|
|||||||
using Producer.Models.Configuration;
|
|
||||||
|
|
||||||
namespace Producer.Extensions;
|
namespace Producer.Extensions;
|
||||||
|
|
||||||
public static class ServiceCollectionExtensions
|
public static class ServiceCollectionExtensions
|
||||||
@@ -29,13 +27,9 @@ public static class ServiceCollectionExtensions
|
|||||||
return services;
|
return services;
|
||||||
}
|
}
|
||||||
|
|
||||||
internal static IServiceCollection RegisterMassTransit(this IServiceCollection services, IConfiguration configuration)
|
internal static IServiceCollection RegisterMassTransit(this IServiceCollection services)
|
||||||
{
|
{
|
||||||
var rabbitConfig = configuration.GetSection(RabbitMqConfiguration.SectionName).Get<RabbitMqConfiguration>();
|
var rabbitConfig = services.LoadConfigurationFromEnv<RabbitMqConfiguration>();
|
||||||
|
|
||||||
ArgumentNullException.ThrowIfNull(rabbitConfig, nameof(rabbitConfig));
|
|
||||||
|
|
||||||
services.AddSingleton(rabbitConfig);
|
|
||||||
|
|
||||||
services.AddMassTransit(busConfigurator =>
|
services.AddMassTransit(busConfigurator =>
|
||||||
{
|
{
|
||||||
@@ -56,8 +50,8 @@ public static class ServiceCollectionExtensions
|
|||||||
internal static IServiceCollection AddQuartz(this IServiceCollection services, IConfiguration configuration)
|
internal static IServiceCollection AddQuartz(this IServiceCollection services, IConfiguration configuration)
|
||||||
{
|
{
|
||||||
var scrapeConfiguration = services.LoadConfigurationFromConfig<ScrapeConfiguration>(configuration, ScrapeConfiguration.SectionName);
|
var scrapeConfiguration = services.LoadConfigurationFromConfig<ScrapeConfiguration>(configuration, ScrapeConfiguration.SectionName);
|
||||||
var githubConfiguration = services.LoadConfigurationFromConfig<GithubConfiguration>(configuration, GithubConfiguration.SectionName);
|
var githubConfiguration = services.LoadConfigurationFromEnv<GithubConfiguration>();
|
||||||
var rabbitConfig = services.LoadConfigurationFromConfig<RabbitMqConfiguration>(configuration, RabbitMqConfiguration.SectionName);
|
var rabbitConfig = services.LoadConfigurationFromEnv<RabbitMqConfiguration>();
|
||||||
|
|
||||||
services
|
services
|
||||||
.AddTransient<SyncEzTvJob>()
|
.AddTransient<SyncEzTvJob>()
|
||||||
|
|||||||
@@ -16,8 +16,10 @@ global using Npgsql;
|
|||||||
global using Quartz;
|
global using Quartz;
|
||||||
global using Producer.Crawlers;
|
global using Producer.Crawlers;
|
||||||
global using Producer.Crawlers.Sites;
|
global using Producer.Crawlers.Sites;
|
||||||
|
global using Producer.Extensions;
|
||||||
global using Producer.Interfaces;
|
global using Producer.Interfaces;
|
||||||
global using Producer.Jobs;
|
global using Producer.Jobs;
|
||||||
global using Producer.Models;
|
global using Producer.Models;
|
||||||
|
global using Producer.Models.Configuration;
|
||||||
global using Producer.Services;
|
global using Producer.Services;
|
||||||
global using Serilog;
|
global using Serilog;
|
||||||
@@ -2,8 +2,8 @@
|
|||||||
|
|
||||||
public class GithubConfiguration
|
public class GithubConfiguration
|
||||||
{
|
{
|
||||||
public const string SectionName = "GithubSettings";
|
private const string Prefix = "GITHUB";
|
||||||
public const string Filename = "github.json";
|
private const string PatVariable = "PAT";
|
||||||
|
|
||||||
public string? PAT { get; set; }
|
public string? PAT { get; init; } = Prefix.GetOptionalEnvironmentVariableAsString(PatVariable);
|
||||||
}
|
}
|
||||||
@@ -9,21 +9,11 @@ public class PostgresConfiguration
|
|||||||
private const string DatabaseVariable = "DB";
|
private const string DatabaseVariable = "DB";
|
||||||
private const string PortVariable = "PORT";
|
private const string PortVariable = "PORT";
|
||||||
|
|
||||||
private string Host { get; init; } = Environment.GetEnvironmentVariable($"{Prefix}_{HostVariable}") ??
|
private string Host { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(HostVariable);
|
||||||
throw new InvalidOperationException($"Environment variable {Prefix}_{HostVariable} is not set");
|
private string Username { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(UsernameVariable);
|
||||||
|
private string Password { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(PasswordVariable);
|
||||||
private string Username { get; init; } = Environment.GetEnvironmentVariable($"{Prefix}_{UsernameVariable}") ??
|
private string Database { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(DatabaseVariable);
|
||||||
throw new InvalidOperationException($"Environment variable {Prefix}_{UsernameVariable} is not set");
|
private int PORT { get; init; } = Prefix.GetEnvironmentVariableAsInt(PortVariable, 5432);
|
||||||
|
|
||||||
private string Password { get; init; } = Environment.GetEnvironmentVariable($"{Prefix}_{PasswordVariable}") ??
|
|
||||||
throw new InvalidOperationException($"Environment variable {Prefix}_{PasswordVariable} is not set");
|
|
||||||
|
|
||||||
private string Database { get; init; } = Environment.GetEnvironmentVariable($"{Prefix}_{DatabaseVariable}") ??
|
|
||||||
throw new InvalidOperationException($"Environment variable {Prefix}_{DatabaseVariable} is not set");
|
|
||||||
|
|
||||||
private int PORT { get; init; } = int.Parse(
|
|
||||||
Environment.GetEnvironmentVariable($"{Prefix}_{PortVariable}") ??
|
|
||||||
throw new InvalidOperationException($"Environment variable {Prefix}_{PortVariable} is not set"));
|
|
||||||
|
|
||||||
public string StorageConnectionString => $"Host={Host};Port={PORT};Username={Username};Password={Password};Database={Database};";
|
public string StorageConnectionString => $"Host={Host};Port={PORT};Username={Username};Password={Password};Database={Database};";
|
||||||
}
|
}
|
||||||
@@ -1,18 +1,25 @@
|
|||||||
namespace Producer.Models.Configuration;
|
namespace Producer.Models.Configuration;
|
||||||
|
|
||||||
public class RabbitMqConfiguration
|
public class RabbitMqConfiguration
|
||||||
{
|
{
|
||||||
public const string SectionName = "RabbitMqConfiguration";
|
private const string Prefix = "RABBITMQ";
|
||||||
public const string Filename = "rabbitmq.json";
|
private const string HostVariable = "HOST";
|
||||||
|
private const string UsernameVariable = "USER";
|
||||||
|
private const string PasswordVariable = "PASSWORD";
|
||||||
|
private const string QueueNameVariable = "QUEUE_NAME";
|
||||||
|
private const string DurableVariable = "DURABLE";
|
||||||
|
private const string MaxQueueSizeVariable = "MAX_QUEUE_SIZE";
|
||||||
|
private const string MaxPublishBatchSizeVariable = "MAX_PUBLISH_BATCH_SIZE";
|
||||||
|
private const string PublishIntervalInSecondsVariable = "PUBLISH_INTERVAL_IN_SECONDS";
|
||||||
|
|
||||||
public string? Host { get; set; }
|
public string? Username { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(UsernameVariable);
|
||||||
public string? Username { get; set; }
|
public string? Host { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(HostVariable);
|
||||||
public string? Password { get; set; }
|
public string? Password { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(PasswordVariable);
|
||||||
public string? QueueName { get; set; }
|
public string? QueueName { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(QueueNameVariable);
|
||||||
public bool Durable { get; set; }
|
public bool Durable { get; init; } = Prefix.GetEnvironmentVariableAsBool(DurableVariable, true);
|
||||||
public int MaxQueueSize { get; set; }
|
public int MaxQueueSize { get; init; } = Prefix.GetEnvironmentVariableAsInt(MaxQueueSizeVariable, 0);
|
||||||
public int MaxPublishBatchSize { get; set; } = 500;
|
public int MaxPublishBatchSize { get; set; } = Prefix.GetEnvironmentVariableAsInt(MaxPublishBatchSizeVariable, 500);
|
||||||
public int PublishIntervalInSeconds { get; set; } = 1000 * 10;
|
public int PublishIntervalInSeconds { get; set; } = Prefix.GetEnvironmentVariableAsInt(PublishIntervalInSecondsVariable, 1000 * 10);
|
||||||
|
|
||||||
public void Validate()
|
public void Validate()
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -32,15 +32,6 @@
|
|||||||
<None Include="Configuration\logging.json">
|
<None Include="Configuration\logging.json">
|
||||||
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
|
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
|
||||||
</None>
|
</None>
|
||||||
<Content Remove="Configuration\rabbitmq.json" />
|
|
||||||
<None Include="Configuration\rabbitmq.json">
|
|
||||||
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
|
|
||||||
</None>
|
|
||||||
<Content Remove="Configuration\github.json" />
|
|
||||||
<None Include="Configuration\github.json">
|
|
||||||
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
|
|
||||||
</None>
|
|
||||||
<Content Remove="Configuration\postgres.json" />
|
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
</Project>
|
</Project>
|
||||||
|
|||||||
@@ -1,6 +1,4 @@
|
|||||||
using Producer.Extensions;
|
var builder = WebApplication.CreateBuilder(args);
|
||||||
|
|
||||||
var builder = WebApplication.CreateBuilder(args);
|
|
||||||
|
|
||||||
builder.Configuration
|
builder.Configuration
|
||||||
.AddScrapeConfiguration();
|
.AddScrapeConfiguration();
|
||||||
@@ -9,7 +7,7 @@ builder.Host
|
|||||||
.SetupSerilog(builder.Configuration);
|
.SetupSerilog(builder.Configuration);
|
||||||
|
|
||||||
builder.Services
|
builder.Services
|
||||||
.RegisterMassTransit(builder.Configuration)
|
.RegisterMassTransit()
|
||||||
.AddDataStorage()
|
.AddDataStorage()
|
||||||
.AddCrawlers()
|
.AddCrawlers()
|
||||||
.AddQuartz(builder.Configuration);
|
.AddQuartz(builder.Configuration);
|
||||||
|
|||||||
@@ -1,5 +1,3 @@
|
|||||||
using Producer.Models.Configuration;
|
|
||||||
|
|
||||||
namespace Producer.Services;
|
namespace Producer.Services;
|
||||||
|
|
||||||
public class DapperDataStorage(PostgresConfiguration configuration, RabbitMqConfiguration rabbitConfig, ILogger<DapperDataStorage> logger) : IDataStorage
|
public class DapperDataStorage(PostgresConfiguration configuration, RabbitMqConfiguration rabbitConfig, ILogger<DapperDataStorage> logger) : IDataStorage
|
||||||
|
|||||||
@@ -1,6 +1,4 @@
|
|||||||
using Producer.Models.Configuration;
|
namespace Producer.Services;
|
||||||
|
|
||||||
namespace Producer.Services;
|
|
||||||
|
|
||||||
public class TorrentPublisher(
|
public class TorrentPublisher(
|
||||||
ISendEndpointProvider sendEndpointProvider,
|
ISendEndpointProvider sendEndpointProvider,
|
||||||
|
|||||||
Reference in New Issue
Block a user