Run pre-commit

This commit is contained in:
purple_emily
2024-03-08 14:34:53 +00:00
parent 31e16df720
commit 79409915cf
221 changed files with 525 additions and 526 deletions

View File

@@ -12,14 +12,14 @@ public abstract class BaseCrawler(ILogger<BaseCrawler> logger, IDataStorage stor
protected async Task<InsertTorrentResult> InsertTorrents(IReadOnlyCollection<Torrent> torrent)
{
var result = await storage.InsertTorrents(torrent);
if (!result.Success)
{
logger.LogWarning("Ingestion Failed: [{Error}]", result.ErrorMessage);
return result;
}
logger.LogInformation("Ingestion Successful - Wrote {Count} new torrents", result.InsertedCount);
return result;
}
}
}

View File

@@ -3,7 +3,7 @@ namespace Producer.Features.CrawlerSupport;
public abstract class BaseJsonCrawler(IHttpClientFactory httpClientFactory, ILogger<BaseJsonCrawler> logger, IDataStorage storage) : BaseCrawler(logger, storage)
{
private readonly HttpClient _client = httpClientFactory.CreateClient(Literals.CrawlerClient);
protected virtual async Task Execute(string collectionName)
{
logger.LogInformation("Starting {Source} crawl", Source);
@@ -16,13 +16,13 @@ public abstract class BaseJsonCrawler(IHttpClientFactory httpClientFactory, ILog
.Select(ParseTorrent)
.Where(x => x is not null)
.ToList();
if (torrents.Count == 0)
{
logger.LogWarning("No torrents found in {Source} response", Source);
return;
}
await InsertTorrents(torrents!);
}
@@ -40,6 +40,6 @@ public abstract class BaseJsonCrawler(IHttpClientFactory httpClientFactory, ILog
torrent.InfoHash = infoHash;
}
}
protected abstract Torrent? ParseTorrent(JsonElement item);
}
}

View File

@@ -5,22 +5,22 @@ public abstract class BaseXmlCrawler(IHttpClientFactory httpClientFactory, ILogg
public override async Task Execute()
{
logger.LogInformation("Starting {Source} crawl", Source);
using var client = httpClientFactory.CreateClient(Literals.CrawlerClient);
var xml = await client.GetStringAsync(Url);
var xmlRoot = XElement.Parse(xml);
var torrents = xmlRoot.Descendants("item")
.Select(ParseTorrent)
.Where(x => x is not null)
.ToList();
if (torrents.Count == 0)
{
logger.LogWarning("No torrents found in {Source} response", Source);
return;
}
await InsertTorrents(torrents!);
}
@@ -40,4 +40,4 @@ public abstract class BaseXmlCrawler(IHttpClientFactory httpClientFactory, ILogg
}
protected abstract Torrent? ParseTorrent(XElement itemNode);
}
}

View File

@@ -4,8 +4,8 @@ public class CrawlerProvider(IServiceProvider serviceProvider) : ICrawlerProvide
{
public IEnumerable<ICrawler> GetAll() =>
serviceProvider.GetServices<ICrawler>();
public ICrawler Get(string name) =>
public ICrawler Get(string name) =>
serviceProvider.GetRequiredKeyedService<ICrawler>(name);
}
}

View File

@@ -3,4 +3,4 @@ namespace Producer.Features.CrawlerSupport;
public interface ICrawler
{
Task Execute();
}
}

View File

@@ -3,6 +3,6 @@ namespace Producer.Features.CrawlerSupport;
public interface ICrawlerProvider
{
IEnumerable<ICrawler> GetAll();
ICrawler Get(string name);
}
}

View File

@@ -3,4 +3,4 @@ namespace Producer.Features.CrawlerSupport;
public static class Literals
{
public const string CrawlerClient = "Scraper";
}
}

View File

@@ -2,4 +2,4 @@ namespace Producer.Features.CrawlerSupport;
public record InsertTorrentResult(bool Success, int InsertedCount = 0, string? ErrorMessage = null);
public record UpdatedTorrentResult(bool Success, int UpdatedCount = 0, string? ErrorMessage = null);
public record PageIngestedResult(bool Success, string? ErrorMessage = null);
public record PageIngestedResult(bool Success, string? ErrorMessage = null);

View File

@@ -7,4 +7,4 @@ public class Scraper
public int IntervalSeconds { get; set; } = 60;
public bool Enabled { get; set; } = true;
}
}

View File

@@ -5,11 +5,11 @@ internal static class ServiceCollectionExtensions
internal static IServiceCollection AddCrawlers(this IServiceCollection services)
{
services.AddHttpClient(Literals.CrawlerClient);
var crawlerTypes = Assembly.GetAssembly(typeof(ICrawler))
.GetTypes()
.Where(t => t is {IsClass: true, IsAbstract: false} && typeof(ICrawler).IsAssignableFrom(t));
foreach (var type in crawlerTypes)
{
services.AddKeyedTransient(typeof(ICrawler), type.Name, type);
@@ -20,4 +20,4 @@ internal static class ServiceCollectionExtensions
return services;
}
}
}

View File

@@ -17,4 +17,4 @@ public class Torrent
public bool Processed { get; set; } = false;
public DateTime CreatedAt { get; set; } = DateTime.UtcNow;
public DateTime UpdatedAt { get; set; } = DateTime.UtcNow;
}
}