mirror of
https://github.com/knightcrawler-stremio/knightcrawler.git
synced 2024-12-20 03:29:51 +00:00
Run pre-commit
This commit is contained in:
@@ -12,14 +12,14 @@ public abstract class BaseCrawler(ILogger<BaseCrawler> logger, IDataStorage stor
|
||||
protected async Task<InsertTorrentResult> InsertTorrents(IReadOnlyCollection<Torrent> torrent)
|
||||
{
|
||||
var result = await storage.InsertTorrents(torrent);
|
||||
|
||||
|
||||
if (!result.Success)
|
||||
{
|
||||
logger.LogWarning("Ingestion Failed: [{Error}]", result.ErrorMessage);
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
logger.LogInformation("Ingestion Successful - Wrote {Count} new torrents", result.InsertedCount);
|
||||
return result;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ namespace Producer.Features.CrawlerSupport;
|
||||
public abstract class BaseJsonCrawler(IHttpClientFactory httpClientFactory, ILogger<BaseJsonCrawler> logger, IDataStorage storage) : BaseCrawler(logger, storage)
|
||||
{
|
||||
private readonly HttpClient _client = httpClientFactory.CreateClient(Literals.CrawlerClient);
|
||||
|
||||
|
||||
protected virtual async Task Execute(string collectionName)
|
||||
{
|
||||
logger.LogInformation("Starting {Source} crawl", Source);
|
||||
@@ -16,13 +16,13 @@ public abstract class BaseJsonCrawler(IHttpClientFactory httpClientFactory, ILog
|
||||
.Select(ParseTorrent)
|
||||
.Where(x => x is not null)
|
||||
.ToList();
|
||||
|
||||
|
||||
if (torrents.Count == 0)
|
||||
{
|
||||
logger.LogWarning("No torrents found in {Source} response", Source);
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
await InsertTorrents(torrents!);
|
||||
}
|
||||
|
||||
@@ -40,6 +40,6 @@ public abstract class BaseJsonCrawler(IHttpClientFactory httpClientFactory, ILog
|
||||
torrent.InfoHash = infoHash;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
protected abstract Torrent? ParseTorrent(JsonElement item);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,22 +5,22 @@ public abstract class BaseXmlCrawler(IHttpClientFactory httpClientFactory, ILogg
|
||||
public override async Task Execute()
|
||||
{
|
||||
logger.LogInformation("Starting {Source} crawl", Source);
|
||||
|
||||
|
||||
using var client = httpClientFactory.CreateClient(Literals.CrawlerClient);
|
||||
var xml = await client.GetStringAsync(Url);
|
||||
var xmlRoot = XElement.Parse(xml);
|
||||
|
||||
|
||||
var torrents = xmlRoot.Descendants("item")
|
||||
.Select(ParseTorrent)
|
||||
.Where(x => x is not null)
|
||||
.ToList();
|
||||
|
||||
|
||||
if (torrents.Count == 0)
|
||||
{
|
||||
logger.LogWarning("No torrents found in {Source} response", Source);
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
await InsertTorrents(torrents!);
|
||||
}
|
||||
|
||||
@@ -40,4 +40,4 @@ public abstract class BaseXmlCrawler(IHttpClientFactory httpClientFactory, ILogg
|
||||
}
|
||||
|
||||
protected abstract Torrent? ParseTorrent(XElement itemNode);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,8 +4,8 @@ public class CrawlerProvider(IServiceProvider serviceProvider) : ICrawlerProvide
|
||||
{
|
||||
public IEnumerable<ICrawler> GetAll() =>
|
||||
serviceProvider.GetServices<ICrawler>();
|
||||
|
||||
public ICrawler Get(string name) =>
|
||||
|
||||
public ICrawler Get(string name) =>
|
||||
serviceProvider.GetRequiredKeyedService<ICrawler>(name);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,4 +3,4 @@ namespace Producer.Features.CrawlerSupport;
|
||||
public interface ICrawler
|
||||
{
|
||||
Task Execute();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,6 @@ namespace Producer.Features.CrawlerSupport;
|
||||
public interface ICrawlerProvider
|
||||
{
|
||||
IEnumerable<ICrawler> GetAll();
|
||||
|
||||
|
||||
ICrawler Get(string name);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,4 +3,4 @@ namespace Producer.Features.CrawlerSupport;
|
||||
public static class Literals
|
||||
{
|
||||
public const string CrawlerClient = "Scraper";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,4 +2,4 @@ namespace Producer.Features.CrawlerSupport;
|
||||
|
||||
public record InsertTorrentResult(bool Success, int InsertedCount = 0, string? ErrorMessage = null);
|
||||
public record UpdatedTorrentResult(bool Success, int UpdatedCount = 0, string? ErrorMessage = null);
|
||||
public record PageIngestedResult(bool Success, string? ErrorMessage = null);
|
||||
public record PageIngestedResult(bool Success, string? ErrorMessage = null);
|
||||
|
||||
@@ -7,4 +7,4 @@ public class Scraper
|
||||
public int IntervalSeconds { get; set; } = 60;
|
||||
|
||||
public bool Enabled { get; set; } = true;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,11 +5,11 @@ internal static class ServiceCollectionExtensions
|
||||
internal static IServiceCollection AddCrawlers(this IServiceCollection services)
|
||||
{
|
||||
services.AddHttpClient(Literals.CrawlerClient);
|
||||
|
||||
|
||||
var crawlerTypes = Assembly.GetAssembly(typeof(ICrawler))
|
||||
.GetTypes()
|
||||
.Where(t => t is {IsClass: true, IsAbstract: false} && typeof(ICrawler).IsAssignableFrom(t));
|
||||
|
||||
|
||||
foreach (var type in crawlerTypes)
|
||||
{
|
||||
services.AddKeyedTransient(typeof(ICrawler), type.Name, type);
|
||||
@@ -20,4 +20,4 @@ internal static class ServiceCollectionExtensions
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,4 +17,4 @@ public class Torrent
|
||||
public bool Processed { get; set; } = false;
|
||||
public DateTime CreatedAt { get; set; } = DateTime.UtcNow;
|
||||
public DateTime UpdatedAt { get; set; } = DateTime.UtcNow;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user