mirror of
https://github.com/knightcrawler-stremio/knightcrawler.git
synced 2024-12-20 03:29:51 +00:00
Big rewrite - distributed consumers for ingestion / scraping(scalable) - single producer written in c#.
Changed from page scraping to rss xml scraping Includes RealDebridManager hashlist decoding (requires a github readonly PAT as requests must be authenticated) - This allows ingestion of 200k+ entries in a few hours. Simplifies a lot of torrentio to deal with new data
This commit is contained in:
6
src/producer/Interfaces/ICrawler.cs
Normal file
6
src/producer/Interfaces/ICrawler.cs
Normal file
@@ -0,0 +1,6 @@
|
||||
namespace Scraper.Interfaces;
|
||||
|
||||
public interface ICrawler
|
||||
{
|
||||
Task Execute();
|
||||
}
|
||||
8
src/producer/Interfaces/ICrawlerProvider.cs
Normal file
8
src/producer/Interfaces/ICrawlerProvider.cs
Normal file
@@ -0,0 +1,8 @@
|
||||
namespace Scraper.Interfaces;
|
||||
|
||||
public interface ICrawlerProvider
|
||||
{
|
||||
IEnumerable<ICrawler> GetAll();
|
||||
|
||||
ICrawler Get(string name);
|
||||
}
|
||||
10
src/producer/Interfaces/IDataStorage.cs
Normal file
10
src/producer/Interfaces/IDataStorage.cs
Normal file
@@ -0,0 +1,10 @@
|
||||
namespace Scraper.Interfaces;
|
||||
|
||||
public interface IDataStorage
|
||||
{
|
||||
Task<InsertTorrentResult> InsertTorrents(IReadOnlyCollection<Torrent> torrents, CancellationToken cancellationToken = default);
|
||||
Task<IReadOnlyCollection<Torrent>> GetPublishableTorrents(CancellationToken cancellationToken = default);
|
||||
Task<UpdatedTorrentResult> SetTorrentsProcessed(IReadOnlyCollection<Torrent> torrents, CancellationToken cancellationToken = default);
|
||||
Task<bool> PageIngested(string pageId, CancellationToken cancellationToken = default);
|
||||
Task<PageIngestedResult> MarkPageAsIngested(string pageId, CancellationToken cancellationToken = default);
|
||||
}
|
||||
6
src/producer/Interfaces/IIpService.cs
Normal file
6
src/producer/Interfaces/IIpService.cs
Normal file
@@ -0,0 +1,6 @@
|
||||
namespace Scraper.Interfaces;
|
||||
|
||||
public interface IIpService
|
||||
{
|
||||
Task GetPublicIpAddress();
|
||||
}
|
||||
6
src/producer/Interfaces/IMessagePublisher.cs
Normal file
6
src/producer/Interfaces/IMessagePublisher.cs
Normal file
@@ -0,0 +1,6 @@
|
||||
namespace Scraper.Interfaces;
|
||||
|
||||
public interface IMessagePublisher
|
||||
{
|
||||
Task PublishAsync(IEnumerable<Torrent> torrents, CancellationToken cancellationToken = default);
|
||||
}
|
||||
Reference in New Issue
Block a user