tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #98 from Gabisonfire/feat/nyaasi

Browse Source 
Adds Nyaa Crawler
This commit is contained in:
iPromKnight
2024-02-27 17:39:48 +00:00
committed by GitHub
parent 4cbbb314b2 6c4282b6de
commit aad59c31e4
4 changed files with 52 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/producer/Configuration/scrapers.json
View File

@@ -6,6 +6,11 @@
"IntervalSeconds": 60, "IntervalSeconds": 60,
"Enabled": true "Enabled": true
}, },
{
"Name": "SyncNyaaJob",
"IntervalSeconds": 60,
"Enabled": true
},
{ {
"Name": "SyncTpbJob", "Name": "SyncTpbJob",
"IntervalSeconds": 60, "IntervalSeconds": 60,

32
src/producer/Crawlers/Sites/NyaaCrawler.cs Normal file
View File

@@ -0,0 +1,32 @@
namespace Producer.Crawlers.Sites;
public class NyaaCrawler(IHttpClientFactory httpClientFactory, ILogger<NyaaCrawler> logger, IDataStorage storage) : BaseXmlCrawler(httpClientFactory, logger, storage)
{
protected override string Url => "https://nyaa.si/?page=rss&c=1_2&f=0";
protected override string Source => "Nyaa";
private static readonly XNamespace XmlNamespace = "https://nyaa.si/xmlns/nyaa";
protected override IReadOnlyDictionary<string, string> Mappings =>
new Dictionary<string, string>
{
[nameof(Torrent.Name)] = "title",
[nameof(Torrent.Size)] = "size",
[nameof(Torrent.Seeders)] = "seeders",
[nameof(Torrent.Leechers)] = "leechers",
[nameof(Torrent.InfoHash)] = "infoHash",
[nameof(Torrent.Category)] = "category",
};
protected override Torrent ParseTorrent(XElement itemNode) =>
new()
{
Source = Source,
Name = itemNode.Element(Mappings[nameof(Torrent.Name)])?.Value,
Size = itemNode.Element(XmlNamespace + Mappings[nameof(Torrent.Size)])?.Value,
Seeders = int.Parse(itemNode.Element(XmlNamespace + Mappings[nameof(Torrent.Seeders)])?.Value ?? "0"),
Leechers = int.Parse(itemNode.Element(XmlNamespace + Mappings[nameof(Torrent.Leechers)])?.Value ?? "0"),
InfoHash = itemNode.Element(XmlNamespace + Mappings[nameof(Torrent.InfoHash)])?.Value,
Category = itemNode.Element(Mappings[nameof(Torrent.Category)])?.Value.ToLowerInvariant(),
};
}

3
src/producer/Extensions/ServiceCollectionExtensions.cs
View File

@@ -10,6 +10,7 @@ public static class ServiceCollectionExtensions
services services
.AddKeyedTransient<ICrawler, EzTvCrawler>(nameof(EzTvCrawler)) .AddKeyedTransient<ICrawler, EzTvCrawler>(nameof(EzTvCrawler))
.AddKeyedTransient<ICrawler, NyaaCrawler>(nameof(NyaaCrawler))
.AddKeyedTransient<ICrawler, YtsCrawler>(nameof(YtsCrawler)) .AddKeyedTransient<ICrawler, YtsCrawler>(nameof(YtsCrawler))
.AddKeyedTransient<ICrawler, TpbCrawler>(nameof(TpbCrawler)) .AddKeyedTransient<ICrawler, TpbCrawler>(nameof(TpbCrawler))
.AddKeyedTransient<ICrawler, TgxCrawler>(nameof(TgxCrawler)) .AddKeyedTransient<ICrawler, TgxCrawler>(nameof(TgxCrawler))
@@ -60,6 +61,7 @@ public static class ServiceCollectionExtensions
services services
.AddTransient<SyncEzTvJob>() .AddTransient<SyncEzTvJob>()
.AddTransient<SyncNyaaJob>()
.AddTransient<SyncTpbJob>() .AddTransient<SyncTpbJob>()
.AddTransient<SyncYtsJob>() .AddTransient<SyncYtsJob>()
.AddTransient<SyncTgxJob>() .AddTransient<SyncTgxJob>()
@@ -75,6 +77,7 @@ public static class ServiceCollectionExtensions
quartz => quartz =>
{ {
AddJobWithTrigger<SyncEzTvJob>(quartz, SyncEzTvJob.Key, SyncEzTvJob.Trigger, scrapeConfiguration); AddJobWithTrigger<SyncEzTvJob>(quartz, SyncEzTvJob.Key, SyncEzTvJob.Trigger, scrapeConfiguration);
AddJobWithTrigger<SyncNyaaJob>(quartz, SyncNyaaJob.Key, SyncNyaaJob.Trigger, scrapeConfiguration);
AddJobWithTrigger<SyncTpbJob>(quartz, SyncTpbJob.Key, SyncTpbJob.Trigger, scrapeConfiguration); AddJobWithTrigger<SyncTpbJob>(quartz, SyncTpbJob.Key, SyncTpbJob.Trigger, scrapeConfiguration);
AddJobWithTrigger<SyncYtsJob>(quartz, SyncYtsJob.Key, SyncYtsJob.Trigger, scrapeConfiguration); AddJobWithTrigger<SyncYtsJob>(quartz, SyncYtsJob.Key, SyncYtsJob.Trigger, scrapeConfiguration);
AddJobWithTrigger<SyncTgxJob>(quartz, SyncTgxJob.Key, SyncTgxJob.Trigger, scrapeConfiguration); AddJobWithTrigger<SyncTgxJob>(quartz, SyncTgxJob.Key, SyncTgxJob.Trigger, scrapeConfiguration);

12
src/producer/Jobs/SyncNyaaJob.cs Normal file
View File

@@ -0,0 +1,12 @@
using Producer.Crawlers.Sites;
namespace Producer.Jobs;
[DisallowConcurrentExecution]
public class SyncNyaaJob(ICrawlerProvider crawlerProvider) : BaseJob(crawlerProvider)
{
private const string JobName = nameof(NyaaCrawler);
public static readonly JobKey Key = new(JobName, nameof(Crawlers));
public static readonly TriggerKey Trigger = new($"{JobName}-trigger", nameof(Crawlers));
protected override string Crawler => nameof(NyaaCrawler);
}