tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #96 from iPromKnight/dmm-gittrees

Browse Source 
Fix DMM so that all pages are enumerated
This commit is contained in:
Gabisonfire
2024-02-27 09:13:40 -05:00
committed by GitHub
parent 92dc4c5d66 49a6283f26
commit 4cbbb314b2
1 changed files with 10 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
src/producer/Crawlers/Sites/DebridMediaManagerCrawler.cs
View File

@@ -13,9 +13,11 @@ public partial class DebridMediaManagerCrawler(
[GeneratedRegex(@"[sS]([0-9]{1,2})|seasons?[\s-]?([0-9]{1,2})", RegexOptions.IgnoreCase, "en-GB")] [GeneratedRegex(@"[sS]([0-9]{1,2})|seasons?[\s-]?([0-9]{1,2})", RegexOptions.IgnoreCase, "en-GB")]
private static partial Regex SeasonMatcher(); private static partial Regex SeasonMatcher();
private const string DownloadBaseUrl = "https://raw.githubusercontent.com/debridmediamanager/hashlists/main";
protected override IReadOnlyDictionary<string, string> Mappings => new Dictionary<string, string>(); protected override IReadOnlyDictionary<string, string> Mappings => new Dictionary<string, string>();
protected override string Url => "https://api.github.com/repos/debridmediamanager/hashlists/contents"; protected override string Url => "https://api.github.com/repos/debridmediamanager/hashlists/git/trees/main?recursive=1";
protected override string Source => "DMM"; protected override string Source => "DMM";
public override async Task Execute() public override async Task Execute()
@@ -28,7 +30,11 @@ public partial class DebridMediaManagerCrawler(
var json = JsonDocument.Parse(jsonBody); var json = JsonDocument.Parse(jsonBody);
foreach (var entry in json.RootElement.EnumerateArray()) var entriesArray = json.RootElement.GetProperty("tree");
logger.LogInformation("Found {Entries} total DMM pages", entriesArray.GetArrayLength());
foreach (var entry in entriesArray.EnumerateArray())
{ {
await ParsePage(entry, client); await ParsePage(entry, client);
} }
@@ -43,9 +49,7 @@ public partial class DebridMediaManagerCrawler(
return; return;
} }
var url = entry.GetProperty("download_url").GetString(); var pageSource = await client.GetStringAsync($"{DownloadBaseUrl}/{name}");
var pageSource = await client.GetStringAsync(url);
await ExtractPageContents(pageSource, name); await ExtractPageContents(pageSource, name);
} }
@@ -130,7 +134,7 @@ public partial class DebridMediaManagerCrawler(
private async Task<(bool Success, string? Name)> IsAlreadyIngested(JsonElement entry) private async Task<(bool Success, string? Name)> IsAlreadyIngested(JsonElement entry)
{ {
var name = entry.GetProperty("name").GetString(); var name = entry.GetProperty("path").GetString();
if (string.IsNullOrEmpty(name)) if (string.IsNullOrEmpty(name))
{ {