mirror of
https://github.com/knightcrawler-stremio/knightcrawler.git
synced 2024-12-20 03:29:51 +00:00
Run pre-commit
This commit is contained in:
@@ -8,12 +8,12 @@ public partial class DebridMediaManagerCrawler(
|
||||
{
|
||||
[GeneratedRegex("""<iframe src="https:\/\/debridmediamanager.com\/hashlist#(.*)"></iframe>""")]
|
||||
private static partial Regex HashCollectionMatcher();
|
||||
|
||||
|
||||
[GeneratedRegex(@"[sS]([0-9]{1,2})|seasons?[\s-]?([0-9]{1,2})", RegexOptions.IgnoreCase, "en-GB")]
|
||||
private static partial Regex SeasonMatcher();
|
||||
|
||||
private const string DownloadBaseUrl = "https://raw.githubusercontent.com/debridmediamanager/hashlists/main";
|
||||
|
||||
|
||||
protected override IReadOnlyDictionary<string, string> Mappings => new Dictionary<string, string>();
|
||||
protected override string Url => "https://api.github.com/repos/debridmediamanager/hashlists/git/trees/main?recursive=1";
|
||||
protected override string Source => "DMM";
|
||||
@@ -23,13 +23,13 @@ public partial class DebridMediaManagerCrawler(
|
||||
var client = httpClientFactory.CreateClient("Scraper");
|
||||
client.DefaultRequestHeaders.Authorization = new("Bearer", githubConfiguration.PAT);
|
||||
client.DefaultRequestHeaders.UserAgent.ParseAdd("curl");
|
||||
|
||||
|
||||
var jsonBody = await client.GetStringAsync(Url);
|
||||
|
||||
|
||||
var json = JsonDocument.Parse(jsonBody);
|
||||
|
||||
var entriesArray = json.RootElement.GetProperty("tree");
|
||||
|
||||
|
||||
logger.LogInformation("Found {Entries} total DMM pages", entriesArray.GetArrayLength());
|
||||
|
||||
foreach (var entry in entriesArray.EnumerateArray())
|
||||
@@ -41,21 +41,21 @@ public partial class DebridMediaManagerCrawler(
|
||||
private async Task ParsePage(JsonElement entry, HttpClient client)
|
||||
{
|
||||
var (pageIngested, name) = await IsAlreadyIngested(entry);
|
||||
|
||||
|
||||
if (string.IsNullOrEmpty(name) || pageIngested)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
var pageSource = await client.GetStringAsync($"{DownloadBaseUrl}/{name}");
|
||||
|
||||
|
||||
await ExtractPageContents(pageSource, name);
|
||||
}
|
||||
|
||||
private async Task ExtractPageContents(string pageSource, string name)
|
||||
{
|
||||
var match = HashCollectionMatcher().Match(pageSource);
|
||||
|
||||
|
||||
if (!match.Success)
|
||||
{
|
||||
logger.LogWarning("Failed to match hash collection for {Name}", name);
|
||||
@@ -64,32 +64,32 @@ public partial class DebridMediaManagerCrawler(
|
||||
}
|
||||
|
||||
var encodedJson = match.Groups.Values.ElementAtOrDefault(1);
|
||||
|
||||
|
||||
if (string.IsNullOrEmpty(encodedJson?.Value))
|
||||
{
|
||||
logger.LogWarning("Failed to extract encoded json for {Name}", name);
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
await ProcessExtractedContentsAsTorrentCollection(encodedJson.Value, name);
|
||||
}
|
||||
|
||||
private async Task ProcessExtractedContentsAsTorrentCollection(string encodedJson, string name)
|
||||
{
|
||||
var decodedJson = LZString.DecompressFromEncodedURIComponent(encodedJson);
|
||||
|
||||
|
||||
var json = JsonDocument.Parse(decodedJson);
|
||||
|
||||
await InsertTorrentsForPage(json);
|
||||
|
||||
|
||||
var result = await Storage.MarkPageAsIngested(name);
|
||||
|
||||
|
||||
if (!result.Success)
|
||||
{
|
||||
logger.LogWarning("Failed to mark page as ingested: [{Error}]", result.ErrorMessage);
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
logger.LogInformation("Successfully marked page as ingested");
|
||||
}
|
||||
|
||||
@@ -109,7 +109,7 @@ public partial class DebridMediaManagerCrawler(
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
torrent.Category = SeasonMatcher().IsMatch(torrent.Name) ? "tv" : "movies";
|
||||
|
||||
return torrent;
|
||||
@@ -120,16 +120,16 @@ public partial class DebridMediaManagerCrawler(
|
||||
var torrents = json.RootElement.EnumerateArray()
|
||||
.Select(ParseTorrent)
|
||||
.ToList();
|
||||
|
||||
|
||||
if (torrents.Count == 0)
|
||||
{
|
||||
logger.LogWarning("No torrents found in {Source} response", Source);
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
await InsertTorrents(torrents!);
|
||||
}
|
||||
|
||||
|
||||
private async Task<(bool Success, string? Name)> IsAlreadyIngested(JsonElement entry)
|
||||
{
|
||||
var name = entry.GetProperty("path").GetString();
|
||||
@@ -138,9 +138,9 @@ public partial class DebridMediaManagerCrawler(
|
||||
{
|
||||
return (false, null);
|
||||
}
|
||||
|
||||
|
||||
var pageIngested = await Storage.PageIngested(name);
|
||||
|
||||
return (pageIngested, name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user