tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #108 from iPromKnight/hotfix-rethrow

Browse Source 
Rethrow so polly captures failures on requests.
This commit is contained in:
iPromKnight
2024-03-03 19:22:37 +00:00
committed by GitHub
parent 9e32e5fe19 c61e9e94e1
commit 5dd53de40c
2 changed files with 4 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
src/producer/Features/Crawlers/Torrentio/TorrentioCrawler.cs
View File

@@ -144,10 +144,8 @@ public partial class TorrentioCrawler(
catch (Exception error)
{
logger.LogError(error, "page processing error {TorrentioInstance}: {ImdbId}", instance.Name, imdbId);
logger.LogWarning("Setting {TorrentioInstance} to possibly rate limited for Five minutes", instance.Name);
instance.SetPossiblyRateLimited(_instanceStates);
throw;
}
return null;
}
private async Task<List<Torrent?>?> RunRequest(TorrentioInstance instance, string urlSlug, string imdbId, HttpClient client)
@@ -178,15 +176,10 @@ public partial class TorrentioCrawler(
var torrent = ParseTorrentDetails(title, instance, infoHash, imdId);
if (torrent is null || string.IsNullOrEmpty(torrent.Name))
{
return null;
}
return torrent;
return string.IsNullOrEmpty(torrent.Name) ? null : torrent;
}
private Torrent? ParseTorrentDetails(string title, TorrentioInstance instance, string infoHash, string imdbId)
private Torrent ParseTorrentDetails(string title, TorrentioInstance instance, string infoHash, string imdbId)
{
try
{
@@ -226,7 +219,7 @@ public partial class TorrentioCrawler(
catch (Exception e)
{
logger.LogError(e, "Error parsing torrent details");
return null;
throw;
}
}
}

13
src/producer/Features/Crawlers/Torrentio/TorrentioInstanceExtensions.cs
View File

@@ -21,20 +21,7 @@ public static class TorrentioInstancesExtensions
return remaining > TimeSpan.Zero ? remaining : TimeSpan.Zero;
}
public static void SetPossiblyRateLimited(this TorrentioInstance instance, Dictionary<string, TorrentioScrapeInstance> scraperState, int minutesToWait = 5)
{
var state = instance.EnsureStateExists(scraperState);
// Set the start time to 15 minutes in the past so that the next check will result in a rate limit period of 15 minutes
var startedAt = DateTime.UtcNow.AddMinutes(-minutesToWait);
var requestCount = instance.RateLimit.RequestLimit;
// Update the scraper state for the instance
state.StartedAt = startedAt;
state.RequestCount = requestCount;
}
public static long TotalProcessedRequests(this TorrentioInstance instance, Dictionary<string, TorrentioScrapeInstance> scraperState) =>
!scraperState.TryGetValue(instance.Name, out var state) ? 0 : state.TotalProcessed;