tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #111 from iPromKnight/change-retry

Browse Source 
change default retry window to be a larger delay
This commit is contained in:
iPromKnight
2024-03-03 20:29:16 +00:00
committed by GitHub
parent 8298d4021f a609af66f9
commit ab9d26e78e
3 changed files with 18 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/producer/Configuration/torrentio.json
View File

@@ -9,7 +9,7 @@
"IntervalInSeconds": 3600, "IntervalInSeconds": 3600,
"MongoBatchSize": 300, "MongoBatchSize": 300,
"ExceptionLimit": 5, "ExceptionLimit": 5,
"ExceptionIntervalInSeconds": 60 "ExceptionIntervalInSeconds": 300
} }
} }
] ]

31
src/producer/Features/Crawlers/Torrentio/TorrentioCrawler.cs
View File

@@ -69,33 +69,27 @@ public partial class TorrentioCrawler(
{ {
try try
{ {
await state.RetryPolicy.ExecuteAsync( var currentCount = processedItemsCount;
await state.ResiliencyPolicy.ExecuteAsync(
async () => async () =>
{ {
processedItemsCount++;
var waitTime = instance.CalculateWaitTime(state); var waitTime = instance.CalculateWaitTime(state);
if (waitTime > TimeSpan.Zero) if (waitTime > TimeSpan.Zero)
{ {
logger.LogInformation("Rate limit reached for {TorrentioInstance}", instance.Name); logger.LogInformation("Rate limit reached for {TorrentioInstance}", instance.Name);
logger.LogInformation( logger.LogInformation("Waiting for {TorrentioInstance}: {WaitTime} seconds", instance.Name, waitTime / 1000.0);
"Waiting for {TorrentioInstance}: {WaitTime} seconds", instance.Name, waitTime / 1000.0);
await Task.Delay(waitTime); await Task.Delay(waitTime);
} }
if (processedItemsCount % 2 == 0) if (currentCount % 2 == 0)
{ {
var randomWait = Random.Shared.Next(1000, 5000); var randomWait = Random.Shared.Next(1000, 5000);
logger.LogInformation( logger.LogInformation("Waiting for {TorrentioInstance}: {WaitTime} seconds", instance.Name, randomWait / 1000.0);
"Waiting for {TorrentioInstance}: {WaitTime} seconds", instance.Name, randomWait / 1000.0);
await Task.Delay(randomWait); await Task.Delay(randomWait);
} }
await state.ResiliencyPolicy.ExecuteAsync(
async () =>
{
var torrentInfo = await ScrapeInstance(instance, item.ImdbId, client); var torrentInfo = await ScrapeInstance(instance, item.ImdbId, client);
if (torrentInfo is not null) if (torrentInfo is not null)
@@ -103,12 +97,12 @@ public partial class TorrentioCrawler(
newTorrents.AddRange(torrentInfo.Where(x => x != null).Select(x => x!)); newTorrents.AddRange(torrentInfo.Where(x => x != null).Select(x => x!));
} }
}); });
});
processedItemsCount++;
} }
catch (Exception error) catch (Exception)
{ {
logger.LogError(error, "page processing error in TorrentioCrawler"); logger.LogWarning("Too many policy failures. Setting possibly rate limited for {TorrentioInstance}", instance.Name);
logger.LogInformation("Setting possibly rate limited for {TorrentioInstance}", instance.Name);
instance.SetPossiblyRateLimited(state); instance.SetPossiblyRateLimited(state);
} }
} }
@@ -131,7 +125,7 @@ public partial class TorrentioCrawler(
sleepDurationProvider: retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)), sleepDurationProvider: retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)),
onRetry: (exception, timeSpan, retryCount, _) => onRetry: (exception, timeSpan, retryCount, _) =>
{ {
logger.LogWarning($"Retry {retryCount} encountered an exception: {exception.Message}. Pausing for {timeSpan.Seconds} seconds instance {instance.Name}"); logger.LogWarning("Retry {RetryCount} encountered an exception: {Message}. Pausing for {Timespan} seconds instance {TorrentioInstance}", retryCount, exception.Message, timeSpan.Seconds, instance.Name);
}); });
var circuitBreakerPolicy = Policy var circuitBreakerPolicy = Policy
@@ -146,10 +140,9 @@ public partial class TorrentioCrawler(
onReset: () => logger.LogInformation("Circuit closed for {TorrentioInstance}, calls will flow again", instance.Name), onReset: () => logger.LogInformation("Circuit closed for {TorrentioInstance}, calls will flow again", instance.Name),
onHalfOpen: () => logger.LogInformation("Circuit is half-open for {TorrentioInstance}, next call is a trial if it should close or break again", instance.Name)); onHalfOpen: () => logger.LogInformation("Circuit is half-open for {TorrentioInstance}, next call is a trial if it should close or break again", instance.Name));
var policyWrap = Policy.WrapAsync(circuitBreakerPolicy, Policy.TimeoutAsync(TimeSpan.FromSeconds(30))); var policyWrap = Policy.WrapAsync(retryPolicy, circuitBreakerPolicy);
state.ResiliencyPolicy = policyWrap; state.ResiliencyPolicy = policyWrap;
state.RetryPolicy = retryPolicy;
} }
private async Task<List<Torrent?>?> ScrapeInstance(TorrentioInstance instance, string imdbId, HttpClient client) private async Task<List<Torrent?>?> ScrapeInstance(TorrentioInstance instance, string imdbId, HttpClient client)

1
src/producer/Features/Crawlers/Torrentio/TorrentioScrapeInstance.cs
View File

@@ -7,5 +7,4 @@ public class TorrentioScrapeInstance
public int TotalProcessed { get; set; } public int TotalProcessed { get; set; }
public string? LastProcessedImdbId { get; set; } public string? LastProcessedImdbId { get; set; }
public IAsyncPolicy? ResiliencyPolicy { get; set; } public IAsyncPolicy? ResiliencyPolicy { get; set; }
public IAsyncPolicy? RetryPolicy { get; set; }
} }