mirror of
https://github.com/knightcrawler-stremio/knightcrawler.git
synced 2024-12-20 03:29:51 +00:00
Big rewrite - distributed consumers for ingestion / scraping(scalable) - single producer written in c#.
Changed from page scraping to rss xml scraping Includes RealDebridManager hashlist decoding (requires a github readonly PAT as requests must be authenticated) - This allows ingestion of 200k+ entries in a few hours. Simplifies a lot of torrentio to deal with new data
This commit is contained in:
55
src/node/consumer/lib/promises.js
Normal file
55
src/node/consumer/lib/promises.js
Normal file
@@ -0,0 +1,55 @@
|
||||
/**
|
||||
* Execute promises in sequence one after another.
|
||||
*/
|
||||
export async function sequence(promises) {
|
||||
return promises.reduce((promise, func) =>
|
||||
promise.then(result => func().then(Array.prototype.concat.bind(result))), Promise.resolve([]));
|
||||
}
|
||||
|
||||
/**
|
||||
* Return first resolved promise as the result.
|
||||
*/
|
||||
export async function first(promises) {
|
||||
return Promise.all(promises.map((p) => {
|
||||
// If a request fails, count that as a resolution so it will keep
|
||||
// waiting for other possible successes. If a request succeeds,
|
||||
// treat it as a rejection so Promise.all immediately bails out.
|
||||
return p.then(
|
||||
(val) => Promise.reject(val),
|
||||
(err) => Promise.resolve(err)
|
||||
);
|
||||
})).then(
|
||||
// If '.all' resolved, we've just got an array of errors.
|
||||
(errors) => Promise.reject(errors),
|
||||
// If '.all' rejected, we've got the result we wanted.
|
||||
(val) => Promise.resolve(val)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Delay promise
|
||||
*/
|
||||
export async function delay(duration) {
|
||||
return new Promise((resolve) => setTimeout(resolve, duration));
|
||||
}
|
||||
|
||||
/**
|
||||
* Timeout promise after a set time in ms
|
||||
*/
|
||||
export async function timeout(timeoutMs, promise, message = 'Timed out') {
|
||||
return Promise.race([
|
||||
promise,
|
||||
new Promise(function (resolve, reject) {
|
||||
setTimeout(function () {
|
||||
reject(message);
|
||||
}, timeoutMs);
|
||||
})
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return most common value from given array.
|
||||
*/
|
||||
export function mostCommonValue(array) {
|
||||
return array.sort((a, b) => array.filter(v => v === a).length - array.filter(v => v === b).length).pop();
|
||||
}
|
||||
Reference in New Issue
Block a user