tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] updates scapers limiter usage

Browse Source
This commit is contained in:
TheBeastLT
2020-03-13 22:12:18 +01:00
parent 19052cc366
commit 542c96a2c3
8 changed files with 114 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
scraper/scrapers/kickass/kickass_api.js
View File

@@ -2,6 +2,7 @@ const cheerio = require('cheerio');
const needle = require('needle');
const moment = require('moment');
const decode = require('magnet-uri');
const Promises = require('../../lib/promises');
const defaultProxies = [
'https://katcr.co'
@@ -26,7 +27,7 @@ function torrent(torrentId, config = {}, retries = 2) {
}
const proxyList = config.proxyList || defaultProxies;
return raceFirstSuccessful(proxyList
return Promises.first(proxyList
.map((proxyUrl) => singleRequest(`${proxyUrl}/torrent/${torrentId}`, config)))
.then((body) => parseTorrentPage(body))
.then((torrent) => ({ torrentId, ...torrent }))
@@ -41,7 +42,7 @@ function search(keyword, config = {}, retries = 2) {
const page = config.page || 1;
const category = config.category;
return raceFirstSuccessful(proxyList
return Promises.first(proxyList
.map((proxyUrl) => singleRequest(`${proxyUrl}/search/${keyword}/${page}/99/${category}`, config)))
.then((body) => parseTableBody(body))
.catch((err) => search(keyword, config, retries - 1));
@@ -55,7 +56,7 @@ function browse(config = {}, retries = 2) {
const page = config.page || 1;
const category = config.category;
return raceFirstSuccessful(proxyList
return Promises.first(proxyList
.map((proxyUrl) => singleRequest(`${proxyUrl}/category/${category}/page/${page}`, config)))
.then((body) => parseTableBody(body))
.catch((err) => browse(config, retries - 1));
@@ -94,9 +95,12 @@ function parseTableBody(body) {
$('.table > tbody > tr').each((i, element) => {
const row = $(element);
const magnetLink = row.find('a[title="Torrent magnet link"]').attr('href');
torrents.push({
torrentId: row.find('a[class="torrents_table__torrent_title"]').first().attr('href').replace('/torrent/', ''),
name: row.find('a[class="torrents_table__torrent_title"]').first().children('b').text(),
infoHash: decode(magnetLink).infoHash,
magnetLink: magnetLink,
torrentId: row.find('a[class="torrents_table__torrent_title"]').first().attr('href').replace('/torrent/', ''),
category: row.find('span[class="torrents_table__upload_info"]').first().children('a').first().attr('href')
.match(/category\/([^\/]+)/)[1],
seeders: parseInt(row.find('td[data-title="Seed"]').first().text()),
@@ -167,21 +171,4 @@ function parseSize(sizeText) {
return Math.floor(parseFloat(sizeText.replace(/[',]/g, '')) * scale);
}
function raceFirstSuccessful(promises) {
return Promise.all(promises.map((p) => {
// If a request fails, count that as a resolution so it will keep
// waiting for other possible successes. If a request succeeds,
// treat it as a rejection so Promise.all immediately bails out.
return p.then(
(val) => Promise.reject(val),
(err) => Promise.resolve(err)
);
})).then(
// If '.all' resolved, we've just got an array of errors.
(errors) => Promise.reject(errors),
// If '.all' rejected, we've got the result we wanted.
(val) => Promise.resolve(val)
);
}
module.exports = { torrent, search, browse, Categories };