tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] updates 1337x search api to extend results

Browse Source
This commit is contained in:
TheBeastLT
2020-04-25 10:29:10 +02:00
parent 43c2e737f8
commit 10615e4aed
2 changed files with 13 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
scraper/scrapers/1337x/1337x_api.js
View File

@@ -9,6 +9,7 @@ const defaultProxies = [
'https://1337x.to' 'https://1337x.to'
]; ];
const defaultTimeout = 10000; const defaultTimeout = 10000;
const maxSearchPage = 50;
const Categories = { const Categories = {
MOVIE: 'Movies', MOVIE: 'Movies',
@@ -42,10 +43,19 @@ function search(keyword, config = {}, retries = 2) {
} }
const proxyList = config.proxyList || defaultProxies; const proxyList = config.proxyList || defaultProxies;
const page = config.page || 1; const page = config.page || 1;
const category = config.category;
const extendToPage = Math.min(maxSearchPage, (config.extendToPage || 1))
const requestUrl = proxyUrl => category
? `${proxyUrl}/category-search/${keyword}/${category}/${page}/`
: `${proxyUrl}/search/${keyword}/${page}/`;
return Promises.first(proxyList return Promises.first(proxyList
.map((proxyUrl) => singleRequest(`${proxyUrl}/search/${keyword}/${page}/`, config))) .map(proxyUrl => singleRequest(requestUrl(proxyUrl), config)))
.then((body) => parseTableBody(body)) .then(body => parseTableBody(body))
.then(torrents => torrents.length === 40 && page < extendToPage
? search(keyword, { ...config, page: page + 1 }).catch(() => [])
.then(nextTorrents => torrents.concat(nextTorrents))
: torrents)
.catch((err) => search(keyword, config, retries - 1)); .catch((err) => search(keyword, config, retries - 1));
} }

2
scraper/scrapers/1337x/1337x_scraper.js
View File

@@ -73,7 +73,7 @@ async function processTorrentRecord(record) {
infoHash: torrentFound.infoHash, infoHash: torrentFound.infoHash,
provider: NAME, provider: NAME,
torrentId: torrentFound.torrentId, torrentId: torrentFound.torrentId,
title: torrentFound.name.replace(/\t|\s+/g, ' '), title: torrentFound.name.replace(/\t|\s+/g, ' ').trim(),
type: TYPE_MAPPING[torrentFound.category], type: TYPE_MAPPING[torrentFound.category],
size: torrentFound.size, size: torrentFound.size,
seeders: torrentFound.seeders, seeders: torrentFound.seeders,