tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] rework scraper scheduling and added seeders updating

Browse Source
This commit is contained in:
TheBeastLT
2020-04-23 16:33:08 +02:00
parent d01e2c9b35
commit 9ea3932af1
19 changed files with 128 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
scraper/scrapers/1337x/1337x_scraper.js
View File

@@ -27,10 +27,7 @@ async function scrape() {
}
async function updateSeeders(torrent) {
return limiter.schedule(() => leetx.torrent(torrent.torrentId)
.then(record => (torrent.seeders = record.seeders, torrent))
.catch(() => updateCurrentSeeders(torrent))
.then(updated => updateTorrentSeeders(updated)));
return limiter.schedule(() => leetx.torrent(torrent.torrentId));
}
async function scrapeLatestTorrents() {

2
scraper/scrapers/eztv/eztv_api.js
View File

@@ -33,7 +33,7 @@ function search(imdbId, config = {}, retries = 2) {
return Promises.first(defaultProxies
.map(proxyUrl => singleRequest(`${proxyUrl}/api/get-torrents?limit=${limit}&page=${page}&imdb_id=${id}`, config)))
.then(results => parseResults(results))
.then(torrents => torrents.length === limit && page < maxPage
.then(torrents => torrents.length === limit && page < maxPage && !torrents.find(t => t.imdbId === imdbId)
? search(imdbId, { ...config, page: page + 1 })
.catch(() => [])
.then(nextTorrents => torrents.concat(nextTorrents))

14
scraper/scrapers/eztv/eztv_scraper.js
View File

@@ -4,14 +4,12 @@ const eztv = require('./eztv_api');
const { Type } = require('../../lib/types');
const Promises = require('../../lib/promises');
const repository = require('../../lib/repository');
const { updateCurrentSeeders } = require('../../lib/torrent');
const { createTorrentEntry, getStoredTorrentEntry, updateTorrentSeeders } = require('../../lib/torrentEntries');
const NAME = 'EZTV';
const UNTIL_PAGE = 10;
const limiter = new Bottleneck({ maxConcurrent: 20 });
const limiter = new Bottleneck({ maxConcurrent: 1 });
async function scrape() {
const scrapeStart = moment();
@@ -26,11 +24,9 @@ async function scrape() {
.then(() => console.log(`[${moment()}] finished ${NAME} scrape`));
}
async function updateSeeders(torrent) {
return limiter.schedule(() => eztv.torrent(torrent.torrentId)
.then(record => (torrent.seeders = record.seeders, torrent))
.catch(() => updateCurrentSeeders(torrent))
.then(updated => updateTorrentSeeders(updated)));
async function updateSeeders(torrent, getImdbIdsMethod) {
return getImdbIdsMethod().then(imdbIds => Promises.sequence(imdbIds
.map(imdbId => limiter.schedule(() => eztv.search(imdbId)))));
}
async function scrapeLatestTorrents() {
@@ -45,7 +41,7 @@ async function scrapeLatestTorrentsForCategory(page = 1) {
// return Promises.delay(30000).then(() => scrapeLatestTorrentsForCategory(page))
return Promise.resolve([]);
})
.then(torrents => Promise.all(torrents.map(torrent => limiter.schedule(() => processTorrentRecord(torrent)))))
.then(torrents => Promise.all(torrents.map(torrent => processTorrentRecord(torrent))))
.then(resolved => resolved.length > 0 && page < UNTIL_PAGE
? scrapeLatestTorrentsForCategory(page + 1)
: Promise.resolve());

3
scraper/scrapers/horriblesubs/horriblesubs_scraper.js
View File

@@ -37,8 +37,7 @@ async function scrape() {
}
async function updateSeeders(torrent) {
return entryLimiter.schedule(() => updateCurrentSeeders(torrent)
.then(updated => updateTorrentSeeders(updated)));
return Promise.resolve([]);
}
async function _scrapeLatestEntries() {

5
scraper/scrapers/kickass/kickass_scraper.js
View File

@@ -27,10 +27,7 @@ async function scrape() {
}
async function updateSeeders(torrent) {
return limiter.schedule(() => kickass.torrent(torrent.torrentId)
.then(record => (torrent.seeders = record.seeders, torrent))
.catch(() => updateCurrentSeeders(torrent))
.then(updated => updateTorrentSeeders(updated)));
return limiter.schedule(() => kickass.torrent(torrent.torrentId));
}
async function scrapeLatestTorrents() {

12
scraper/scrapers/rarbg/rarbg_scraper.js
View File

@@ -26,15 +26,9 @@ async function scrape() {
.then(() => console.log(`[${moment()}] finished ${NAME} scrape`));
}
async function updateSeeders(torrent) {
const imdbIds = await repository.getFiles(torrent)
.then(files => files.map(file => file.imdbId))
.then(ids => Array.from(new Set(ids)));
return Promise.all(imdbIds.map(imdbId => limiter.schedule(() => rarbg.search(imdbId, SEARCH_OPTIONS, 'imdb'))))
.then(results => results.reduce((a, b) => a.concat(b), []))
.then(results => results.map(result => toTorrent(result)))
.then(torrents => Promise.all(torrents.map(updated => updateTorrentSeeders(updated))));
async function updateSeeders(torrent, getImdbIdsMethod) {
return getImdbIdsMethod().then(imdbIds => Promises.sequence(imdbIds
.map(imdbId => limiter.schedule(() => rarbg.search(imdbId, SEARCH_OPTIONS, 'imdb')))));
}
async function scrapeLatestTorrents() {

5
scraper/scrapers/thepiratebay/thepiratebay_scraper.js
View File

@@ -39,10 +39,7 @@ async function scrape() {
}
async function updateSeeders(torrent) {
return limiter.schedule(() => thepiratebay.torrent(torrent.torrentId)
.then(record => (torrent.seeders = record.seeders, torrent))
.catch(() => updateCurrentSeeders(torrent))
.then(updated => updateTorrentSeeders(updated)));
return limiter.schedule(() => thepiratebay.torrent(torrent.torrentId));
}
async function scrapeLatestTorrents() {

4
scraper/scrapers/yts/yts_scraper.js
View File

@@ -24,9 +24,7 @@ async function scrape() {
}
async function updateSeeders(torrent) {
return limiter.schedule(() => yts.torrent(torrent.torrentId)
.then(records => records.map(record => ({ ...record, provider: NAME })))
.then(records => Promise.all(records.map(record => updateTorrentSeeders(record)))));
return limiter.schedule(() => yts.torrent(torrent.torrentId));
}
async function scrapeLatestTorrents() {