tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] adds update seeders cron

Browse Source
This commit is contained in:
TheBeastLT
2020-04-08 15:53:02 +02:00
parent d3cc8c6b57
commit ce0e330172
10 changed files with 118 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
scraper/scrapers/1337x/1337x_scraper.js
View File

@@ -4,6 +4,7 @@ const leetx = require('./1337x_api');
const { Type } = require('../../lib/types');
const repository = require('../../lib/repository');
const Promises = require('../../lib/promises');
const { updateCurrentSeeders } = require('../../lib/torrent');
const { createTorrentEntry, getStoredTorrentEntry, updateTorrentSeeders } = require('../../lib/torrentEntries');
const NAME = '1337x';
@@ -25,6 +26,13 @@ async function scrape() {
.then(() => console.log(`[${moment()}] finished ${NAME} scrape`));
}
async function updateSeeders(torrent) {
return limiter.schedule(() => leetx.torrent(torrent.torrentId)
.then(record => (torrent.seeders = record.seeders, torrent))
.catch(() => updateCurrentSeeders(torrent))
.then(updated => updateTorrentSeeders(updated)));
}
async function scrapeLatestTorrents() {
const allowedCategories = [
leetx.Categories.MOVIE,
@@ -80,15 +88,6 @@ async function processTorrentRecord(record) {
return createTorrentEntry(torrent);
}
async function updateSeeders() {
const startDate = moment().subtract(7, 'day').toDate();
const endDate = moment().subtract(1, 'day').toDate();
return repository.getTorrentsUpdatedBetween(NAME, startDate, endDate)
.then(torrents => Promise.all(torrents.map(torrent => limiter.schedule(() => leetx.torrent(torrent.torrentId)
.then(foundTorrent => updateTorrentSeeders(foundTorrent))
.catch(error => console.warn(error))))))
}
function typeMapping() {
const mapping = {};
mapping[leetx.Categories.MOVIE] = Type.MOVIE;
@@ -98,4 +97,4 @@ function typeMapping() {
return mapping;
}
module.exports = { scrape, NAME };
module.exports = { scrape, updateSeeders, NAME };