tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] checks for torrent contents when updating existing torrent

Browse Source
This commit is contained in:
TheBeastLT
2020-05-06 11:59:53 +02:00
parent 0ad7324a6a
commit 9ce3f72e5d
12 changed files with 125 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
scraper/scrapers/yts/yts_scraper.js
View File

@@ -3,12 +3,12 @@ const Bottleneck = require('bottleneck');
const yts = require('./yts_api');
const { Type } = require('../../lib/types');
const repository = require('../../lib/repository');
const { createTorrentEntry, getStoredTorrentEntry, updateTorrentSeeders } = require('../../lib/torrentEntries');
const { createTorrentEntry, checkAndUpdateTorrent } = require('../../lib/torrentEntries');
const NAME = 'YTS';
const UNTIL_PAGE = 2;
const limiter = new Bottleneck({ maxConcurrent: 20 });
const limiter = new Bottleneck({ maxConcurrent: 10 });
async function scrape() {
const scrapeStart = moment();
@@ -45,8 +45,8 @@ async function scrapeLatestTorrentsForCategory(page = 1) {
}
async function processTorrentRecord(record) {
if (await getStoredTorrentEntry(record)) {
return updateTorrentSeeders(record);
if (await checkAndUpdateTorrent(record)) {
return record;
}
if (!record || !record.size) {