tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] update tpb fake removal cron and concurrency

Browse Source
This commit is contained in:
TheBeastLT
2021-06-29 16:27:04 +02:00
parent 991065b4b6
commit 446108a9fe
2 changed files with 5 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
scraper/scrapers/thepiratebay/thepiratebay_fakes_removal.js
View File

@@ -1,19 +1,20 @@
const moment = require('moment');
const { Sequelize } = require('sequelize');
const Bottleneck = require('bottleneck');
const thepiratebay = require('./thepiratebay_api.js');
const { Type } = require('../../lib/types');
const repository = require('../../lib/repository');
const Promises = require('../../lib/promises');
const NAME = 'ThePirateBay';
const EMPTY_HASH = '0000000000000000000000000000000000000000';
const Op = Sequelize.Op;
const limiter = new Bottleneck({ maxConcurrent: 10 });
async function scrape() {
console.log(`Starting ${NAME} fake removal...`);
const startCreatedAt = moment().subtract(14, 'day');
const endCreatedAt = moment().subtract(1, 'day');
const endCreatedAt = moment();
const whereQuery = {
provider: NAME,
type: Type.MOVIE,
@@ -22,7 +23,7 @@ async function scrape() {
return repository.getTorrentsBasedOnQuery(whereQuery)
.then(torrents => {
console.log(`Checking for ${NAME} fake entries in ${torrents.length} torrents`);
return Promises.sequence(torrents.map(torrent => () => removeIfFake(torrent)))
return Promise.all(torrents.map(torrent => limiter.schedule(() => removeIfFake(torrent))))
})
.then(results => {
const removed = results.filter(result => result);