diff --git a/scraper/scheduler/scrapers.js b/scraper/scheduler/scrapers.js index 1f1254e..5339cbb 100644 --- a/scraper/scheduler/scrapers.js +++ b/scraper/scheduler/scrapers.js @@ -18,7 +18,7 @@ module.exports = [ { scraper: rarbgScraper, name: rarbgScraper.NAME, cron: '0 0 */1 ? * *' }, { scraper: rutorScraper, name: rutorScraper.NAME, cron: '0 0 */4 ? * *' }, { scraper: thepiratebayScraper, name: thepiratebayScraper.NAME, cron: '0 0 */2 ? * *' }, - { scraper: thepiratebayFakeRemoval, name: thepiratebayFakeRemoval.NAME, cron: '0 30 1 ? * * *' }, + { scraper: thepiratebayFakeRemoval, name: thepiratebayFakeRemoval.NAME, cron: '0 0 */12 ? * *' }, { scraper: torrentGalaxyScraper, name: torrentGalaxyScraper.NAME, cron: '0 0 */4 ? * *' }, { scraper: leetxScraper, name: leetxScraper.NAME, cron: '0 0 */4 ? * *' }, // { scraper: kickassScraper, name: kickassScraper.NAME, cron: '0 0 */4 ? * *' }, diff --git a/scraper/scrapers/thepiratebay/thepiratebay_fakes_removal.js b/scraper/scrapers/thepiratebay/thepiratebay_fakes_removal.js index beb9070..93917ad 100644 --- a/scraper/scrapers/thepiratebay/thepiratebay_fakes_removal.js +++ b/scraper/scrapers/thepiratebay/thepiratebay_fakes_removal.js @@ -1,19 +1,20 @@ const moment = require('moment'); const { Sequelize } = require('sequelize'); +const Bottleneck = require('bottleneck'); const thepiratebay = require('./thepiratebay_api.js'); const { Type } = require('../../lib/types'); const repository = require('../../lib/repository'); -const Promises = require('../../lib/promises'); const NAME = 'ThePirateBay'; const EMPTY_HASH = '0000000000000000000000000000000000000000'; const Op = Sequelize.Op; +const limiter = new Bottleneck({ maxConcurrent: 10 }); async function scrape() { console.log(`Starting ${NAME} fake removal...`); const startCreatedAt = moment().subtract(14, 'day'); - const endCreatedAt = moment().subtract(1, 'day'); + const endCreatedAt = moment(); const whereQuery = { provider: NAME, type: Type.MOVIE, @@ -22,7 +23,7 @@ async function scrape() { return repository.getTorrentsBasedOnQuery(whereQuery) .then(torrents => { console.log(`Checking for ${NAME} fake entries in ${torrents.length} torrents`); - return Promises.sequence(torrents.map(torrent => () => removeIfFake(torrent))) + return Promise.all(torrents.map(torrent => limiter.schedule(() => removeIfFake(torrent)))) }) .then(results => { const removed = results.filter(result => result);