tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

remove scraper module

Browse Source
This commit is contained in:
TheBeastLT
2022-06-16 17:44:38 +03:00
parent a253e8e0b0
commit 554c07d636
69 changed files with 0 additions and 61771 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
scraper/scheduler/scheduler.js
View File

@@ -1,14 +0,0 @@
const { scheduleScraping, scrapeAll } = require('./scraper')
const { scheduleUpdateSeeders, scheduleUpdateSeedersForNewTorrents } = require('./seeders')
function startScraper() {
if (process.env.ENABLE_SCHEDULING) {
scheduleScraping();
scheduleUpdateSeeders();
scheduleUpdateSeedersForNewTorrents();
} else {
scrapeAll()
}
}
module.exports = { startScraper }

28
scraper/scheduler/scraper.js
View File

@@ -1,28 +0,0 @@
const schedule = require('node-schedule');
const scrapers = require('./scrapers');
const { sequence } = require('../lib/promises')
function scheduleScraping() {
const allCrons = scrapers.reduce((crons, provider) => {
crons[provider.cron] = (crons[provider.cron] || []).concat(provider)
return crons;
}, {});
Object.entries(allCrons).forEach(([cron, providers]) => schedule.scheduleJob(cron, () => _scrapeProviders(providers)))
}
function scrapeAll() {
return _scrapeProviders(scrapers)
}
async function _scrapeProviders(providers) {
return sequence(providers.map(provider => () => _singleScrape(provider)));
}
async function _singleScrape(provider) {
return provider.scraper.scrape().catch(error => {
console.warn(`Failed ${provider.name} scraping due: `, error);
return Promise.resolve()
})
}
module.exports = { scheduleScraping, scrapeAll }

47
scraper/scheduler/scrapers.js
View File

@@ -1,47 +0,0 @@
const thepiratebayScraper = require('../scrapers/thepiratebay/thepiratebay_scraper');
const thepiratebayFakeRemoval = require('../scrapers/thepiratebay/thepiratebay_fakes_removal');
const ytsScraper = require('../scrapers/yts/yts_scraper');
const ytsFullScraper = require('../scrapers/yts/yts_full_scraper');
const eztvScraper = require('../scrapers/eztv/eztv_scraper');
const leetxScraper = require('../scrapers/1337x/1337x_scraper');
const rarbgScraper = require('../scrapers/rarbg/rarbg_scraper');
const nyaaPantsuScraper = require('../scrapers/nyaapantsu/nyaa_pantsu_scraper');
const nyaaSiScraper = require('../scrapers/nyaasi/nyaa_si_scraper');
const erairawsScraper = require('../scrapers/erairaws/erairaws_scraper');
const torrentGalaxyScraper = require('../scrapers/torrentgalaxy/torrentgalaxy_scraper');
const rutorScraper = require('../scrapers/rutor/rutor_scraper');
const Comando = require('../scrapers/comando/comando_scraper')
const ComoEuBaixo = require('../scrapers/comoeubaixo/comoeubaixo_scraper')
const Lapumia = require('../scrapers/lapumia/lapumia_scraper')
const OndeBaixa = require('../scrapers/ondebaixa/ondebaixa_scraper');
const AnimesTorrent = require('../scrapers/animestorrent/animestorrent_scraper')
const DarkMahou = require('../scrapers/darkmahou/darkmahou_scraper')
const torrent9Scraper = require('../scrapers/torrent9/torrent9_scraper');
module.exports = [
{ scraper: ytsScraper, name: ytsScraper.NAME, cron: '0 0 */4 ? * *' },
{ scraper: ytsFullScraper, name: ytsFullScraper.NAME, cron: '0 0 0 * * 0' },
{ scraper: eztvScraper, name: eztvScraper.NAME, cron: '0 0 */4 ? * *' },
{ scraper: nyaaSiScraper, name: nyaaSiScraper.NAME, cron: '0 0 */4 ? * *' },
{ scraper: nyaaPantsuScraper, name: nyaaPantsuScraper.NAME, cron: '0 0 */4 ? * *' },
{ scraper: rarbgScraper, name: rarbgScraper.NAME, cron: '0 0 */1 ? * *' },
{ scraper: rutorScraper, name: rutorScraper.NAME, cron: '0 0 */4 ? * *' },
{ scraper: thepiratebayScraper, name: thepiratebayScraper.NAME, cron: '0 0 */2 ? * *' },
{ scraper: thepiratebayFakeRemoval, name: thepiratebayFakeRemoval.NAME, cron: '0 0 */12 ? * *' },
{ scraper: torrentGalaxyScraper, name: torrentGalaxyScraper.NAME, cron: '0 0 */4 ? * *' },
{ scraper: leetxScraper, name: leetxScraper.NAME, cron: '0 0 */4 ? * *' },
{ scraper: torrent9Scraper, name: torrent9Scraper.NAME, cron: '0 0 */4 ? * *' },
{ scraper: Comando, name: Comando.NAME, cron: '0 0 */4 ? * *' },
{ scraper: ComoEuBaixo, name: ComoEuBaixo.NAME, cron: '0 0 */4 ? * *' },
{ scraper: Lapumia, name: Lapumia.NAME, cron: '0 0 */4 ? * *' },
{ scraper: OndeBaixa, name: OndeBaixa.NAME, cron: '0 0 */4 ? * *' },
// { scraper: AnimesTorrent, name: AnimesTorrent.NAME, cron: '0 0 */4 ? * *' },
// { scraper: DarkMahou, name: DarkMahou.NAME, cron: '0 0 */4 ? * *' },
{ scraper: erairawsScraper, name: erairawsScraper.NAME, cron: '0 0 */4 ? * *' },
// { scraper: require('../scrapers/rarbg/rarbg_dump_scraper') }
// { scraper: require('../scrapers/1337x/1337x_search_scraper') }
// { scraper: require('../scrapers/rarbg/rarbg_dump_scraper') }
// { scraper: require('../scrapers/thepiratebay/thepiratebay_dump_scraper') }
// { scraper: require('../scrapers/thepiratebay/thepiratebay_unofficial_dump_scraper') }
// { scraper: require('../scrapers/thepiratebay/thepiratebay_update_size_scraper') }
];

64
scraper/scheduler/seeders.js
View File

@@ -1,64 +0,0 @@
const Bottleneck = require('bottleneck');
const repository = require('../lib/repository')
const { delay } = require('../lib/promises')
const { updateCurrentSeeders } = require('../lib/torrent')
const { updateTorrentSeeders } = require('../lib/torrentEntries')
const DELAY_MS = 0; // 0 seconds
const DELAY_NEW_MS = 30_000; // 30 seconds
const DELAY_NO_NEW_MS = 300_000; // 300 seconds
const DELAY_FAILED_TORRENTS_MS = 5_000; // 5 seconds
const updateLimiter = new Bottleneck({ maxConcurrent: 5 });
const statistics = {};
const statisticsNew = {};
function scheduleUpdateSeeders() {
console.log('Starting seeders update...')
getTorrents()
.then(torrents => updateCurrentSeeders(torrents))
.then(updatedTorrents => Promise.all(
updatedTorrents.map(updated => updateLimiter.schedule(() => updateTorrentSeeders(updated)))))
.then(torrents => updateStatistics(torrents, statistics))
.then(() => console.log('Finished seeders update:', statistics))
.catch(error => console.warn('Failed seeders update:', error))
.then(() => delay(DELAY_MS))
.then(() => scheduleUpdateSeeders());
}
function scheduleUpdateSeedersForNewTorrents() {
console.log('Starting seeders update for new torrents...')
getNewTorrents()
.then(torrents => updateCurrentSeeders(torrents))
.then(updatedTorrents => Promise.all(
updatedTorrents.map(updated => updateLimiter.schedule(() => updateTorrentSeeders(updated)))))
.then(torrents => updateStatistics(torrents, statisticsNew))
.then(() => console.log('Finished seeders update for new torrents:', statisticsNew))
.catch(error => console.warn('Failed seeders update for new torrents:', error))
.then(() => delay(DELAY_NEW_MS))
.then(() => scheduleUpdateSeedersForNewTorrents());
}
async function getTorrents() {
return repository.getUpdateSeedersTorrents()
.catch(() => delay(DELAY_FAILED_TORRENTS_MS).then(() => getTorrents()));
}
async function getNewTorrents() {
return repository.getUpdateSeedersNewTorrents()
.catch(() => delay(DELAY_FAILED_TORRENTS_MS).then(() => getNewTorrents()))
.then(torrents => {
if (!torrents.length) {
console.log('No new torrents to update seeders')
return delay(DELAY_NO_NEW_MS).then(() => getNewTorrents())
}
return torrents;
});
}
function updateStatistics(updatedTorrents, statisticsObject) {
const totalTorrents = updatedTorrents.map(nested => nested.length).reduce((a, b) => a + b, 0);
const date = new Date().toISOString().replace(/T.*/, '');
statisticsObject[date] = (statisticsObject[date] || 0) + totalTorrents;
}
module.exports = { scheduleUpdateSeeders, scheduleUpdateSeedersForNewTorrents }