tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] includes Movies/x265/1080 category for rarbg

Browse Source
This commit is contained in:
TheBeastLT
2020-05-07 13:14:05 +02:00
parent 3b2617cd3c
commit 0d98b6b3ae
2 changed files with 20 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
scraper/scrapers/rarbg/rarbg_dump_scraper.js
View File

@@ -4,10 +4,28 @@ const rarbg = require('rarbg-api');
const decode = require('magnet-uri'); const decode = require('magnet-uri');
const { Type } = require('../../lib/types'); const { Type } = require('../../lib/types');
const { createTorrentEntry, checkAndUpdateTorrent } = require('../../lib/torrentEntries'); const { createTorrentEntry, checkAndUpdateTorrent } = require('../../lib/torrentEntries');
const NAME = 'RARBG'; const NAME = 'RARBG';
const limiter = new Bottleneck({ maxConcurrent: 1, minTime: 2500 }); const limiter = new Bottleneck({ maxConcurrent: 1, minTime: 2500 });
const entryLimiter = new Bottleneck({ maxConcurrent: 40 }); const entryLimiter = new Bottleneck({ maxConcurrent: 40 });
const allowedCategories = [
rarbg.CATEGORY['4K_MOVIES_X264_4k'],
rarbg.CATEGORY['4K_X265_4k'],
rarbg.CATEGORY['4k_X264_4k_HDR'],
rarbg.CATEGORY.MOVIES_XVID,
rarbg.CATEGORY.MOVIES_XVID_720P,
[54], // Movies/x265/1080
rarbg.CATEGORY.MOVIES_X264,
rarbg.CATEGORY.MOVIES_X264_1080P,
rarbg.CATEGORY.MOVIES_X264_720P,
rarbg.CATEGORY.MOVIES_X264_3D,
rarbg.CATEGORY.MOVIES_BD_REMUX,
rarbg.CATEGORY.TV_EPISODES,
rarbg.CATEGORY.TV_UHD_EPISODES,
rarbg.CATEGORY.TV_HD_EPISODES
].reduce((a, b) => a.concat(b), [])
const searchOptions = { limit: 100, category: allowedCategories, sort: 'seeders', format: 'json_extended', ranked: 0 }
async function scrape() { async function scrape() {
console.log(`[${moment()}] starting ${NAME} dump scrape...`); console.log(`[${moment()}] starting ${NAME} dump scrape...`);
@@ -22,7 +40,7 @@ async function scrape() {
} }
async function getTorrentsForImdbId(imdbId, retries = 5) { async function getTorrentsForImdbId(imdbId, retries = 5) {
return rarbg.search(imdbId, { limit: 100, sort: 'seeders', format: 'json_extended', ranked: 0 }, 'imdb') return rarbg.search(imdbId, searchOptions, 'imdb')
.then(torrents => torrents.map(torrent => ({ .then(torrents => torrents.map(torrent => ({
name: torrent.title, name: torrent.title,
infoHash: decode(torrent.download).infoHash, infoHash: decode(torrent.download).infoHash,

1
scraper/scrapers/rarbg/rarbg_scraper.js
View File

@@ -39,6 +39,7 @@ async function scrapeLatestTorrents() {
rarbg.CATEGORY['4k_X264_4k_HDR'], rarbg.CATEGORY['4k_X264_4k_HDR'],
rarbg.CATEGORY.MOVIES_XVID, rarbg.CATEGORY.MOVIES_XVID,
rarbg.CATEGORY.MOVIES_XVID_720P, rarbg.CATEGORY.MOVIES_XVID_720P,
[54], // Movies/x265/1080
rarbg.CATEGORY.MOVIES_X264, rarbg.CATEGORY.MOVIES_X264,
rarbg.CATEGORY.MOVIES_X264_1080P, rarbg.CATEGORY.MOVIES_X264_1080P,
rarbg.CATEGORY.MOVIES_X264_720P, rarbg.CATEGORY.MOVIES_X264_720P,