76 lines
2.3 KiB
JavaScript
76 lines
2.3 KiB
JavaScript
const moment = require('moment');
|
|
const Bottleneck = require('bottleneck');
|
|
const rarbg = require('rarbg-api');
|
|
const decode = require('magnet-uri');
|
|
const { Type } = require('../../lib/types');
|
|
const {
|
|
createTorrentEntry,
|
|
getStoredTorrentEntry,
|
|
updateTorrentSeeders
|
|
} = require('../../lib/torrentEntries');
|
|
|
|
const NAME = 'RARBG';
|
|
|
|
const limiter = new Bottleneck({ maxConcurrent: 1, minTime: 2500 });
|
|
const entryLimiter = new Bottleneck({ maxConcurrent: 40 });
|
|
|
|
async function scrape() {
|
|
console.log(`[${moment()}] starting ${NAME} dump scrape...`);
|
|
const movieImdbIds = require('./rargb_movie_imdb_ids_2020-03-09.json');
|
|
const seriesImdbIds = require('./rargb_series_imdb_ids_2020-03-09.json');
|
|
const allImdbIds = [].concat(movieImdbIds).concat(seriesImdbIds);
|
|
|
|
return Promise.all(allImdbIds.map(imdbId => limiter.schedule(() => getTorrentsForImdbId(imdbId))
|
|
.then(torrents => Promise.all(torrents.map(t => entryLimiter.schedule(() => processTorrentRecord(t)))))))
|
|
.then(() => console.log(`[${moment()}] finished ${NAME} dump scrape`));
|
|
}
|
|
|
|
async function getTorrentsForImdbId(imdbId) {
|
|
return rarbg.search(imdbId, { limit: 100, sort: 'seeders', format: 'json_extended', ranked: 0 }, 'imdb')
|
|
.then(torrents => torrents.map(torrent => ({
|
|
name: torrent.title,
|
|
infoHash: decode(torrent.download).infoHash,
|
|
magnetLink: torrent.download,
|
|
seeders: torrent.seeders,
|
|
leechers: torrent.leechers,
|
|
category: torrent.category,
|
|
size: torrent.size,
|
|
uploadDate: new Date(torrent.pubdate),
|
|
imdbId: torrent.episode_info && torrent.episode_info.imdb
|
|
})))
|
|
.catch((err) => []);
|
|
}
|
|
|
|
async function processTorrentRecord(record) {
|
|
if (await getStoredTorrentEntry(record)) {
|
|
return updateTorrentSeeders(record);
|
|
}
|
|
|
|
const torrent = {
|
|
provider: NAME,
|
|
infoHash: record.infoHash,
|
|
title: record.name,
|
|
type: getType(record.category),
|
|
seeders: record.seeders,
|
|
size: record.size,
|
|
uploadDate: record.uploadDate,
|
|
imdbId: record.imdbId
|
|
};
|
|
|
|
return createTorrentEntry(torrent);
|
|
}
|
|
|
|
const seriesCategories = [
|
|
'TV Episodes',
|
|
'Movies/TV-UHD-episodes',
|
|
'TV HD Episodes',
|
|
];
|
|
|
|
function getType(category) {
|
|
if (seriesCategories.includes(category)) {
|
|
return Type.SERIES;
|
|
}
|
|
return Type.MOVIE;
|
|
}
|
|
|
|
module.exports = { scrape }; |