moves scraper to a subpackage

scraper/scrapers/rarbg/rarbg_dump_scraper.js

@@ -0,0 +1,76 @@
+const moment = require('moment');
+const Bottleneck = require('bottleneck');
+const rarbg = require('rarbg-api');
+const decode = require('magnet-uri');
+const { Type } = require('../../lib/types');
+const {
+  createTorrentEntry,
+  getStoredTorrentEntry,
+  updateTorrentSeeders
+} = require('../../lib/torrentEntries');
+
+const NAME = 'RARBG';
+
+const limiter = new Bottleneck({ maxConcurrent: 1, minTime: 2500 });
+const entryLimiter = new Bottleneck({ maxConcurrent: 40 });
+
+async function scrape() {
+  console.log(`[${moment()}] starting ${NAME} dump scrape...`);
+  const movieImdbIds = require('./rargb_movie_imdb_ids_2020-03-09.json');
+  const seriesImdbIds = require('./rargb_series_imdb_ids_2020-03-09.json');
+  const allImdbIds = [].concat(movieImdbIds).concat(seriesImdbIds);
+
+  return Promise.all(allImdbIds.map(imdbId => limiter.schedule(() => getTorrentsForImdbId(imdbId))
+      .then(torrents => Promise.all(torrents.map(t => entryLimiter.schedule(() => processTorrentRecord(t)))))))
+      .then(() => console.log(`[${moment()}] finished ${NAME} dump scrape`));
+}
+
+async function getTorrentsForImdbId(imdbId) {
+  return rarbg.search(imdbId, { limit: 100, sort: 'seeders', format: 'json_extended', ranked: 0 }, 'imdb')
+      .then(torrents => torrents.map(torrent => ({
+        name: torrent.title,
+        infoHash: decode(torrent.download).infoHash,
+        magnetLink: torrent.download,
+        seeders: torrent.seeders,
+        leechers: torrent.leechers,
+        category: torrent.category,
+        size: torrent.size,
+        uploadDate: new Date(torrent.pubdate),
+        imdbId: torrent.episode_info && torrent.episode_info.imdb
+      })))
+      .catch((err) => []);
+}
+
+async function processTorrentRecord(record) {
+  if (await getStoredTorrentEntry(record)) {
+    return updateTorrentSeeders(record);
+  }
+
+  const torrent = {
+    provider: NAME,
+    infoHash: record.infoHash,
+    title: record.name,
+    type: getType(record.category),
+    seeders: record.seeders,
+    size: record.size,
+    uploadDate: record.uploadDate,
+    imdbId: record.imdbId
+  };
+
+  return createTorrentEntry(torrent);
+}
+
+const seriesCategories = [
+  'TV Episodes',
+  'Movies/TV-UHD-episodes',
+  'TV HD Episodes',
+];
+
+function getType(category) {
+  if (seriesCategories.includes(category)) {
+    return Type.SERIES;
+  }
+  return Type.MOVIE;
+}
+
+module.exports = { scrape };