knightcrawler/scraper/scrapers/torrentgalaxy/torrentgalaxy_scraper.js

const moment = require('moment');
const Bottleneck = require('bottleneck');
const torrentGalaxy = require('./torrentgalaxy_api');
const { Type } = require('../../lib/types');
const repository = require('../../lib/repository');
const Promises = require('../../lib/promises');
const { createTorrentEntry, checkAndUpdateTorrent } = require('../../lib/torrentEntries');

const NAME = 'TorrentGalaxy';
const TYPE_MAPPING = typeMapping();

const api_limiter = new Bottleneck({ maxConcurrent: 1, minTime: 5000 });
const limiter = new Bottleneck({ maxConcurrent: 10 });
const allowedCategories = [
  torrentGalaxy.Categories.ANIME,
  torrentGalaxy.Categories.MOVIE_4K,
  torrentGalaxy.Categories.MOVIE_PACKS,
  torrentGalaxy.Categories.MOVIE_SD,
  torrentGalaxy.Categories.MOVIE_HD,
  torrentGalaxy.Categories.MOVIE_CAM,
  torrentGalaxy.Categories.MOVIE_BOLLYWOOD,
  torrentGalaxy.Categories.TV_SD,
  torrentGalaxy.Categories.TV_HD,
  torrentGalaxy.Categories.TV_PACKS,
  torrentGalaxy.Categories.DOCUMENTARIES,
];
const packCategories = [
  torrentGalaxy.Categories.MOVIE_PACKS,
  torrentGalaxy.Categories.TV_PACKS
];

async function scrape() {
  const scrapeStart = moment();
  const lastScrape = await repository.getProvider({ name: NAME });
  console.log(`[${scrapeStart}] starting ${NAME} scrape...`);

  // const ids = ['14212584'];
  // return Promise.all(ids.map(id => limiter.schedule(() => torrentGalaxy.torrent(id)
  //     .then(torrent => processTorrentRecord(torrent)))))
  //     .then(() => console.log(`[${moment()}] finished ${NAME} scrape`));
  return scrapeLatestTorrents()
      .then(() => {
        lastScrape.lastScraped = scrapeStart;
        return lastScrape.save();
      })
      .then(() => console.log(`[${moment()}] finished ${NAME} scrape`));
}

async function updateSeeders(torrent) {
  return limiter.schedule(() => torrentGalaxy.torrent(torrent.torrentId));
}

async function scrapeLatestTorrents() {
  return Promises.sequence(allowedCategories.map(category => () => scrapeLatestTorrentsForCategory(category)))
      .then(entries => entries.reduce((a, b) => a.concat(b), []));
}

async function scrapeLatestTorrentsForCategory(category, page = 1) {
  console.log(`Scrapping ${NAME} ${category} category page ${page}`);
  return api_limiter.schedule(() => torrentGalaxy.browse({ category, page }))
      .catch(error => {
        console.warn(`Failed ${NAME} scrapping for [${page}] ${category} due: `, error);
        return Promise.resolve([]);
      })
      .then(torrents => Promise.all(torrents.map(torrent => limiter.schedule(() => processTorrentRecord(torrent)))))
      .then(resolved => resolved.length > 0 && page < getMaxPage(category)
          ? scrapeLatestTorrentsForCategory(category, page + 1)
          : Promise.resolve());
}

async function processTorrentRecord(record) {
  if (!record || !TYPE_MAPPING[record.category] || !record.verified) {
    return Promise.resolve('Invalid torrent record');
  }

  const torrent = {
    provider: NAME,
    infoHash: record.infoHash,
    torrentId: record.torrentId,
    torrentLink: record.torrentLink,
    title: record.name.replace(/\t|\s+/g, ' '),
    type: TYPE_MAPPING[record.category],
    size: record.size,
    seeders: record.seeders,
    uploadDate: record.uploadDate,
    imdbId: record.imdbId,
    pack: packCategories.includes(record.category),
    languages: !(record.languages || '').includes('Other') ? record.languages : undefined
  };

  if (await checkAndUpdateTorrent(torrent)) {
    return torrent;
  }
  const isOld = moment(torrent.uploadDate).isBefore(moment().subtract(18, 'month'));
  if (torrent.seeders === 0 && isOld && !torrent.pack) {
    console.log(`Skipping old unseeded torrent [${torrent.infoHash}] ${torrent.title}`)
    return torrent;
  }

  return createTorrentEntry(torrent).then(() => torrent);
}

function typeMapping() {
  const mapping = {};
  mapping[torrentGalaxy.Categories.MOVIE_SD] = Type.MOVIE;
  mapping[torrentGalaxy.Categories.MOVIE_HD] = Type.MOVIE;
  mapping[torrentGalaxy.Categories.MOVIE_4K] = Type.MOVIE;
  mapping[torrentGalaxy.Categories.MOVIE_CAM] = Type.MOVIE;
  mapping[torrentGalaxy.Categories.MOVIE_PACKS] = Type.MOVIE;
  mapping[torrentGalaxy.Categories.MOVIE_BOLLYWOOD] = Type.MOVIE;
  mapping[torrentGalaxy.Categories.DOCUMENTARIES] = Type.MOVIE;
  mapping[torrentGalaxy.Categories.TV_SD] = Type.SERIES;
  mapping[torrentGalaxy.Categories.TV_HD] = Type.SERIES;
  mapping[torrentGalaxy.Categories.TV_PACKS] = Type.SERIES;
  mapping[torrentGalaxy.Categories.TV_SPORT] = Type.SERIES;
  mapping[torrentGalaxy.Categories.ANIME] = Type.ANIME;
  return mapping;
}

function getMaxPage(category) {
  switch (category) {
    case torrentGalaxy.Categories.TV_SD:
    case torrentGalaxy.Categories.TV_HD:
    case torrentGalaxy.Categories.MOVIE_SD:
    case torrentGalaxy.Categories.MOVIE_HD:
      return 5;
    default:
      return 1;
  }
}

module.exports = { scrape, updateSeeders, NAME };