knightcrawler/scraper/scrapers/nyaapantsu/nyaa_pantsu_scraper.js

const moment = require('moment');
const Bottleneck = require('bottleneck');
const pantsu = require('./nyaa_pantsu_api');
const { Type } = require('../../lib/types');
const Promises = require('../../lib/promises');
const repository = require('../../lib/repository');
const { updateCurrentSeeders, updateTorrentSize } = require('../../lib/torrent');
const { createTorrentEntry, checkAndUpdateTorrent } = require('../../lib/torrentEntries');

const NAME = 'NyaaPantsu';
const UNTIL_PAGE = 5

const limiter = new Bottleneck({ maxConcurrent: 5 });

async function scrape() {
  const scrapeStart = moment();
  const lastScrape = await repository.getProvider({ name: NAME });
  console.log(`[${scrapeStart}] starting ${NAME} scrape...`);

  // const ids = ['1033095'];
  // return Promise.all(ids.map(id => limiter.schedule(() => pantsu.torrent(id)
  //     .then(torrent => processTorrentRecord(torrent)))))
  //     .then(() => console.log(`[${moment()}] finished ${NAME} scrape`));
  return scrapeLatestTorrents()
      .then(() => {
        lastScrape.lastScraped = scrapeStart;
        return lastScrape.save();
      })
      .then(() => console.log(`[${moment()}] finished ${NAME} scrape`));
}

async function updateSeeders(torrent) {
  return limiter.schedule(() => pantsu.torrent(torrent.torrentId))
      .then(foundTorrent => {
        if (Number.isInteger(foundTorrent.seeders)) {
          return [foundTorrent];
        }
        return []
      });
}

async function scrapeLatestTorrents() {
  const allowedCategories = [
    pantsu.Categories.ANIME.ENGLISH
  ];

  return Promises.sequence(allowedCategories.map(category => () => scrapeLatestTorrentsForCategory(category)))
      .then(entries => entries.reduce((a, b) => a.concat(b), []));
}

async function scrapeLatestTorrentsForCategory(category, page = 1) {
  console.log(`Scrapping ${NAME} ${category} category page ${page}`);
  return pantsu.browse(({ page }))
      .catch(error => {
        console.warn(`Failed ${NAME} scrapping for [${page}] ${category} due: `, error);
        return Promise.resolve([]);
      })
      .then(torrents => Promise.all(torrents.map(torrent => limiter.schedule(() => processTorrentRecord(torrent)
          .catch(error => {
            console.warn(`Failed processing [${torrent.infoHash}] ${torrent.title} due: `, error);
            return Promise.resolve();
          })))))
      .then(resolved => resolved.length > 0 && page < UNTIL_PAGE
          ? scrapeLatestTorrentsForCategory(category, page + 1)
          : Promise.resolve());
}

async function processTorrentRecord(record) {
  if (!record || await checkAndUpdateTorrent(record)) {
    return record;
  }

  if (!record.size) {
    await updateTorrentSize(record)
  }
  if (record.seeders === null || record.seeders === undefined) {
    await updateCurrentSeeders(record);
  }

  const torrent = {
    infoHash: record.infoHash,
    torrentLink: record.torrentLink,
    provider: NAME,
    torrentId: record.torrentId,
    title: record.title,
    type: Type.ANIME,
    size: record.size,
    seeders: record.seeders,
    uploadDate: record.uploadDate,
    languages: record.languages,
    files: record.files || undefined
  };

  return createTorrentEntry(torrent).then(() => torrent);
}

module.exports = { scrape, updateSeeders, NAME };