const moment = require('moment'); const Bottleneck = require('bottleneck'); const { parse } = require('parse-torrent-title'); const rutor = require('./rutor_api'); const { Type } = require('../../lib/types'); const repository = require('../../lib/repository'); const Promises = require('../../lib/promises'); const { createTorrentEntry, checkAndUpdateTorrent } = require('../../lib/torrentEntries'); const NAME = 'Rutor'; const TYPE_MAPPING = { 'kino': Type.MOVIE, 'nashe_kino': Type.MOVIE, 'nauchno_popularnoe': Type.MOVIE, 'inostrannoe': Type.MOVIE, 'seriali': Type.SERIES, 'nashi_seriali': Type.SERIES, 'tv': Type.SERIES, 'multiki': Type.MOVIE, 'anime': Type.ANIME }; const api_limiter = new Bottleneck({ maxConcurrent: 1, minTime: 5000 }); const api_entry_limiter = new Bottleneck({ maxConcurrent: 1, minTime: 2500 }); const limiter = new Bottleneck({ maxConcurrent: 10 }); const allowedCategories = [ rutor.Categories.FOREIGN_FILMS, rutor.Categories.FOREIGN_RELEASES, rutor.Categories.RUSSIAN_FILMS, rutor.Categories.FOREIGN_SERIES, rutor.Categories.RUSSIAN_SERIES, rutor.Categories.SCIENCE_FILMS, rutor.Categories.RUSSIAN_ANIMATION, rutor.Categories.ANIME ]; async function scrape() { const scrapeStart = moment(); const lastScrape = await repository.getProvider({ name: NAME }); console.log(`[${scrapeStart}] starting ${NAME} scrape...`); // const ids = [ // '637799' // ]; // return Promise.all(ids.map(id => api_entry_limiter.schedule(() => rutor.torrent(id)) // .then(torrent => processTorrentRecord(torrent)))) // .then(() => console.log(`[${moment()}] finished ${NAME} scrape`)); return scrapeLatestTorrents() .then(() => { lastScrape.lastScraped = scrapeStart; return lastScrape.save(); }) .then(() => console.log(`[${moment()}] finished ${NAME} scrape`)); } async function updateSeeders(torrent) { return Promise.resolve([]); } async function scrapeLatestTorrents() { return Promises.sequence(allowedCategories.map(category => () => scrapeLatestTorrentsForCategory(category))) .then(entries => entries.reduce((a, b) => a.concat(b), [])); } async function scrapeLatestTorrentsForCategory(category, page = 1) { console.log(`Scrapping ${NAME} ${category} category page ${page}`); return api_limiter.schedule(() => rutor.browse({ category, page })) .catch(error => { console.warn(`Failed ${NAME} scrapping for [${page}] ${category} due: `, error); return Promise.resolve([]); }) .then(torrents => Promise.all(torrents.map(torrent => limiter.schedule(() => processTorrentRecord(torrent))))) .then(resolved => resolved.length > 0 && page < getMaxPage(category) ? scrapeLatestTorrentsForCategory(category, page + 1) : Promise.resolve()); } async function processTorrentRecord(record) { if (await checkAndUpdateTorrent(record)) { return record; } const isOld = moment(record.uploadDate).isBefore(moment().subtract(18, 'month')); if (record.seeders === 0 && isOld) { console.log(`Skipping old unseeded torrent [${record.infoHash}] ${record.title}`) return record; } const foundTorrent = await api_entry_limiter.schedule(() => rutor.torrent(record.torrentId).catch(() => undefined)); if (!foundTorrent || !TYPE_MAPPING[foundTorrent.category]) { return Promise.resolve(`${NAME}: Invalid torrent record: ${record.torrentId}`); } if (!foundTorrent.imdbId && disallowWithoutImdbId(foundTorrent)) { return Promise.resolve(`${NAME}: No imdbId defined: ${record.torrentId}`); } const torrent = { provider: NAME, infoHash: foundTorrent.infoHash, torrentId: foundTorrent.torrentId, torrentLink: foundTorrent.torrentLink, trackers: foundTorrent.trackers, title: foundTorrent.title, type: TYPE_MAPPING[foundTorrent.category], size: foundTorrent.size, seeders: foundTorrent.seeders, uploadDate: foundTorrent.uploadDate, imdbId: foundTorrent.imdbId, languages: foundTorrent.languages || undefined, }; return createTorrentEntry(torrent).then(() => torrent); } function getMaxPage(category) { switch (category) { case rutor.Categories.FOREIGN_FILMS: case rutor.Categories.FOREIGN_SERIES: return 2; default: return 1; } } function disallowWithoutImdbId(torrent) { if (['kino', 'anime'].includes(torrent.category)) { return false; // allow to search foreign movie and anime ids via search } // allow to search id for non russian series titles via search return !(torrent.category === 'seriali' && !parse(torrent.title).title.match(/[\u0400-\u04ff]/i)); } module.exports = { scrape, updateSeeders, NAME };