diff --git a/scraper/index.js b/scraper/index.js index c726b13..d75eb1d 100644 --- a/scraper/index.js +++ b/scraper/index.js @@ -6,6 +6,7 @@ const { connect, getUpdateSeedersTorrents } = require('./lib/repository'); const thepiratebayScraper = require('./scrapers/thepiratebay/thepiratebay_scraper'); const horribleSubsScraper = require('./scrapers/horriblesubs/horriblesubs_scraper'); const ytsScraper = require('./scrapers/yts/yts_scraper'); +const eztvScraper = require('./scrapers/eztv/eztv_scraper'); const leetxScraper = require('./scrapers/1337x/1337x_scraper'); const kickassScraper = require('./scrapers/kickass/kickass_scraper'); const rarbgScraper = require('./scrapers/rarbg/rarbg_scraper'); @@ -17,6 +18,7 @@ const PROVIDERS = [ // require('./scrapers/thepiratebay/thepiratebay_update_size_scraper') // require('./scrapers/1337x/1337x_dump_scraper') ytsScraper, + eztvScraper, horribleSubsScraper, rarbgScraper, thepiratebayScraper, diff --git a/scraper/scrapers/eztv/eztv_api.js b/scraper/scrapers/eztv/eztv_api.js new file mode 100644 index 0000000..e561770 --- /dev/null +++ b/scraper/scrapers/eztv/eztv_api.js @@ -0,0 +1,128 @@ +const cheerio = require('cheerio'); +const needle = require('needle'); +const moment = require('moment'); +const Promises = require('../../lib/promises'); +const { getRandomUserAgent } = require('./../../lib/request_helper'); + +const defaultProxies = [ + 'https://eztv.io' +]; +const defaultTimeout = 30000; +const limit = 100; +const maxPage = 5; + +function torrent(torrentId, config = {}, retries = 2) { + if (!torrentId || retries === 0) { + return Promise.reject(new Error(`Failed ${torrentId} search`)); + } + + return Promises.first(defaultProxies + .map(proxyUrl => singleRequest(`${proxyUrl}/ep/${torrentId}`, config))) + .then(body => parseTorrentPage(body)) + .then(torrent => ({ torrentId, ...torrent })) + .catch(error => torrent(torrentId, config, retries - 1)); +} + +function search(imdbId, config = {}, retries = 2) { + if (!imdbId || retries === 0) { + return Promise.reject(new Error(`Failed ${imdbId} search`)); + } + const id = imdbId.replace('tt', ''); + const page = config.page || 1; + + return Promises.first(defaultProxies + .map(proxyUrl => singleRequest(`${proxyUrl}/api/get-torrents?limit=${limit}&page=${page}&imdb_id=${id}`, config))) + .then(results => parseResults(results)) + .then(torrents => torrents.length === limit && page < maxPage + ? search(imdbId, { ...config, page: page + 1 }) + .catch(() => []) + .then(nextTorrents => torrents.concat(nextTorrents)) + : torrents) + .catch(error => search(imdbId, config, retries - 1)); +} + +function browse(config = {}, retries = 2) { + if (retries === 0) { + return Promise.reject(new Error(`Failed browse request`)); + } + const page = config.page || 1; + + return Promises.first(defaultProxies + .map(proxyUrl => singleRequest(`${proxyUrl}/api/get-torrents?limit=${limit}&page=${page}`, config))) + .then(results => parseResults(results)) + .catch(error => browse(config, retries - 1)); +} + +function singleRequest(requestUrl, config = {}) { + const timeout = config.timeout || defaultTimeout; + const options = { userAgent: getRandomUserAgent(), open_timeout: timeout, follow: 2 }; + + return needle('get', requestUrl, options) + .then(response => { + if (!response.body) { + return Promise.reject(`No body: ${requestUrl}`); + } + return Promise.resolve(response.body); + }); +} + +function parseResults(results) { + if (!results || !Array.isArray(results.torrents)) { + console.log('Incorrect results: ', results); + return Promise.reject('Incorrect results') + } + return results.torrents.map(torrent => parseTorrent(torrent)); +} + +function parseTorrent(torrent) { + return { + name: torrent.title.replace(/EZTV$/, ''), + torrentId: torrent.episode_url.replace(/.*\/ep\//, ''), + infoHash: torrent.hash.trim().toLowerCase(), + magnetLink: torrent.magnet_url, + torrentLink: torrent.torrent_url, + seeders: torrent.seeds, + size: torrent.size_bytes, + uploadDate: new Date(torrent.date_released_unix * 1000), + imdbId: torrent.imdb_id !== '0' && 'tt' + torrent.imdb_id || undefined + } +} + +function parseTorrentPage(body) { + return new Promise((resolve, reject) => { + const $ = cheerio.load(body); + + if (!$) { + reject(new Error('Failed loading body')); + } + const content = $('table[class="forum_header_border_normal"]'); + const torrent = { + name: content.find('h1 > span').text().replace(/EZTV$/, ''), + infoHash: content.find('b:contains(\'Torrent Hash:\')')[0].nextSibling.data.trim().toLowerCase(), + magnetLink: content.find('a[title="Magnet Link"]').attr('href'), + torrentLink: content.find('a[title="Download Torrent"]').attr('href'), + seeders: parseInt(content.find('span[class="stat_red"]').first().text(), 10) || 0, + size: parseSize(content.find('b:contains(\'Filesize:\')')[0].nextSibling.data), + uploadDate: moment(content.find('b:contains(\'Released:\')')[0].nextSibling.data, 'Do MMM YYYY').toDate(), + showUrl: content.find('.episode_left_column a').attr('href') + }; + resolve(torrent); + }); +} + +function parseSize(sizeText) { + if (!sizeText) { + return undefined; + } + let scale = 1; + if (sizeText.includes('GB')) { + scale = 1024 * 1024 * 1024 + } else if (sizeText.includes('MB')) { + scale = 1024 * 1024; + } else if (sizeText.includes('KB') || sizeText.includes('kB')) { + scale = 1024; + } + return Math.floor(parseFloat(sizeText.replace(/[',]/g, '')) * scale); +} + +module.exports = { torrent, search, browse }; diff --git a/scraper/scrapers/eztv/eztv_scraper.js b/scraper/scrapers/eztv/eztv_scraper.js new file mode 100644 index 0000000..98092ec --- /dev/null +++ b/scraper/scrapers/eztv/eztv_scraper.js @@ -0,0 +1,78 @@ +const moment = require('moment'); +const Bottleneck = require('bottleneck'); +const eztv = require('./eztv_api'); +const { Type } = require('../../lib/types'); +const Promises = require('../../lib/promises'); +const repository = require('../../lib/repository'); + +const { updateCurrentSeeders } = require('../../lib/torrent'); +const { createTorrentEntry, getStoredTorrentEntry, updateTorrentSeeders } = require('../../lib/torrentEntries'); + +const NAME = 'EZTV'; +const UNTIL_PAGE = 10; + +const limiter = new Bottleneck({ maxConcurrent: 20 }); + +async function scrape() { + const scrapeStart = moment(); + const lastScrape = await repository.getProvider({ name: NAME }); + console.log(`[${scrapeStart}] starting ${NAME} scrape...`); + + return scrapeLatestTorrents() + .then(() => { + lastScrape.lastScraped = scrapeStart; + return lastScrape.save(); + }) + .then(() => console.log(`[${moment()}] finished ${NAME} scrape`)); +} + +async function updateSeeders(torrent) { + return limiter.schedule(() => eztv.torrent(torrent.torrentId) + .then(record => (torrent.seeders = record.seeders, torrent)) + .catch(() => updateCurrentSeeders(torrent)) + .then(updated => updateTorrentSeeders(updated))); +} + +async function scrapeLatestTorrents() { + return scrapeLatestTorrentsForCategory(1654); +} + +async function scrapeLatestTorrentsForCategory(page = 1) { + console.log(`Scrapping ${NAME} page ${page}`); + return eztv.browse(({ page })) + .catch(error => { + console.warn(`Failed ${NAME} scrapping for [${page}] due: `, error); + // return Promises.delay(30000).then(() => scrapeLatestTorrentsForCategory(page)) + return Promise.resolve([]); + }) + .then(torrents => Promise.all(torrents.map(torrent => limiter.schedule(() => processTorrentRecord(torrent))))) + .then(resolved => resolved.length > 0 && page < UNTIL_PAGE + ? scrapeLatestTorrentsForCategory(page + 1) + : Promise.resolve()); +} + +async function processTorrentRecord(record) { + if (await getStoredTorrentEntry(record)) { + return updateTorrentSeeders(record); + } + + if (!record || !record.size) { + return Promise.resolve('Invalid torrent record'); + } + + const torrent = { + infoHash: record.infoHash, + provider: NAME, + torrentId: record.torrentId, + title: record.name.replace(/\t|\s+/g, ' ').trim(), + type: Type.SERIES, + size: record.size, + seeders: record.seeders, + uploadDate: record.uploadDate, + imdbId: record.imdbId, + }; + + return createTorrentEntry(torrent).then(() => torrent); +} + +module.exports = { scrape, updateSeeders, NAME }; \ No newline at end of file