remove scraper module

scraper/scrapers/ondebaixa/ondebaixa_api.js

@@ -1,151 +0,0 @@
-const axios = require('axios');
-const cheerio = require("cheerio");
-const decode = require('magnet-uri');
-const { escapeHTML } = require('../../lib/metadata');
-const { getRandomUserAgent } = require('../../lib/requestHelper');
-const { isPtDubbed, sanitizePtName, sanitizePtOriginalName, sanitizePtLanguages } = require('../scraperHelper')
-
-const defaultTimeout = 30000;
-const maxSearchPage = 50
-
-const baseUrl = 'https://ondebaixa.com';
-
-const Categories = {
-  MOVIE: 'filmes',
-  TV: 'series',
-  ANIME: 'anime',
-  DESENHOS: 'desenhos'
-};
-
-function torrent(torrentId, config = {}, retries = 2) {
-  if (!torrentId || retries === 0) {
-    return Promise.reject(new Error(`Failed ${torrentId} query`));
-  }
-  const slug = encodeURIComponent(torrentId.split("/")[3]);
-  return singleRequest(`${baseUrl}/${slug}/`, config)
-      .then((body) => parseTorrentPage(body))
-      .then((torrent) => torrent.map(el => ({ torrentId: slug, ...el })))
-      .catch((err) => {
-        console.warn(`Failed OndeBaixo ${slug} request: `, err);
-        return torrent(torrentId, config, retries - 1)
-      });
-}
-
-function search(keyword, config = {}, retries = 2) {
-  if (!keyword || retries === 0) {
-    return Promise.reject(new Error(`Failed ${keyword} search`));
-  }
-  const page = config.page || 1;
-  const extendToPage = Math.min(maxSearchPage, (config.extendToPage || 1))
-
-  return singleRequest(`${baseUrl}/${keyword}/${page}/`, config)
-      .then(body => parseTableBody(body))
-      .then(torrents => torrents.length === 40 && page < extendToPage
-          ? search(keyword, { ...config, page: page + 1 }).catch(() => [])
-              .then(nextTorrents => torrents.concat(nextTorrents))
-          : torrents)
-      .catch((err) => search(keyword, config, retries - 1));
-}
-
-function browse(config = {}, retries = 2) {
-  if (retries === 0) {
-    return Promise.reject(new Error(`Failed browse request`));
-  }
-  const page = config.page || 1;
-  const category = config.category;
-  const requestUrl = category ? `${baseUrl}/${category}/${page}/` : `${baseUrl}/${page}/`;
-
-  return singleRequest(requestUrl, config)
-      .then((body) => parseTableBody(body))
-      .catch((err) => browse(config, retries - 1));
-}
-
-function singleRequest(requestUrl, config = {}) {
-  const timeout = config.timeout || defaultTimeout;
-  const options = { headers: { 'User-Agent': getRandomUserAgent() }, timeout: timeout };
-
-  return axios.get(requestUrl, options)
-      .then((response) => {
-        const body = response.data;
-        if (!body) {
-          throw new Error(`No body: ${requestUrl}`);
-        } else if (body.includes('502: Bad gateway') ||
-            body.includes('403 Forbidden')) {
-          throw new Error(`Invalid body contents: ${requestUrl}`);
-        }
-        return body;
-      })
-      .catch(error => Promise.reject(error.message || error));
-}
-
-function parseTableBody(body) {
-  return new Promise((resolve, reject) => {
-    const $ = cheerio.load(body);
-
-    if (!$) {
-      reject(new Error('Failed loading body'));
-    }
-
-    const torrents = [];
-
-    $('div.capa_larga.align-middle').each((i, element) => {
-      const row = $(element);
-      torrents.push({
-        name: row.find("a").text(),
-        torrentId: row.find("a").attr("href")
-      });
-    });
-    resolve(torrents);
-  });
-}
-
-function parseTorrentPage(body) {
-  return new Promise((resolve, reject) => {
-    const $ = cheerio.load(body);
-
-    if (!$) {
-      reject(new Error('Failed loading body'));
-    }
-    const magnets = $(`a[href^="magnet"]`)
-        .filter((i, elem) => isPtDubbed($(elem).attr('title')))
-        .map((i, elem) => $(elem).attr("href")).get();
-    const details = $('div#informacoes')
-    const category = details.find('span:contains(\'Gêneros: \')').next().html()
-    const torrents = magnets.map(magnetLink => {
-      const decodedMagnet = decode(magnetLink);
-      const name = sanitizePtName(escapeHTML(decodedMagnet.name || '').replace(/\+/g, ' '));
-      const originalTitle = details.find('span:contains(\'Título Original: \')').next().text().trim();
-      const year = details.find('span:contains(\'Ano de Lançamento: \')').next().text().trim();
-      const fallBackTitle = `${originalTitle.trim()} ${year.trim()} ${name.trim()}`;
-      return {
-        title: name.length > 5 ? name : fallBackTitle,
-        originalName: sanitizePtOriginalName(originalTitle),
-        year: year,
-        infoHash: decodedMagnet.infoHash,
-        magnetLink: magnetLink,
-        category: parseCategory(category),
-        uploadDate: new Date($('time').attr('datetime')),
-        languages: sanitizePtLanguages(details.find('span:contains(\'Idioma\')').next().text())
-      }
-    });
-    resolve(torrents.filter((x) => x));
-  });
-}
-
-function parseCategory(body) {
-  const $ = cheerio.load(body)
-  if ($("a[href*='anime']").text()) {
-    return Categories.ANIME
-  }
-  if ($("a[href*='series']").text()) {
-    return Categories.TV
-  }
-  if ($("a[href*='filmes']").text()) {
-    return Categories.MOVIE
-  }
-  if ($("a[href*='desenhos']").text()) {
-    return Categories.TV
-  }
-}
-
-module.exports = { torrent, search, browse, Categories };

scraper/scrapers/ondebaixa/ondebaixa_scraper.js

@@ -1,113 +0,0 @@
-const moment = require("moment");
-const Bottleneck = require("bottleneck");
-const ondebaixa = require("./ondebaixa_api");
-const { Type } = require("../../lib/types");
-const repository = require("../../lib/repository");
-const Promises = require("../../lib/promises");
-const { createTorrentEntry, checkAndUpdateTorrent } = require("../../lib/torrentEntries");
-const { updateCurrentSeeders, updateTorrentSize } = require("../../lib/torrent");
-const { getImdbId } = require("../../lib/metadata");
-
-const NAME = "OndeBaixa";
-const UNTIL_PAGE = 5;
-const TYPE_MAPPING = typeMapping();
-
-const limiter = new Bottleneck({ maxConcurrent: 5 });
-
-async function scrape() {
-  const scrapeStart = moment();
-  const lastScrape = await repository.getProvider({ name: NAME });
-  console.log(`[${scrapeStart}] starting ${NAME} scrape...`);
-
-  return scrapeLatestTorrents()
-      .then(() => {
-        lastScrape.lastScraped = scrapeStart;
-        return lastScrape.save();
-      })
-      .then(() => console.log(`[${moment()}] finished ${NAME} scrape`));
-}
-
-async function updateSeeders(torrent) {
-  return limiter.schedule(() => ondebaixa.torrent(torrent.torrentId));
-}
-
-async function scrapeLatestTorrents() {
-  const allowedCategories = [
-    ondebaixa.Categories.MOVIE,
-    ondebaixa.Categories.TV,
-    ondebaixa.Categories.DESENHOS
-  ];
-
-  return Promises.sequence(
-      allowedCategories.map(
-          (category) => () => scrapeLatestTorrentsForCategory(category)
-      )
-  ).then((entries) => entries.reduce((a, b) => a.concat(b), []));
-}
-
-async function scrapeLatestTorrentsForCategory(category, page = 1) {
-  console.log(`Scrapping ${NAME} ${category} category page ${page}`);
-  return ondebaixa
-      .browse({ category, page })
-      .catch((error) => {
-        console.warn(`Failed ${NAME} scrapping for [${page}] ${category} due: `, error);
-        return Promise.resolve([]);
-      })
-      .then((torrents) => Promise.all(torrents.map((torrent) => limiter.schedule(() => processEntry(torrent)))))
-      .then((resolved) => resolved.length > 0 && page < untilPage(category)
-          ? scrapeLatestTorrentsForCategory(category, page + 1)
-          : Promise.resolve());
-}
-
-async function processEntry(entry) {
-  return ondebaixa.torrent(entry.torrentId)
-      .then(records => Promises.sequence(records.map(record => () => processTorrentRecord(record))))
-      .catch(() => undefined);
-}
-
-async function processTorrentRecord(foundTorrent) {
-  if (await checkAndUpdateTorrent({ provider: NAME, ...foundTorrent })) {
-    return foundTorrent;
-  }
-
-  if (!foundTorrent.size) {
-    await updateTorrentSize(foundTorrent);
-  }
-  if (!Number.isInteger(foundTorrent.seeders)) {
-    await updateCurrentSeeders(foundTorrent);
-  }
-  if (!foundTorrent.imdbId && TYPE_MAPPING[foundTorrent.category] !== Type.ANIME) {
-    const info = { title: foundTorrent.originalName, year: foundTorrent.year };
-    foundTorrent.imdbId = await getImdbId(info, TYPE_MAPPING[foundTorrent.category]).catch(() => undefined);
-  }
-
-  const torrent = {
-    infoHash: foundTorrent.infoHash,
-    provider: NAME,
-    torrentId: foundTorrent.torrentId,
-    title: foundTorrent.title,
-    type: TYPE_MAPPING[foundTorrent.category],
-    imdbId: foundTorrent.imdbId,
-    uploadDate: foundTorrent.uploadDate,
-    seeders: foundTorrent.seeders,
-    size: foundTorrent.size,
-    files: foundTorrent.files,
-    languages: foundTorrent.languages
-  };
-  return createTorrentEntry(torrent);
-}
-
-function typeMapping() {
-  const mapping = {};
-  mapping[ondebaixa.Categories.MOVIE] = Type.MOVIE;
-  mapping[ondebaixa.Categories.TV] = Type.SERIES;
-  mapping[ondebaixa.Categories.DESENHOS] = Type.SERIES;
-  mapping[ondebaixa.Categories.ANIME] = Type.ANIME;
-  return mapping;
-}
-
-function untilPage(category) {
-  return UNTIL_PAGE;
-}
-
-module.exports = { scrape, updateSeeders, NAME };