format ondebaixa provider

scraper/scrapers/ondebaixa/ondebaixa_api.js

@@ -3,7 +3,8 @@ const cheerio = require("cheerio");
 const decode = require('magnet-uri');
 const Promises = require('../../lib/promises');
 const { escapeHTML } = require('../../lib/metadata');
-const { getRandomUserAgent } = require("../../lib/requestHelper");
+const { getRandomUserAgent } = require('../../lib/requestHelper');
+const { isPtDubbed, sanitizePtName, sanitizePtOriginalName, sanitizePtLanguages } = require('../scraperHelper')
 
 const defaultTimeout = 10000;
 const maxSearchPage = 50
@@ -121,15 +122,16 @@ function parseTorrentPage(body) {
     const isAnime = parseCategory(category) === Categories.ANIME
     const torrent = magnets.map(magnetLink => {
       const name = escapeHTML(decode(magnetLink).name.replace(/\+/g, ' '))
-      if(isDubled(name) || isAnime) {
+      if (isPtDubbed(name) || isAnime) {
         return {
-          name: parseText(name),
+          title: sanitizePtName(name),
-          original_name: parseName(details.find('span:contains(\'Título Original: \')').next().text()),
+          originalName: sanitizePtOriginalName(details.find('span:contains(\'Título Original: \')').next().text()),
           year: details.find('span:contains(\'Ano de Lançamento: \')').next().text().trim(),
           infoHash: decode(magnetLink).infoHash,
           magnetLink: magnetLink,
           category: parseCategory(category),
           uploadDate: new Date($('time').attr('datetime')),
+          languages: sanitizePtLanguages(details.find('span:contains(\'Idioma\')')[0].nextSibling.nodeValue)
         };
       }
     })
@@ -137,57 +139,20 @@ function parseTorrentPage(body) {
   });
 }
 
-function parseName(name) {
-  return name
-  .replace(/S01|S02|S03|S04|S05|S06|S07|S08|S09/g, '')
-  .trim()
-}
-
-function isDubled(name){
-  name = name.toLowerCase()
-  if(name.includes('dublado')){
-    return true
-  }
-  if(name.includes('dual')){
-    return true
-  }
-  if(name.includes('nacional')){
-    return true
-  }
-  return false
-}
-
 function parseCategory(body) {
   const $ = cheerio.load(body)
-  const isAnime = $("a[href*='anime']").text()
+  if ($("a[href*='anime']").text()) {
-  const isSerie = $("a[href*='series']").text()
-  const isMovie = $("a[href*='filmes']").text()
-  const isDesenho = $("a[href*='desenhos']").text()
-  if(isAnime) {
     return Categories.ANIME
   }
-  if(isSerie) {
+  if ($("a[href*='series']").text()) {
     return Categories.TV
   }
-  if(isMovie) {
+  if ($("a[href*='filmes']").text()) {
     return Categories.MOVIE
   }
-  if(isDesenho) {
+  if ($("a[href*='desenhos']").text()) {
     return Categories.TV
   }
 }
 
-function parseText(text) {
-  return text
-  .replace(/\n|\t/g, "")
-  .replace(/1A|2A|3A|4A|5A|6A|7A|8A|9A/g, '')
-  .replace(/COMOEUBAIXO.COM|COMANDO.TO|TEMPORADA|COMPLETA/g, '')
-  .replace(/MKV|MP4/g, '')
-  .replace(/[-]/g, '')
-  .replace(/[.]/g, ' ')
-  .trim()
-  .replace(/ /g, '.')
-  .trim()
-}
-
 module.exports = { torrent, search, browse, Categories };

scraper/scrapers/ondebaixa/ondebaixa_scraper.js

@@ -1,6 +1,6 @@
 const moment = require("moment");
 const Bottleneck = require("bottleneck");
-const leetx = require("./ondebaixa_api");
+const ondebaixa = require("./ondebaixa_api");
 const { Type } = require("../../lib/types");
 const repository = require("../../lib/repository");
 const Promises = require("../../lib/promises");
@@ -28,14 +28,14 @@ async function scrape() {
 }
 
 async function updateSeeders(torrent) {
-  return limiter.schedule(() => leetx.torrent(torrent.torrentId));
+  return limiter.schedule(() => ondebaixa.torrent(torrent.torrentId));
 }
 
 async function scrapeLatestTorrents() {
   const allowedCategories = [
-    leetx.Categories.MOVIE,
+    ondebaixa.Categories.MOVIE,
-    leetx.Categories.TV,
+    ondebaixa.Categories.TV,
-    leetx.Categories.DESENHOS
+    ondebaixa.Categories.DESENHOS
   ];
 
   return Promises.sequence(
@@ -46,88 +46,71 @@ async function scrapeLatestTorrents() {
 }
 
 async function scrapeLatestTorrentsForCategory(category, page = 1) {
-  console.log({Scraper: `Scrapping ${NAME} ${category} category page ${page}`});
+  console.log(`Scrapping ${NAME} ${category} category page ${page}`);
-  return leetx
+  return ondebaixa
       .browse({ category, page })
       .catch((error) => {
-      console.warn(
+        console.warn(`Failed ${NAME} scrapping for [${page}] ${category} due: `, error);
-        `Failed ${NAME} scrapping for [${page}] ${category} due: `,
-        error
-      );
         return Promise.resolve([]);
       })
-    .then((torrents) => Promise.all(torrents.map((torrent) => limiter.schedule(() => processTorrentRecord(torrent)))))
+      .then((torrents) => Promise.all(torrents.map((torrent) => limiter.schedule(() => processEntry(torrent)))))
-    .then((resolved) => resolved.length > 0 && page < untilPage(category) ? scrapeLatestTorrentsForCategory(category, page + 1) : Promise.resolve());
+      .then((resolved) => resolved.length > 0 && page < untilPage(category)
+          ? scrapeLatestTorrentsForCategory(category, page + 1)
+          : Promise.resolve());
 }
 
-async function processTorrentRecord(record) {
+async function processEntry(entry) {
-  if (await checkAndUpdateTorrent({ provider: NAME, ...record })) {
+  return ondebaixa.torrent(entry.torrentId)
-    return record;
+      .then(records => Promises.sequence(records.map(record => () => processTorrentRecord(record))))
-  }
-  const torrentEntrys = await leetx
-    .torrent(record.torrentId)
       .catch(() => undefined);
-  if (torrentEntrys === undefined) {
+}
-    return Promise.resolve([])
+
+async function processTorrentRecord(foundTorrent) {
+  if (await checkAndUpdateTorrent({ provider: NAME, ...foundTorrent })) {
+    return foundTorrent;
   }
-  return await Promise.allSettled(
+
-    torrentEntrys.map(async (torrentFound) => {
+  if (!foundTorrent.size) {
-      if (!torrentFound || !TYPE_MAPPING[torrentFound.category]) {
+    await updateTorrentSize(foundTorrent);
-        return Promise.resolve("Invalid torrent record");
   }
-      if (isNaN(torrentFound.uploadDate)) {
+  if (!Number.isInteger(foundTorrent.seeders)) {
-        console.warn(
+    await updateCurrentSeeders(foundTorrent);
-          `Incorrect upload date for [${torrentFound.infoHash}] ${torrentFound.name}`
-        );
-        return;
   }
-      if (await checkAndUpdateTorrent(torrentFound)) {
+  if (!foundTorrent.imdbId && TYPE_MAPPING[foundTorrent.category] !== Type.ANIME) {
-        return torrentFound;
+    const info = { title: foundTorrent.originalName, year: foundTorrent.year };
-      }
+    foundTorrent.imdbId = await getImdbId(info, TYPE_MAPPING[foundTorrent.category]).catch(() => undefined);
-      if (!torrentFound.size) {
-        await updateTorrentSize(torrentFound)
-        .catch((err) => Promise.resolve(err))
-      }
-      if (!torrentFound.seeders) {
-        await updateCurrentSeeders(torrentFound)
-        .then(response => response.seeders === 0 ? delete response.seeders : response)
-      }
-      if (!torrentFound.imdbId) {
-        torrentFound.imdbId = await getImdbId(torrentFound.original_name, torrentFound.year, TYPE_MAPPING[torrentFound.category])
   }
 
   const torrent = {
-        infoHash: torrentFound.infoHash,
+    infoHash: foundTorrent.infoHash,
     provider: NAME,
-        torrentId: torrentFound.torrentId,
+    torrentId: foundTorrent.torrentId,
-        name: torrentFound.original_name,
+    title: foundTorrent.title,
-        title: torrentFound.name.replace(/\t|\s+/g, " ").trim(),
+    type: TYPE_MAPPING[foundTorrent.category],
-        type: TYPE_MAPPING[torrentFound.category],
+    imdbId: foundTorrent.imdbId,
-        year: torrentFound.year,
+    uploadDate: foundTorrent.uploadDate,
-        imdbId: torrentFound.imdbId,
+    seeders: foundTorrent.seeders,
-        uploadDate: torrentFound.uploadDate,
+    size: foundTorrent.size,
-        seeders: torrentFound.seeders,
+    files: foundTorrent.files,
-        size: torrentFound.size,
+    languages: foundTorrent.languages
-        files: torrentFound.files
   };
   return createTorrentEntry(torrent);
-    })
-  );
 }
 
 function typeMapping() {
   const mapping = {};
-  mapping[leetx.Categories.MOVIE] = Type.MOVIE;
+  mapping[ondebaixa.Categories.MOVIE] = Type.MOVIE;
-  mapping[leetx.Categories.TV] = Type.SERIES;
+  mapping[ondebaixa.Categories.TV] = Type.SERIES;
-  mapping[leetx.Categories.ANIME] = Type.ANIME;
+  mapping[ondebaixa.Categories.DESENHOS] = Type.SERIES;
+  mapping[ondebaixa.Categories.ANIME] = Type.ANIME;
   return mapping;
 }
 
 function untilPage(category) {
-  if (leetx.Categories.DESENHOS === category) {
+  if (ondebaixa.Categories.DESENHOS === category) {
     return 5;
   }
-  if (leetx.Categories.TV === category) {
+  if (ondebaixa.Categories.TV === category) {
     return 5;
   }
   return UNTIL_PAGE;