format comoeubaixo provider

scraper/scrapers/comoeubaixo/comoeubaixo_api.js

@@ -3,7 +3,8 @@ const cheerio = require("cheerio");
 const decode = require('magnet-uri');
 const Promises = require('../../lib/promises');
 const { escapeHTML } = require('../../lib/metadata');
-const { getRandomUserAgent } = require("../../lib/requestHelper");
+const { getRandomUserAgent } = require('../../lib/requestHelper');
+const { isPtDubbed, sanitizePtName, sanitizePtLanguages } = require('../scraperHelper')
 
 const defaultTimeout = 10000;
 const maxSearchPage = 50
@@ -97,7 +98,8 @@ function parseTableBody(body) {
       const row = $(element);
       torrents.push({
         name: row.find("a").text(),
-        torrentId: row.find("a").attr("href")
+        torrentId: row.find("a").attr("href"),
+        isTorrent: !!row.find("p:contains(\'Torrent\')")[0]
       });
     });
     resolve(torrents);
@@ -121,16 +123,19 @@ function parseTorrentPage(body) {
     const isAnime = category === Categories.ANIME
     const torrent = magnets.map(magnetLink => {
       const name = escapeHTML(decode(magnetLink).name.replace(/\+/g, ' '))
-      if(isDubled(name) || isAnime) {
+      const sanitizedTitle = sanitizePtName(name);
+      const originalTitle = details.find('strong:contains(\'Baixar\')')[0].nextSibling.nodeValue.split('-')[0];
+      const year = details.find('strong:contains(\'Data de Lançamento: \')').next().text().trim();
+      const fallBackTitle = `${originalTitle.trim()} ${year.trim()} ${sanitizedTitle.trim()}`;
+      if (isPtDubbed(name) || isAnime) {
         return {
-          name: parseText(name),
+          title: sanitizedTitle.length > 4 ? sanitizedTitle : fallBackTitle,
-          original_name: parseName(details.find('strong:contains(\'Baixar\')')[0].nextSibling.nodeValue.split('-')[0]),
-          year: details.find('strong:contains(\'Data de Lançamento: \')').next().text().trim(),
           infoHash: decode(magnetLink).infoHash,
           magnetLink: magnetLink,
           category: category,
           uploadDate: new Date($('time').attr('datetime')),
           imdbId: details.find('a[href*="imdb.com"]').attr('href').split('/')[4],
+          languages: sanitizePtLanguages(details.find('strong:contains(\'Idioma\')')[0].nextSibling.nodeValue)
         };
       }
     })
@@ -138,37 +143,4 @@ function parseTorrentPage(body) {
   });
 }
 
-function parseName(name) {
-  return name
-  .replace(/S01|S02|S03|S04|S05|S06|S07|S08|S09/g, '')
-  .trim()
-}
-
-function isDubled(name){
-  name = name.toLowerCase()
-  if(name.includes('dublado')){
-    return true
-  }
-  if(name.includes('dual')){
-    return true
-  }
-  if(name.includes('nacional')){
-    return true
-  }
-  return false
-}
-
-function parseText(text) {
-  return text
-  .replace(/\n|\t/g, "")
-  .replace(/1A|2A|3A|4A|5A|6A|7A|8A|9A/g, '')
-  .replace(/COMOEUBAIXO.COM|COMANDO.TO|TEMPORADA|COMPLETA/g, '')
-  .replace(/MKV|MP4/g, '')
-  .replace(/[-]/g, '')
-  .replace(/[.]/g, ' ')
-  .trim()
-  .replace(/ /g, '.')
-  .trim()
-}
-
 module.exports = { torrent, search, browse, Categories };

scraper/scrapers/comoeubaixo/comoeubaixo_scraper.js

@@ -1,15 +1,14 @@
 const moment = require("moment");
 const Bottleneck = require("bottleneck");
-const leetx = require("./comoeubaixo_api");
+const comoeubaixo = require("./comoeubaixo_api");
 const { Type } = require("../../lib/types");
 const repository = require("../../lib/repository");
 const Promises = require("../../lib/promises");
 const { createTorrentEntry, checkAndUpdateTorrent } = require("../../lib/torrentEntries");
 const { updateCurrentSeeders, updateTorrentSize } = require("../../lib/torrent");
-const { getImdbId } = require("../../lib/metadata");
 
 const NAME = "ComoEuBaixo";
-const UNTIL_PAGE = 5;
+const UNTIL_PAGE = 3;
 const TYPE_MAPPING = typeMapping();
 
 const limiter = new Bottleneck({ maxConcurrent: 5 });
@@ -28,108 +27,87 @@ async function scrape() {
 }
 
 async function updateSeeders(torrent) {
-  return limiter.schedule(() => leetx.torrent(torrent.torrentId));
+  return limiter.schedule(() => comoeubaixo.torrent(torrent.torrentId));
 }
 
 async function scrapeLatestTorrents() {
   const allowedCategories = [
-    leetx.Categories.MOVIE,
+    comoeubaixo.Categories.MOVIE,
-    leetx.Categories.TV,
+    comoeubaixo.Categories.TV,
-    leetx.Categories.DESENHOS
+    comoeubaixo.Categories.DESENHOS
   ];
 
-  return Promises.sequence(
+  return Promises.sequence(allowedCategories
-    allowedCategories.map(
+          .map((category) => () => scrapeLatestTorrentsForCategory(category)))
-      (category) => () => scrapeLatestTorrentsForCategory(category)
+      .then((entries) => entries.reduce((a, b) => a.concat(b), []));
-    )
-  ).then((entries) => entries.reduce((a, b) => a.concat(b), []));
 }
 
-async function scrapeLatestTorrentsForCategory(category, page = 1) {
+async function scrapeLatestTorrentsForCategory(category, page = 180) {
-  console.log({Scraper: `Scrapping ${NAME} ${category} category page ${page}`});
+  console.log(`Scrapping ${NAME} ${category} category page ${page}`);
-  return leetx
+  return comoeubaixo
       .browse({ category, page })
       .catch((error) => {
-      console.warn(
+        console.warn(`Failed ${NAME} scrapping for [${page}] ${category} due: `, error);
-        `Failed ${NAME} scrapping for [${page}] ${category} due: `,
-        error
-      );
         return Promise.resolve([]);
       })
-    .then((torrents) => Promise.all(torrents.map((torrent) => limiter.schedule(() => processTorrentRecord(torrent)))))
+      .then((torrents) => Promise.all(torrents.map((torrent) => limiter.schedule(() => processEntry(torrent)))))
-    .then((resolved) => resolved.length > 0 && page < untilPage(category) ? scrapeLatestTorrentsForCategory(category, page + 1) : Promise.resolve());
+      .then((resolved) => resolved.length > 0 && page < untilPage(category)
+          ? scrapeLatestTorrentsForCategory(category, page + 1)
+          : Promise.resolve());
 }
 
-async function processTorrentRecord(record) {
+async function processEntry(entry) {
-  if (await checkAndUpdateTorrent({ provider: NAME, ...record })) {
+  if (!entry.isTorrent) {
-    return record;
+    return entry;
   }
-  const torrentEntrys = await leetx
+  return comoeubaixo.torrent(entry.torrentId)
-    .torrent(record.torrentId)
+      .then(records => Promises.sequence(records.map(record => () => processTorrentRecord(record))))
       .catch(() => undefined);
-  if (torrentEntrys === undefined) {
-    return Promise.resolve([])
 }
-  return await Promise.allSettled(
+
-    torrentEntrys.map(async (torrentFound) => {
+async function processTorrentRecord(foundTorrent) {
-      if (!torrentFound || !TYPE_MAPPING[torrentFound.category]) {
+  if (await checkAndUpdateTorrent({ provider: NAME, ...foundTorrent })) {
-        return Promise.resolve("Invalid torrent record");
+    return foundTorrent;
   }
-      if (isNaN(torrentFound.uploadDate)) {
+
-        console.warn(
+  if (!foundTorrent.size) {
-          `Incorrect upload date for [${torrentFound.infoHash}] ${torrentFound.name}`
+    await updateTorrentSize(foundTorrent);
-        );
-        return;
   }
-      if (await checkAndUpdateTorrent(torrentFound)) {
+  if (!Number.isInteger(foundTorrent.seeders)) {
-        return torrentFound;
+    await updateCurrentSeeders(foundTorrent);
-      }
-      if (!torrentFound.size) {
-        await updateTorrentSize(torrentFound)
-        .catch((err) => Promise.resolve(err))
-      }
-      if (!torrentFound.seeders) {
-        await updateCurrentSeeders(torrentFound)
-        .then(response => response.seeders === 0 ? delete response.seeders : response)
-      }
-      if (!torrentFound.imdbId) {
-        torrentFound.imdbId = await getImdbId(torrentFound.original_name, torrentFound.year, TYPE_MAPPING[torrentFound.category])
   }
 
   const torrent = {
-        infoHash: torrentFound.infoHash,
+    infoHash: foundTorrent.infoHash,
     provider: NAME,
-        torrentId: torrentFound.torrentId,
+    torrentId: foundTorrent.torrentId,
-        name: torrentFound.original_name,
+    title: foundTorrent.title,
-        title: torrentFound.name.replace(/\t|\s+/g, " ").trim(),
+    type: TYPE_MAPPING[foundTorrent.category],
-        type: TYPE_MAPPING[torrentFound.category],
+    imdbId: foundTorrent.imdbId,
-        year: torrentFound.year,
+    uploadDate: foundTorrent.uploadDate,
-        imdbId: torrentFound.imdbId,
+    seeders: foundTorrent.seeders,
-        uploadDate: torrentFound.uploadDate,
+    size: foundTorrent.size,
-        seeders: torrentFound.seeders,
+    files: foundTorrent.files,
-        size: torrentFound.size,
+    languages: foundTorrent.languages
-        files: torrentFound.files
   };
   return createTorrentEntry(torrent);
-    })
-  );
 }
 
 function typeMapping() {
   const mapping = {};
-  mapping[leetx.Categories.MOVIE] = Type.MOVIE;
+  mapping[comoeubaixo.Categories.MOVIE] = Type.MOVIE;
-  mapping[leetx.Categories.TV] = Type.SERIES;
+  mapping[comoeubaixo.Categories.TV] = Type.SERIES;
-  mapping[leetx.Categories.ANIME] = Type.ANIME;
+  mapping[comoeubaixo.Categories.ANIME] = Type.ANIME;
-  mapping[leetx.Categories.DESENHOS] = Type.SERIES;
+  mapping[comoeubaixo.Categories.DESENHOS] = Type.SERIES;
   return mapping;
 }
 
 function untilPage(category) {
-  if (leetx.Categories.DESENHOS === category) {
+  if (comoeubaixo.Categories.DESENHOS === category) {
-    return 5;
+    return UNTIL_PAGE;
   }
-  if (leetx.Categories.TV === category) {
+  if (comoeubaixo.Categories.TV === category) {
-    return 5;
+    return UNTIL_PAGE;
   }
   return UNTIL_PAGE;
 }