format comando provider

2021-09-14 15:10:28 +02:00
parent 4b4cefb3c2
commit 58a0e8286f
2 changed files with 80 additions and 128 deletions
--- a/scraper/scrapers/comando/comando_api.js
+++ b/scraper/scrapers/comando/comando_api.js
@@ -4,10 +4,10 @@ const cheerio = require("cheerio");
 const decode = require('magnet-uri');
 const Promises = require('../../lib/promises');
 const { escapeHTML } = require('../../lib/metadata');
-const { getRandomUserAgent } = require("../../lib/requestHelper");
-moment.locale("pt-br"); 
+const { getRandomUserAgent } = require('../../lib/requestHelper');
+const { isPtDubbed, sanitizePtName, sanitizePtLanguages } = require('../scraperHelper')

-const defaultTimeout = 10000;
+const defaultTimeout = 30000;
 const maxSearchPage = 50

 const defaultProxies = [
@@ -28,7 +28,7 @@ function torrent(torrentId, config = {}, retries = 2) {
  const proxyList = config.proxyList || defaultProxies;
  const slug = torrentId.split("/")[3];
  return Promises.first(proxyList
-      .map((proxyUrl) => singleRequest(`${proxyUrl}/${slug}`, config)))
+          .map((proxyUrl) => singleRequest(`${proxyUrl}/${slug}`, config)))
      .then((body) => parseTorrentPage(body))
      .then((torrent) => torrent.map(el => ({ torrentId: slug, ...el })))
      .catch((err) => torrent(slug, config, retries - 1));
@@ -44,7 +44,7 @@ function search(keyword, config = {}, retries = 2) {
  const requestUrl = proxyUrl => `${proxyUrl}/page/${page}/?s=${keyword}`

  return Promises.first(proxyList
-      .map(proxyUrl => singleRequest(requestUrl(proxyUrl), config)))
+          .map(proxyUrl => singleRequest(requestUrl(proxyUrl), config)))
      .then(body => parseTableBody(body))
      .then(torrents => torrents.length === 40 && page < extendToPage
          ? search(keyword, { ...config, page: page + 1 }).catch(() => [])
@@ -63,7 +63,7 @@ function browse(config = {}, retries = 2) {
  const requestUrl = proxyUrl => `${proxyUrl}/category/${category}/page/${page}/`

  return Promises.first(proxyList
-      .map((proxyUrl) => singleRequest(requestUrl(proxyUrl), config)))
+          .map((proxyUrl) => singleRequest(requestUrl(proxyUrl), config)))
      .then((body) => parseTableBody(body))
      .catch((err) => browse(config, retries - 1));
 }
@@ -104,7 +104,7 @@ function parseTableBody(body) {
    });
    resolve(torrents);
  });
-} 
+}

 function parseTorrentPage(body) {
  return new Promise((resolve, reject) => {
@@ -123,16 +123,18 @@ function parseTorrentPage(body) {
    const imdbIdMatch = details.find('a[href*="imdb.com"]').attr('href')
    const torrent = magnets.map(magnetLink => {
      const name = escapeHTML(decode(magnetLink).name.replace(/\+/g, ' '))
-      if(isDubled(name) || isAnime) {
+      if (isPtDubbed(name) || isAnime) {
        return {
-          name: name.replace(/ /g, '.'),
-          original_name: parseName(details.find('b:contains(\'Original\')')[0].nextSibling.nodeValue.replace(':', '')),
+          title: sanitizePtName(name),
+          originalName: details.find('b:contains(\'Original\')')[0].nextSibling.nodeValue.replace(/: ?/, ''),
          year: details.find('a[href*="comando.to/category/"]').text(),
          infoHash: decode(magnetLink).infoHash,
          magnetLink: magnetLink,
          category: parseCategory($('div.entry-categories').html()),
-          uploadDate: new Date(moment($('a.updated').text(), 'LL', true).format()),
-          imdbId: imdbIdMatch ? imdbIdMatch.split('/')[4] : null
+          uploadDate: new Date(moment($('a.updated').text(), 'LL', 'pt-br').format()),
+          imdbId: imdbIdMatch ? imdbIdMatch.split('/')[4] : null,
+          languages: sanitizePtLanguages(details.find(
+              'b:contains(\'Idioma\'), b:contains(\'Audio\'), b:contains(\'Áudio\')')[0].nextSibling.nodeValue)
        };
      }
    })
@@ -140,41 +142,15 @@ function parseTorrentPage(body) {
  });
 }

-function parseName(name) {
-  return name
-  .replace(/S01|S02|S03|S04|S05|S06|S07|S08|S09/g, '')
-  .trim()
-}
-
-function isDubled(name){
-  name = name.toLowerCase()
-  if(name.includes('dublado')){
-    return true
-  }
-  if(name.includes('dual')){
-    return true
-  }
-  if(name.includes('nacional')){
-    return true
-  }
-  if(name.includes('multi')){
-    return true
-  }
-  return false
-}
-
 function parseCategory(categorys) {
  const $ = cheerio.load(categorys)
-  const isAnime = $('a:contains(\'animes\')').text()
-  const isMovie = $('a:contains(\'Filmes\')').text()
-  const isSerie = $('a:contains(\'Series\')').text()
-  if(isAnime) {
+  if ($('a:contains(\'animes\')').text()) {
    return Categories.ANIME
-  } 
-  if (isMovie) {
+  }
+  if ($('a:contains(\'Filmes\')').text()) {
    return Categories.MOVIE
-  } 
-  if(isSerie) {
+  }
+  if ($('a:contains(\'Series\')').text()) {
    return Categories.TV
  }
 }
--- a/scraper/scrapers/comando/comando_scraper.js
+++ b/scraper/scrapers/comando/comando_scraper.js
@@ -1,6 +1,6 @@
 const moment = require("moment");
 const Bottleneck = require("bottleneck");
-const leetx = require("./comando_api");
+const comando = require("./comando_api");
 const { Type } = require("../../lib/types");
 const repository = require("../../lib/repository");
 const Promises = require("../../lib/promises");
@@ -20,114 +20,90 @@ async function scrape() {
  console.log(`[${scrapeStart}] starting ${NAME} scrape...`);

  return scrapeLatestTorrents()
-    .then(() => {
-      lastScrape.lastScraped = scrapeStart;
-      return lastScrape.save();
-    })
-    .then(() => console.log(`[${moment()}] finished ${NAME} scrape`));
+      .then(() => {
+        lastScrape.lastScraped = scrapeStart;
+        return lastScrape.save();
+      })
+      .then(() => console.log(`[${moment()}] finished ${NAME} scrape`));
 }

 async function updateSeeders(torrent) {
-  return limiter.schedule(() => leetx.torrent(torrent.torrentId));
+  return limiter.schedule(() => comando.torrent(torrent.torrentId));
 }

 async function scrapeLatestTorrents() {
  const allowedCategories = [
-    leetx.Categories.MOVIE,
-    leetx.Categories.TV,
-    leetx.Categories.ANIME,
+    // comando.Categories.MOVIE,
+    comando.Categories.TV
  ];

-  return Promises.sequence(
-    allowedCategories.map(
-      (category) => () => scrapeLatestTorrentsForCategory(category)
-    )
-  ).then((entries) => entries.reduce((a, b) => a.concat(b), []));
+  return Promises.sequence(allowedCategories
+          .map((category) => () => scrapeLatestTorrentsForCategory(category)))
+      .then((entries) => entries.reduce((a, b) => a.concat(b), []));
 }

 async function scrapeLatestTorrentsForCategory(category, page = 1) {
-  console.log({Scraper: `Scrapping ${NAME} ${category} category page ${page}`});
-  return leetx
-    .browse({ category, page })
-    .catch((error) => {
-      console.warn(
-        `Failed ${NAME} scrapping for [${page}] ${category} due: `,
-        error
-      );
-      return Promise.resolve([]);
-    })
-    .then((torrents) => Promise.all(torrents.map((torrent) => limiter.schedule(() => processTorrentRecord(torrent)))))
-    .then((resolved) => resolved.length > 0 && page < untilPage(category) ? scrapeLatestTorrentsForCategory(category, page + 1) : Promise.resolve());
+  console.log(`Scrapping ${NAME} ${category} category page ${page}`);
+  return comando.browse({ category, page })
+      .catch((error) => {
+        console.warn(`Failed ${NAME} scrapping for [${page}] ${category} due: `, error);
+        return Promise.resolve([]);
+      })
+      .then((torrents) => Promise.all(torrents.map((torrent) => limiter.schedule(() => processEntry(torrent)))))
+      .then((resolved) => resolved.length > 0 && page < untilPage(category)
+          ? scrapeLatestTorrentsForCategory(category, page + 1)
+          : Promise.resolve());
 }

-async function processTorrentRecord(record) {
-  if (await checkAndUpdateTorrent({ provider: NAME, ...record })) {
-    return record;
+async function processEntry(entry) {
+  return comando.torrent(entry.torrentId)
+      .then(records => Promises.sequence(records.map(record => () => processTorrentRecord(record))))
+      .catch(() => undefined);
+}
+
+async function processTorrentRecord(foundTorrent) {
+  if (await checkAndUpdateTorrent({ provider: NAME, ...foundTorrent })) {
+    return foundTorrent;
  }
-  const torrentEntrys = await leetx
-    .torrent(record.torrentId)
-    .catch(() => undefined);
-  if (torrentEntrys === undefined) {
-    return Promise.resolve([])
+
+  if (!foundTorrent.size) {
+    await updateTorrentSize(foundTorrent);
  }
-  return await Promise.allSettled(
-    torrentEntrys.map(async (torrentFound) => {
-      if (!torrentFound || !TYPE_MAPPING[torrentFound.category]) {
-        return Promise.resolve("Invalid torrent record");
-      }
-      if (isNaN(torrentFound.uploadDate)) {
-        console.warn(
-          `Incorrect upload date for [${torrentFound.infoHash}] ${torrentFound.name}`
-        );
-        return;
-      }
-      if (await checkAndUpdateTorrent(torrentFound)) {
-        return torrentFound;
-      }
-      if (!torrentFound.size) {
-        await updateTorrentSize(torrentFound)
-        .catch((err) => Promise.resolve(err))
-      }
-      if (!torrentFound.seeders) {
-        await updateCurrentSeeders(torrentFound)
-        .then(response => response.seeders === 0 ? delete response.seeders : response)
-      }
-      if (!torrentFound.imdbId) {
-        torrentFound.imdbId = await getImdbId(torrentFound.original_name, torrentFound.year, TYPE_MAPPING[torrentFound.category])
-      }
-      const torrent = {
-        infoHash: torrentFound.infoHash,
-        provider: NAME,
-        torrentId: torrentFound.torrentId,
-        name: torrentFound.original_name,
-        title: torrentFound.name.replace(/\t|\s+/g, " ").trim(),
-        type: TYPE_MAPPING[torrentFound.category],
-        year: torrentFound.year,
-        imdbId: torrentFound.imdbId,
-        uploadDate: torrentFound.uploadDate,
-        seeders: torrentFound.seeders,
-        size: torrentFound.size,
-        files: torrentFound.files
-      };
-      return createTorrentEntry(torrent);
-    })
-  );
+  if (!Number.isInteger(foundTorrent.seeders)) {
+    await updateCurrentSeeders(foundTorrent);
+  }
+  if (!foundTorrent.imdbId) {
+    const info = { title: foundTorrent.originalName, year: foundTorrent.year };
+    foundTorrent.imdbId = await getImdbId(info, TYPE_MAPPING[foundTorrent.category]).catch(() => undefined);
+  }
+
+  const torrent = {
+    infoHash: foundTorrent.infoHash,
+    provider: NAME,
+    torrentId: foundTorrent.torrentId,
+    title: foundTorrent.title,
+    type: TYPE_MAPPING[foundTorrent.category],
+    imdbId: foundTorrent.imdbId,
+    uploadDate: foundTorrent.uploadDate,
+    seeders: foundTorrent.seeders,
+    size: foundTorrent.size,
+    files: foundTorrent.files,
+    languages: foundTorrent.languages
+  };
+  return createTorrentEntry(torrent);
 }

 function typeMapping() {
  const mapping = {};
-  mapping[leetx.Categories.MOVIE] = Type.MOVIE;
-  mapping[leetx.Categories.DOCUMENTARIES] = Type.SERIES;
-  mapping[leetx.Categories.TV] = Type.SERIES;
-  mapping[leetx.Categories.ANIME] = Type.ANIME;
+  mapping[comando.Categories.MOVIE] = Type.MOVIE;
+  mapping[comando.Categories.DOCUMENTARIES] = Type.SERIES;
+  mapping[comando.Categories.TV] = Type.SERIES;
+  mapping[comando.Categories.ANIME] = Type.ANIME;
  return mapping;
 }

 function untilPage(category) {
-  if (leetx.Categories.ANIME === category) {
-    return 5;
-  }
-  if (leetx.Categories.TV === category) {
+  if (comando.Categories.TV === category) {
    return 5;
  }
  return UNTIL_PAGE;