check dubbed content when extracting magnets

2021-09-15 08:56:49 +02:00
parent c9d1d70b66
commit aaca34b2d6
4 changed files with 71 additions and 90 deletions
--- a/scraper/scrapers/comando/comando_api.js
+++ b/scraper/scrapers/comando/comando_api.js
@@ -31,7 +31,10 @@ function torrent(torrentId, config = {}, retries = 2) {
          .map((proxyUrl) => singleRequest(`${proxyUrl}/${slug}`, config)))
      .then((body) => parseTorrentPage(body))
      .then((torrent) => torrent.map(el => ({ torrentId: slug, ...el })))
-      .catch((err) => torrent(slug, config, retries - 1));
+      .catch((err) => {
+        console.warn(`Failed ${slug} request: `, err);
+        return torrent(slug, config, retries - 1)
+      });
 }

 function search(keyword, config = {}, retries = 2) {
@@ -113,32 +116,30 @@ function parseTorrentPage(body) {
    if (!$) {
      reject(new Error('Failed loading body'));
    }
-    let magnets = [];
-    $(`a[href^="magnet"]`).each((i, section) => {
-      let magnet = $(section).attr("href");
-      magnets.push(magnet);
-    });
-    const details = $('b:contains(\'Original\')').parent()
-    const isAnime = parseCategory($('div.entry-categories').html()) === Categories.ANIME
+    const magnets = $('h2 > strong')
+        .filter((i, elem) => isPtDubbed($(elem).text())).parent()
+        .map((i, elem) => $(elem).nextUntil('h2, hr'))
+        .map((i, elem) => $(elem).find('a[href^="magnet"]'))
+        .map((i, section) => $(section).attr("href")).get();
+    const details = $('b:contains(\'Original\'), strong:contains(\'Original\')').parent()
    const imdbIdMatch = details.find('a[href*="imdb.com"]').attr('href')
-    const torrent = magnets.map(magnetLink => {
-      const name = escapeHTML(decode(magnetLink).name.replace(/\+/g, ' '))
-      if (isPtDubbed(name) || isAnime) {
-        return {
-          title: sanitizePtName(name),
-          originalName: details.find('b:contains(\'Original\')')[0].nextSibling.nodeValue.replace(/: ?/, ''),
-          year: details.find('a[href*="comando.to/category/"]').text(),
-          infoHash: decode(magnetLink).infoHash,
-          magnetLink: magnetLink,
-          category: parseCategory($('div.entry-categories').html()),
-          uploadDate: new Date(moment($('a.updated').text(), 'LL', 'pt-br').format()),
-          imdbId: imdbIdMatch ? imdbIdMatch.split('/')[4] : null,
-          languages: sanitizePtLanguages(details.find(
-              'b:contains(\'Idioma\'), b:contains(\'Audio\'), b:contains(\'Áudio\')')[0].nextSibling.nodeValue)
-        };
+    const torrents = magnets.map(magnetLink => {
+      const originalName = details.find('strong:contains(\'Original\')').next().text().trim() ||
+          details.find('b:contains(\'Original\'), strong:contains(\'Original\')')[0].nextSibling.nodeValue;
+      return {
+        title: sanitizePtName(escapeHTML(decode(magnetLink).name.replace(/\+/g, ' '))),
+        originalName: originalName.replace(/: ?/, '').trim(),
+        year: details.find('a[href*="comando.to/category/"]').text(),
+        infoHash: decode(magnetLink).infoHash,
+        magnetLink: magnetLink,
+        category: parseCategory($('div.entry-categories').html()),
+        uploadDate: new Date(moment($('a.updated').text(), 'LL', 'pt-br').format()),
+        imdbId: imdbIdMatch ? imdbIdMatch.split('/')[4] : null,
+        languages: sanitizePtLanguages(details.find(
+            'b:contains(\'Idioma\'), b:contains(\'Audio\'), b:contains(\'Áudio\')')[0].nextSibling.nodeValue)
      }
-    })
-    resolve(torrent.filter((x) => x));
+    });
+    resolve(torrents.filter((x) => x));
  });
 }

--- a/scraper/scrapers/comoeubaixo/comoeubaixo_api.js
+++ b/scraper/scrapers/comoeubaixo/comoeubaixo_api.js
@@ -113,33 +113,28 @@ function parseTorrentPage(body) {
    if (!$) {
      reject(new Error('Failed loading body'));
    }
-    let magnets = [];
-    $(`a[href^="magnet"]`).each((i, section) => {
-      let magnet = $(section).attr("href");
-      magnets.push(magnet);
-    });
+    const magnets = $(`a[href^="magnet"]`)
+        .filter((i, elem) => isPtDubbed($(elem).attr('title')))
+        .map((i, elem) => $(elem).attr("href")).get();
    const details = $('div#informacoes')
    const category = details.find('strong:contains(\'Gêneros: \')').next().attr('href').split('/')[0]
-    const isAnime = category === Categories.ANIME
-    const torrent = magnets.map(magnetLink => {
+    const torrents = magnets.map(magnetLink => {
      const name = escapeHTML(decode(magnetLink).name.replace(/\+/g, ' '))
      const sanitizedTitle = sanitizePtName(name);
      const originalTitle = details.find('strong:contains(\'Baixar\')')[0].nextSibling.nodeValue.split('-')[0];
      const year = details.find('strong:contains(\'Data de Lançamento: \')').next().text().trim();
      const fallBackTitle = `${originalTitle.trim()} ${year.trim()} ${sanitizedTitle.trim()}`;
-      if (isPtDubbed(name) || isAnime) {
-        return {
-          title: sanitizedTitle.length > 4 ? sanitizedTitle : fallBackTitle,
-          infoHash: decode(magnetLink).infoHash,
-          magnetLink: magnetLink,
-          category: category,
-          uploadDate: new Date($('time').attr('datetime')),
-          imdbId: details.find('a[href*="imdb.com"]').attr('href').split('/')[4],
-          languages: sanitizePtLanguages(details.find('strong:contains(\'Idioma\')')[0].nextSibling.nodeValue)
-        };
-      }
+      return {
+        title: sanitizedTitle.length > 4 ? sanitizedTitle : fallBackTitle,
+        infoHash: decode(magnetLink).infoHash,
+        magnetLink: magnetLink,
+        category: category,
+        uploadDate: new Date($('time').attr('datetime')),
+        imdbId: details.find('a[href*="imdb.com"]').attr('href').split('/')[4],
+        languages: sanitizePtLanguages(details.find('strong:contains(\'Idioma\')')[0].nextSibling.nodeValue)
+      };
    })
-    resolve(torrent.filter((x) => x));
+    resolve(torrents.filter((x) => x));
  });
 }

--- a/scraper/scrapers/lapumia/lapumia_api.js
+++ b/scraper/scrapers/lapumia/lapumia_api.js
@@ -112,32 +112,25 @@ function parseTorrentPage(body) {
    if (!$) {
      reject(new Error('Failed loading body'));
    }
-    let magnets = [];
-    $(`a[href^="magnet"]`).each((i, section) => {
-      let magnet = $(section).attr("href");
-      magnets.push(magnet);
-    });
+    const magnets = $('h2 > span')
+        .filter((i, elem) => isPtDubbed($(elem).text())).parent()
+        .map((i, elem) => $(elem).nextUntil('h2, hr'))
+        .map((i, elem) => $(elem).find('a[href^="magnet"]'))
+        .map((i, section) => $(section).attr("href")).get();
    const category = parseCategory($('div.category').html());
    const details = $('div.content')
-    const isAnime = category === Categories.ANIME
-    const torrent = magnets.map(magnetLink => {
-      const name = escapeHTML(decode(magnetLink).name.replace(/\+/g, ' '))
-      if (isPtDubbed(name) || isAnime) {
-        return {
-          title: sanitizePtName(name),
-          originalName: sanitizePtOriginalName(
-              details.find('b:contains(\'Titulo Original:\')')[0].nextSibling.nodeValue),
-          year: details.find('b:contains(\'Ano de Lançamento:\')')[0].nextSibling.nodeValue.trim(),
-          infoHash: decode(magnetLink).infoHash,
-          magnetLink: magnetLink,
-          category: category,
-          uploadDate: new Date(moment($('div.infos').text().split('•')[0].trim(), 'LL', 'pt-br').format()),
-          imdbId: $('.imdbRatingPlugin').attr('data-title') || null,
-          languages: sanitizePtLanguages(details.find('b:contains(\'Idioma\')')[0].nextSibling.nodeValue)
-        };
-      }
-    })
-    resolve(torrent.filter((x) => x));
+    const torrents = magnets.map(magnetLink => ({
+      title: sanitizePtName(escapeHTML(decode(magnetLink).name.replace(/\+/g, ' '))),
+      originalName: sanitizePtOriginalName(details.find('b:contains(\'Titulo Original:\')')[0].nextSibling.nodeValue),
+      year: details.find('b:contains(\'Ano de Lançamento:\')')[0].nextSibling.nodeValue.trim(),
+      infoHash: decode(magnetLink).infoHash,
+      magnetLink: magnetLink,
+      category: category,
+      uploadDate: new Date(moment($('div.infos').text().split('•')[0].trim(), 'LL', 'pt-br').format()),
+      imdbId: $('.imdbRatingPlugin').attr('data-title') || null,
+      languages: sanitizePtLanguages(details.find('b:contains(\'Idioma\')')[0].nextSibling.nodeValue)
+    }))
+    resolve(torrents.filter((x) => x));
  });
 }

--- a/scraper/scrapers/ondebaixa/ondebaixa_api.js
+++ b/scraper/scrapers/ondebaixa/ondebaixa_api.js
@@ -112,30 +112,22 @@ function parseTorrentPage(body) {
    if (!$) {
      reject(new Error('Failed loading body'));
    }
-    let magnets = [];
-    $(`a[href^="magnet"]`).each((i, section) => {
-      let magnet = $(section).attr("href");
-      magnets.push(magnet);
-    });
+    const magnets = $(`a[href^="magnet"]`)
+        .filter((i, elem) => isPtDubbed($(elem).attr('title')))
+        .map((i, elem) => $(elem).attr("href")).get();
    const details = $('div#informacoes')
    const category = details.find('span:contains(\'Gêneros: \')').next().html()
-    const isAnime = parseCategory(category) === Categories.ANIME
-    const torrent = magnets.map(magnetLink => {
-      const name = escapeHTML(decode(magnetLink).name.replace(/\+/g, ' '))
-      if (isPtDubbed(name) || isAnime) {
-        return {
-          title: sanitizePtName(name),
-          originalName: sanitizePtOriginalName(details.find('span:contains(\'Título Original: \')').next().text()),
-          year: details.find('span:contains(\'Ano de Lançamento: \')').next().text().trim(),
-          infoHash: decode(magnetLink).infoHash,
-          magnetLink: magnetLink,
-          category: parseCategory(category),
-          uploadDate: new Date($('time').attr('datetime')),
-          languages: sanitizePtLanguages(details.find('span:contains(\'Idioma\')')[0].nextSibling.nodeValue)
-        };
-      }
-    })
-    resolve(torrent.filter((x) => x));
+    const torrents = magnets.map(magnetLink => ({
+      title: sanitizePtName(escapeHTML(decode(magnetLink).name.replace(/\+/g, ' '))),
+      originalName: sanitizePtOriginalName(details.find('span:contains(\'Título Original: \')').next().text()),
+      year: details.find('span:contains(\'Ano de Lançamento: \')').next().text().trim(),
+      infoHash: decode(magnetLink).infoHash,
+      magnetLink: magnetLink,
+      category: parseCategory(category),
+      uploadDate: new Date($('time').attr('datetime')),
+      languages: sanitizePtLanguages(details.find('span:contains(\'Idioma\')')[0].nextSibling.nodeValue)
+    }));
+    resolve(torrents.filter((x) => x));
  });
 }