format darkmahou provider

scraper/scrapers/darkmahou/darkmahou_api.js

@@ -8,13 +8,11 @@ const { getRandomUserAgent } = require("../../lib/requestHelper");
 const defaultTimeout = 10000;
 const maxSearchPage = 50;
 
-const limiter = new Bottleneck({ maxConcurrent: 10 });
+const defaultProxies = ['https://darkmahou.com'];
-
-const defaultProxies = ["https://darkmahou.com"];
 
 const Categories = {
-  MOVIE: "movie",
+  MOVIE: 'movie',
-  ANIME: "tv",
+  ANIME: 'tv',
   OVA: 'ova'
 };
 
@@ -133,7 +131,7 @@ function parseTorrentPage(body) {
     const torrent = magnets.map((magnetLink) => {
       return {
         title: decode(magnetLink).name,
-        original_name: details.find('h1.entry-title').text(),
+        originalName: details.find('h1.entry-title').text(),
         year: details.find('b:contains(\'Lançado:\')')[0].nextSibling.nodeValue || '',
         infoHash: decode(magnetLink).infoHash,
         magnetLink: magnetLink,

scraper/scrapers/darkmahou/darkmahou_scraper.js

@@ -1,16 +1,15 @@
 const moment = require("moment");
 const Bottleneck = require("bottleneck");
-const leetx = require("./darkmahou_api");
+const darkmahou = require("./darkmahou_api");
 const { Type } = require("../../lib/types");
 const repository = require("../../lib/repository");
 const Promises = require("../../lib/promises");
 const { createTorrentEntry, checkAndUpdateTorrent } = require("../../lib/torrentEntries");
 const { updateCurrentSeeders, updateTorrentSize } = require("../../lib/torrent");
-const { getImdbId } = require("../../lib/metadata");
+const { getKitsuId } = require("../../lib/metadata");
 
 const NAME = "DarkMahou";
 const UNTIL_PAGE = 5;
-const TYPE_MAPPING = typeMapping();
 
 const limiter = new Bottleneck({ maxConcurrent: 5 });
 
@@ -28,101 +27,79 @@ async function scrape() {
 }
 
 async function updateSeeders(torrent) {
-  return limiter.schedule(() => leetx.torrent(torrent.torrentId));
+  return limiter.schedule(() => darkmahou.torrent(torrent.torrentId));
 }
 
 async function scrapeLatestTorrents() {
   const allowedCategories = [
-    leetx.Categories.MOVIE,
+    darkmahou.Categories.MOVIE,
-    leetx.Categories.ANIME,
+    darkmahou.Categories.ANIME,
-    leetx.Categories.OVA
+    darkmahou.Categories.OVA
   ];
 
-  return Promises.sequence(
+  return Promises.sequence(allowedCategories
-    allowedCategories.map(
+          .map((category) => () => scrapeLatestTorrentsForCategory(category)))
-      (category) => () => scrapeLatestTorrentsForCategory(category)
+      .then((entries) => entries.reduce((a, b) => a.concat(b), []));
-    )
-  ).then((entries) => entries.reduce((a, b) => a.concat(b), []));
 }
 
 async function scrapeLatestTorrentsForCategory(category, page = 1) {
-  console.log({Scraper: `Scrapping ${NAME} ${category} category page ${page}`});
+  console.log(`Scrapping ${NAME} ${category} category page ${page}`);
-  return leetx
+  return darkmahou
       .browse({ category, page })
       .catch((error) => {
-      console.warn(
+        console.warn(`Failed ${NAME} scrapping for [${page}] ${category} due: `, error);
-        `Failed ${NAME} scrapping for [${page}] ${category} due: `,
-        error
-      );
         return Promise.resolve([]);
       })
-    .then((torrents) => Promise.all(torrents.map((torrent) => limiter.schedule(() => processTorrentRecord(torrent)))))
+      .then((torrents) => Promise.all(torrents.map((torrent) => limiter.schedule(() => processEntry(torrent)))))
-    .then((resolved) => resolved.length > 0 && page < untilPage(category) ? scrapeLatestTorrentsForCategory(category, page + 1) : Promise.resolve());
+      .then((resolved) => resolved.length > 0 && page < untilPage(category)
+          ? scrapeLatestTorrentsForCategory(category, page + 1)
+          : Promise.resolve());
 }
 
-async function processTorrentRecord(record) {
+async function processEntry(entry) {
-  if (await checkAndUpdateTorrent({ provider: NAME, ...record })) {
+  return darkmahou.torrent(entry.torrentId)
-    return record;
+      .then(records => Promises.sequence(records.map(record => () => processTorrentRecord(record))))
-  }
-  const torrentEntrys = await leetx
-    .torrent(record.torrentId)
       .catch(() => undefined);
-  if (torrentEntrys === undefined) {
-    return Promise.resolve([])
 }
-  return Promise.allSettled(
+
-    torrentEntrys.map(async (torrentFound) => {
+async function processTorrentRecord(foundTorrent) {
-      if (!torrentFound || !TYPE_MAPPING[torrentFound.category]) {
+  if (await checkAndUpdateTorrent({ provider: NAME, ...foundTorrent })) {
-        return Promise.resolve("Invalid torrent record");
+    return foundTorrent;
   }
-      if (isNaN(torrentFound.uploadDate)) {
+
-        console.warn(
+  if (!foundTorrent.size) {
-          `Incorrect upload date for [${torrentFound.infoHash}] ${torrentFound.name}`
+    await updateTorrentSize(foundTorrent);
-        );
-        return;
   }
-     if (await checkAndUpdateTorrent(torrentFound)) {
+  if (!Number.isInteger(foundTorrent.seeders)) {
-        return torrentFound;
+    await updateCurrentSeeders(foundTorrent);
   }
-      if (!torrentFound.size) {
+  if (!foundTorrent.imdbId && !foundTorrent.kitsuId) {
-        await updateTorrentSize(torrentFound)
+    const info = { title: foundTorrent.originalName, year: foundTorrent.year };
-        .catch((err) => Promise.resolve(err))
+    foundTorrent.kitsuId = await getKitsuId(info).catch(() => undefined);
-      }
-      if (!torrentFound.seeders) {
-        await updateCurrentSeeders(torrentFound)
-        .then(response => response.seeders === 0 ? delete response.seeders : response)
-      }
-      if (!torrentFound.imdbId) {
-        torrentFound.imdbId = await getImdbId(torrentFound.original_name, torrentFound.year, TYPE_MAPPING[torrentFound.category])
   }
+
   const torrent = {
-        infoHash: torrentFound.infoHash,
+    infoHash: foundTorrent.infoHash,
     provider: NAME,
-        torrentId: torrentFound.torrentId,
+    torrentId: foundTorrent.torrentId,
-        title: torrentFound.title.replace(/\t|\s+/g, " ").trim(),
+    title: foundTorrent.title,
     type: Type.ANIME,
-        imdbId: torrentFound.imdbId,
+    imdbId: foundTorrent.imdbId,
-        uploadDate: torrentFound.uploadDate,
+    kitsuId: foundTorrent.kitsuId,
-        seeders: torrentFound.seeders,
+    uploadDate: foundTorrent.uploadDate,
+    seeders: foundTorrent.seeders,
+    size: foundTorrent.size,
+    files: foundTorrent.files,
+    languages: foundTorrent.languages
   };
   return createTorrentEntry(torrent);
-    })
-  );
-}
-
-function typeMapping() {
-  const mapping = {};
-  mapping[leetx.Categories.MOVIE] = Type.MOVIE;
-  mapping[leetx.Categories.ANIME] = Type.SERIES;
-  mapping[leetx.Categories.OVA] = Type.ANIME
-  return mapping;
 }
 
 function untilPage(category) {
-  if (leetx.Categories.ANIME === category) {
+  if (darkmahou.Categories.ANIME === category) {
     return 5;
   }
-  if (leetx.Categories.OVA === category) {
+  if (darkmahou.Categories.OVA === category) {
     return 4;
   }
   return UNTIL_PAGE;