New Commit

scraper/scrapers/darkmahou/darkmahou_api.js

@@ -0,0 +1,150 @@
+const needle = require("needle");
+const Bottleneck = require("bottleneck");
+const cheerio = require("cheerio");
+const decode = require("magnet-uri");
+const Promises = require("../../lib/promises");
+const { getRandomUserAgent } = require("../../lib/requestHelper");
+
+const defaultTimeout = 10000;
+const maxSearchPage = 50;
+
+const limiter = new Bottleneck({ maxConcurrent: 10 });
+
+const defaultProxies = ["https://darkmahou.com"];
+
+const Categories = {
+  MOVIE: "movie",
+  ANIME: "tv",
+  OVA: 'ova'
+};
+
+function torrent(torrentId, config = {}, retries = 2) {
+  if (!torrentId || retries === 0) {
+    return Promise.reject(new Error(`Failed ${torrentId} query`));
+  }
+  const proxyList = config.proxyList || defaultProxies;
+  const slug = torrentId.split("/")[3];
+  return Promises.first(
+    proxyList.map((proxyUrl) => singleRequest(`${proxyUrl}/${slug}`, config))
+  )
+    .then((body) => parseTorrentPage(body))
+    .then((torrent) => torrent.map((el) => ({ torrentId: slug, ...el })))
+    .catch((err) => torrent(slug, config, retries - 1));
+}
+
+function search(keyword, config = {}, retries = 2) {
+  if (!keyword || retries === 0) {
+    return Promise.reject(new Error(`Failed ${keyword} search`));
+  }
+  const proxyList = config.proxyList || defaultProxies;
+  const page = config.page || 1;
+  const extendToPage = Math.min(maxSearchPage, config.extendToPage || 1);
+  const requestUrl = (proxyUrl) => `${proxyUrl}/page/${page}/?s=${keyword}`;
+
+  return Promises.first(
+    proxyList.map((proxyUrl) => singleRequest(requestUrl(proxyUrl), config))
+  )
+    .then((body) => parseTableBody(body))
+    .then((torrents) =>
+      torrents.length === 40 && page < extendToPage
+        ? search(keyword, { ...config, page: page + 1 })
+            .catch(() => [])
+            .then((nextTorrents) => torrents.concat(nextTorrents))
+        : torrents
+    )
+    .catch((err) => search(keyword, config, retries - 1));
+}
+
+function browse(config = {}, retries = 2) {
+  if (retries === 0) {
+    return Promise.reject(new Error(`Failed browse request`));
+  }
+  const proxyList = config.proxyList || defaultProxies;
+  const page = config.page || 1;
+  const category = config.category;
+  const requestUrl = (proxyUrl) =>
+    category
+      ? `${proxyUrl}/category/${category}/page/${page}/`
+      : `${proxyUrl}/page/${page}/`;
+
+  return Promises.first(
+    proxyList.map((proxyUrl) => singleRequest(requestUrl(proxyUrl), config))
+  )
+    .then((body) => parseTableBody(body))
+    .catch((err) => browse(config, retries - 1));
+}
+
+function singleRequest(requestUrl, config = {}) {
+  const timeout = config.timeout || defaultTimeout;
+  const options = {
+    userAgent: getRandomUserAgent(),
+    open_timeout: timeout,
+    follow: 2,
+  };
+
+  return needle("get", requestUrl, options).then((response) => {
+    const body = response.body;
+    if (!body) {
+      throw new Error(`No body: ${requestUrl}`);
+    } else if (
+      body.includes("502: Bad gateway") ||
+      body.includes("403 Forbidden")
+    ) {
+      throw new Error(`Invalid body contents: ${requestUrl}`);
+    }
+    return body;
+  });
+}
+
+function parseTableBody(body) {
+  return new Promise((resolve, reject) => {
+    const $ = cheerio.load(body);
+
+    if (!$) {
+      reject(new Error("Failed loading body"));
+    }
+
+    const torrents = [];
+
+    $("article.bs").each((i, element) => {
+      const row = $(element);
+      torrents.push({
+        name: row.find("span.ntitle").text(),
+        torrentId: row.find("div > a").attr("href"),
+      });
+    });
+    resolve(torrents);
+  });
+}
+
+function parseTorrentPage(body) {
+  return new Promise(async(resolve, reject) => {
+    const $ = cheerio.load(body);
+
+    if (!$) {
+      reject(new Error("Failed loading body"));
+    }
+    let magnets = [];
+    $(`a[href^="magnet"]`).each((i, section) => {
+      const magnet = $(section).attr("href");
+      magnets.push(magnet);
+    });
+    const details = $('div.infox')
+    const torrent = magnets.map((magnetLink) => {
+      if (magnetLink.includes("magnet")) {
+        return {
+          title: decode(magnetLink).name,
+          original_name: details.find('h1.entry-title').text(),
+          year: details.find('b:contains(\'Lançado:\')')[0].nextSibling.nodeValue,          
+          infoHash: decode(magnetLink).infoHash,
+          magnetLink: magnetLink,
+          category: details.find('b:contains(\'Tipo:\')').next().attr('href').split('/')[4],
+          uploadDate: new Date($("time[itemprop=dateModified]").attr("datetime")),
+        };
+      }
+    })
+    resolve(torrent.filter((x) => x));
+  });
+}
+
+module.exports = { torrent, search, browse, Categories };

scraper/scrapers/darkmahou/darkmahou_scraper.js

@@ -0,0 +1,131 @@
+const moment = require("moment");
+const Bottleneck = require("bottleneck");
+const leetx = require("./darkmahou_api");
+const { Type } = require("../../lib/types");
+const repository = require("../../lib/repository");
+const Promises = require("../../lib/promises");
+const { createTorrentEntry, checkAndUpdateTorrent } = require("../../lib/torrentEntries");
+const { updateCurrentSeeders, updateTorrentSize } = require("../../lib/torrent");
+const { getImdbId } = require("../../lib/metadata");
+
+const NAME = "DarkMahou";
+const UNTIL_PAGE = 5;
+const TYPE_MAPPING = typeMapping();
+
+const limiter = new Bottleneck({ maxConcurrent: 5 });
+
+async function scrape() {
+  const scrapeStart = moment();
+  const lastScrape = await repository.getProvider({ name: NAME });
+  console.log(`[${scrapeStart}] starting ${NAME} scrape...`);
+
+  return scrapeLatestTorrents()
+    .then(() => {
+      lastScrape.lastScraped = scrapeStart;
+      return lastScrape.save();
+    })
+    .then(() => console.log(`[${moment()}] finished ${NAME} scrape`));
+}
+
+async function updateSeeders(torrent) {
+  return limiter.schedule(() => leetx.torrent(torrent.torrentId));
+}
+
+async function scrapeLatestTorrents() {
+  const allowedCategories = [
+    leetx.Categories.MOVIE,
+    leetx.Categories.ANIME,
+    leetx.Categories.OVA
+  ];
+
+  return Promises.sequence(
+    allowedCategories.map(
+      (category) => () => scrapeLatestTorrentsForCategory(category)
+    )
+  ).then((entries) => entries.reduce((a, b) => a.concat(b), []));
+}
+
+async function scrapeLatestTorrentsForCategory(category, page = 1) {
+  console.log(`Scrapping ${NAME} ${category} category page ${page}`);
+  return leetx
+    .browse({ category, page })
+    .catch((error) => {
+      console.warn(
+        `Failed ${NAME} scrapping for [${page}] ${category} due: `,
+        error
+      );
+      return Promise.resolve([]);
+    })
+    .then((torrents) => Promise.all(torrents.map((torrent) => limiter.schedule(() => processTorrentRecord(torrent)))))
+    .then((resolved) => resolved.length > 0 && page < untilPage(category) ? scrapeLatestTorrentsForCategory(category, page + 1) : Promise.resolve());
+}
+
+async function processTorrentRecord(record) {
+  if (await checkAndUpdateTorrent({ provider: NAME, ...record })) {
+    return record;
+  }
+  const torrentEntrys = await leetx
+    .torrent(record.torrentId)
+    .catch(() => undefined);
+  if (torrentEntrys === undefined) {
+    return Promise.resolve([])
+  }
+  return Promise.allSettled(
+    torrentEntrys.map(async (torrentFound) => {
+      if (!torrentFound || !TYPE_MAPPING[torrentFound.category]) {
+        return Promise.resolve("Invalid torrent record");
+      }
+      if (isNaN(torrentFound.uploadDate)) {
+        console.warn(
+          `Incorrect upload date for [${torrentFound.infoHash}] ${torrentFound.name}`
+        );
+        return;
+      }
+     if (await checkAndUpdateTorrent(torrentFound)) {
+        return torrentFound;
+      }
+      if (!torrentFound.size) {
+        await updateTorrentSize(torrentFound)
+        .catch((err) => Promise.resolve(err))
+      }
+      if (!torrentFound.seeders) {
+        await updateCurrentSeeders(torrentFound)
+        .then(response => response.seeders === 0 ? delete response.seeders : response)
+      }
+      if (!torrentFound.imdbId) {
+        torrentFound.imdbId = await getImdbId(torrentFound.original_name, torrentFound.year, TYPE_MAPPING[torrentFound.category])
+      }
+      const torrent = {
+        infoHash: torrentFound.infoHash,
+        provider: NAME,
+        torrentId: torrentFound.torrentId,
+        title: torrentFound.title.replace(/\t|\s+/g, " ").trim(),
+        type: Type.ANIME,
+        imdbId: torrentFound.imdbId,
+        uploadDate: torrentFound.uploadDate,
+        seeders: torrentFound.seeders,
+      };
+      return createTorrentEntry(torrent);
+    })
+  );
+}
+
+function typeMapping() {
+  const mapping = {};
+  mapping[leetx.Categories.MOVIE] = Type.MOVIE;
+  mapping[leetx.Categories.ANIME] = Type.SERIES;
+  mapping[leetx.Categories.OVA] = Type.ANIME
+  return mapping;
+}
+
+function untilPage(category) {
+  if (leetx.Categories.ANIME === category) {
+    return 5;
+  }
+  if (leetx.Categories.OVA === category) {
+    return 4;
+  }
+  return UNTIL_PAGE;
+}
+
+module.exports = { scrape, updateSeeders, NAME };