updates tpb dump scrapper

2019-12-29 20:07:15 +01:00
parent 7925f18064
commit e7f46d2adc
12 changed files with 689 additions and 165 deletions
--- a/scrapers/api/thepiratebay.js
+++ b/scrapers/api/thepiratebay.js
@@ -2,9 +2,14 @@ const cheerio = require('cheerio');
 const needle = require('needle');
 const moment = require('moment');

-const defaultProxies = ['https://pirateproxy.sh', 'https://thepiratebay.org'];
+const defaultProxies = [
+    'https://thepiratebay.org',
+  'https://thepiratebay.vip',
+  'https://proxybay.pro',
+  'https://ukpiratebayproxy.com',
+  'https://thepiratebayproxy.info'];
 const dumpUrl = '/static/dump/csv/';
-const defaultTimeout = 5000;
+const defaultTimeout = 30000;

 const errors = {
  REQUEST_ERROR: { code: 'REQUEST_ERROR' },
@@ -76,6 +81,18 @@ Categories = {
  }
 };

+function torrent(torrentId, config = {}, retries = 2) {
+  if (!torrentId || retries === 0) {
+    return Promise.reject(new Error(`Failed ${torrentId} search`));
+  }
+  const proxyList = config.proxyList || defaultProxies;
+
+  return raceFirstSuccessful(proxyList
+      .map((proxyUrl) => singleRequest(`${proxyUrl}/torrent/${torrentId}`, config)))
+      .then((body) => parseTorrentPage(body))
+      .catch((err) => torrent(torrentId, config, retries - 1));
+}
+
 function search(keyword, config = {}, retries = 2) {
  if (!keyword || retries === 0) {
    return Promise.reject(new Error(`Failed ${keyword} search`));
@@ -87,7 +104,7 @@ function search(keyword, config = {}, retries = 2) {
  return raceFirstSuccessful(proxyList
      .map((proxyUrl) => singleRequest(`${proxyUrl}/search/${keyword}/${page}/99/${category}`, config)))
      .then((body) => parseBody(body))
-      .catch(() => search(keyword, config, retries - 1));
+      .catch((err) => search(keyword, config, retries - 1));
 }

 function dumps(config = {}, retries = 2) {
@@ -109,26 +126,23 @@ function dumps(config = {}, retries = 2) {
 function singleRequest(requestUrl, config = {}) {
  const timeout = config.timeout || defaultTimeout;

-  return new Promise(((resolve, reject) => {
-    needle.get(requestUrl,
-        { open_timeout: timeout, follow: 2 },
-        (err, res, body) => {
-          if (err || !body) {
-            reject(err || errors.REQUEST_ERROR);
-          } else if (body.includes('Access Denied') && !body.includes('<title>The Pirate Bay')) {
-            console.log(`Access Denied: ${url}`);
-            reject(new Error(`Access Denied: ${url}`));
-          } else if (body.includes('502: Bad gateway') ||
+  return needle('get', requestUrl, { open_timeout: timeout, follow: 2 })
+      .then((response) =>  {
+        const body = response.body;
+        if (!body) {
+          throw new Error(`No body: ${requestUrl}`);
+        } else if (body.includes('Access Denied') && !body.includes('<title>The Pirate Bay')) {
+          console.log(`Access Denied: ${requestUrl}`);
+          throw new Error(`Access Denied: ${requestUrl}`);
+        } else if (body.includes('502: Bad gateway') ||
            body.includes('403 Forbidden') ||
            body.includes('Database maintenance') ||
            body.includes('Origin DNS error') ||
-            !body.includes('<title>The Pirate Bay')) {
-            reject(errors.REQUEST_ERROR);
-          }
-
-          resolve(body);
-        });
-  }));
+            !(body.includes('<title>The Pirate Bay') || body.includes('TPB</title>') || body.includes(dumpUrl))) {
+          throw new Error(`Invalid body contents: ${requestUrl}`);
+        }
+        return body;
+      });
 }

 function parseBody(body) {
@@ -143,7 +157,7 @@ function parseBody(body) {

    $('table[id=\'searchResult\'] tr').each(function() {
      const name = $(this).find('.detLink').text();
-      if (!name) {
+      if (!name || name === 'Do NOT download any torrent before hiding your IP with a VPN.') {
        return;
      }
      torrents.push({
@@ -152,13 +166,51 @@ function parseBody(body) {
        leechers: parseInt($(this).find('td[align=\'right\']').eq(1).text(), 10),
        magnetLink: $(this).find('a[title=\'Download this torrent using magnet\']').attr('href'),
        category: parseInt($(this).find('a[title=\'More from this category\']').eq(0).attr('href').match(/\d+$/)[0], 10),
-        subcategory: parseInt($(this).find('a[title=\'More from this category\']').eq(1).attr('href').match(/\d+$/)[0], 10)
+        subcategory: parseInt($(this).find('a[title=\'More from this category\']').eq(1).attr('href').match(/\d+$/)[0], 10),
+        size: parseSize($(this).find('.detDesc').text().match(/(?:,\s?Size\s)(.+),/)[1])
      });
    });
    resolve(torrents);
  });
 }

+function parseTorrentPage(body) {
+  return new Promise((resolve, reject) => {
+    const $ = cheerio.load(body);
+
+    if (!$) {
+      reject(new Error(errors.PARSER_ERROR));
+    }
+
+    const torrent = {
+        name: $('div[id=\'title\']').text().trim(),
+        seeders: parseInt($('dl[class=\'col2\']').find('dd').eq(2).text(), 10),
+        leechers: parseInt($('dl[class=\'col2\']').find('dd').eq(3).text(), 10),
+        magnetLink: $('div[id=\'details\']').find('a[title=\'Get this torrent\']').attr('href'),
+        category: Categories.VIDEO.ALL,
+        subcategory: parseInt($('dl[class=\'col1\']').find('a[title=\'More from this category\']').eq(0).attr('href').match(/\d+$/)[0], 10),
+        size: parseSize($('dl[class=\'col1\']').find('dd').eq(2).text().match(/(\d+)(?:.?Bytes)/)[1])
+    };
+    resolve(torrent);
+  });
+}
+
+function parseSize(sizeText) {
+  if (!sizeText) {
+    return undefined;
+  }
+  if (sizeText.includes('GiB')) {
+    return Math.floor(parseFloat(sizeText.trim()) * 1024 * 1024 * 1024);
+  }
+  if (sizeText.includes('MiB')) {
+    return Math.floor(parseFloat(sizeText.trim()) * 1024 * 1024);
+  }
+  if (sizeText.includes('KiB')) {
+    return Math.floor(parseFloat(sizeText.trim()) * 1024);
+  }
+  return Math.floor(parseFloat(sizeText));
+}
+
 function raceFirstSuccessful(promises) {
  return Promise.all(promises.map((p) => {
    // If a request fails, count that as a resolution so it will keep
@@ -176,4 +228,4 @@ function raceFirstSuccessful(promises) {
  );
 }

-module.exports = { search, dumps, Categories };
+module.exports = { torrent, search, dumps, Categories };
--- a/scrapers/horiblesubs_scraper.js
+++ b/scrapers/horiblesubs_scraper.js
@@ -4,6 +4,7 @@ const Bottleneck = require('bottleneck');
 const { parse } = require('parse-torrent-title');
 const decode = require('magnet-uri');
 const horriblesubs = require('./api/horriblesubs');
+const { Type } = require('../lib/types');
 const { torrentFiles, currentSeeders } = require('../lib/torrent');
 const repository = require('../lib/repository');
 const { getImdbId, getMetadata } = require('../lib/metadata');
@@ -72,7 +73,7 @@ async function _constructSingleEntry(metadata, single, mirror) {
    infoHash: mirror.infoHash,
    provider: NAME,
    title: title,
-    type: 'anime',
+    type: Type.ANIME,
    imdbId: metadata.imdbId,
    uploadDate: single.uploadDate,
    seeders: seeders,
--- a/scrapers/piratebay_dump.js
+++ b/scrapers/piratebay_dump.js
@@ -6,9 +6,11 @@ const LineByLineReader = require('line-by-line');
 const fs = require('fs');
 const { parse } = require('parse-torrent-title');
 const pirata = require('./api/thepiratebay');
-const { torrentFiles } = require('../lib/torrent');
+const bing = require('nodejs-bing');
+const { Type } = require('../lib/types');
 const repository = require('../lib/repository');
 const { getImdbId, escapeTitle } = require('../lib/metadata');
+const { parseTorrentFiles } = require('../lib/torrentFiles');

 const NAME = 'ThePirateBay';
 const CSV_FILE_PATH = '/tmp/tpb_dump.csv';
@@ -17,18 +19,27 @@ const limiter = new Bottleneck({maxConcurrent: 40});

 async function scrape() {
  const lastScraped = await repository.getProvider({ name: NAME });
-  const lastDump = await pirata.dumps().then((dumps) => dumps.sort((a, b) => b.updatedAt - a.updatedAt)[0]);
+  const lastDump = { updatedAt: 2147000000 };
+  //const lastDump = await pirata.dumps().then((dumps) => dumps.sort((a, b) => b.updatedAt - a.updatedAt)[0]);

  if (!lastScraped.lastScraped || lastScraped.lastScraped < lastDump.updatedAt) {
    console.log(`starting to scrape tpb dump: ${JSON.stringify(lastDump)}`);
    //await downloadDump(lastDump);

+    let entriesProcessed = 0;
    const lr = new LineByLineReader(CSV_FILE_PATH);
    lr.on('line', (line) => {
      if (line.includes("#ADDED")) {
        return;
      }
+      if (entriesProcessed % 1000 === 0) {
+        console.log(`Processed ${entriesProcessed} entries`);
+      }
      const row = line.match(/(?<=^|;)(".*"|[^;]+)(?=;|$)/g);
+      if (row.length !== 4) {
+        console.log(`Invalid row: ${line}`);
+        return;
+      }
      const torrent = {
        uploadDate: moment(row[0], 'YYYY-MMM-DD HH:mm:ss').toDate(),
        infoHash: Buffer.from(row[1], 'base64').toString('hex'),
@@ -50,9 +61,10 @@ async function scrape() {
      }

      limiter.schedule(() => processTorrentRecord(torrent)
-            .catch((error) => console.log(`failed ${torrent.title} due: ${error}`)))
-            .then(() => limiter.empty())
-            .then((empty) => empty && lr.resume());
+          .catch((error) => console.log(`failed ${torrent.title} due: ${error}`)))
+          .then(() => limiter.empty())
+          .then((empty) => empty && lr.resume())
+          .then(() => entriesProcessed++);
    });
    lr.on('error', (err) => {
        console.log(err);
@@ -77,80 +89,90 @@ const seriesCategories = [
  pirata.Categories.VIDEO.TV_SHOWS_HD
 ];
 async function processTorrentRecord(record) {
-  const persisted = await repository.getSkipTorrent(record)
-      .catch(() => repository.getTorrent(record)).catch(() => undefined);
-  if (persisted) {
+  const alreadyExists = await repository.getSkipTorrent(record)
+      .catch(() => repository.getTorrent(record))
+      .catch(() => undefined);
+  if (alreadyExists) {
     return;
  }

-  let page = 0;
-  let torrentFound;
-  while (!torrentFound && page < 5) {
-    const torrents = await pirata.search(record.title.replace(/[\W\s]+/, ' '), { page: page });
-    torrentFound = torrents.
-    filter(torrent => torrent.magnetLink.toLowerCase().includes(record.infoHash))[0];
-    page = torrents.length === 0 ? 1000 : page + 1;
-  }
+  const torrentFound = await findTorrent(record);

  if (!torrentFound) {
-    console.log(`not found: ${JSON.stringify(record)}`);
+    //console.log(`not found: ${JSON.stringify(record)}`);
    repository.createSkipTorrent(record);
    return;
  }
  if (!allowedCategories.includes(torrentFound.subcategory)) {
-    console.log(`wrong category: ${torrentFound.name}`);
+    //console.log(`wrong category: ${torrentFound.name}`);
    repository.createSkipTorrent(record);
    return;
  }

-  const type = seriesCategories.includes(torrentFound.subcategory) ? 'series' : 'movie';
-  console.log(`imdbId search: ${torrentFound.name}`);
+  const type = seriesCategories.includes(torrentFound.subcategory) ? Type.SERIES : Type.MOVIE;
  const titleInfo = parse(torrentFound.name);
  const imdbId = await getImdbId({
    name: escapeTitle(titleInfo.title).toLowerCase(),
    year: titleInfo.year,
    type: type
-  }).catch(() => undefined);
+  }).catch((error) => undefined);
+  const torrent = {
+    infoHash: record.infoHash,
+    provider: NAME,
+    title: torrentFound.name,
+    size: record.size,
+    type: type,
+    uploadDate: record.uploadDate,
+    seeders: torrentFound.seeders,
+  };

  if (!imdbId) {
    console.log(`imdbId not found: ${torrentFound.name}`);
-    repository.updateTorrent({
-      infoHash: record.infoHash,
-      provider: NAME,
-      title: torrentFound.name,
-      uploadDate: record.uploadDate,
-      seeders: torrentFound.seeders,
-    });
+    repository.createFailedImdbTorrent(torrent);
    return;
  }

-  if (type === 'movie' || titleInfo.episode) {
-    repository.updateTorrent({
-      infoHash: record.infoHash,
-      provider: NAME,
-      title: torrentFound.name,
-      imdbId: imdbId,
-      uploadDate: record.uploadDate,
-      seeders: torrentFound.seeders,
-    });
-    return;
-  }
-
-  const files = await torrentFiles(record).catch(() => []);
+  const files = await parseTorrentFiles(torrent, imdbId);
  if (!files || !files.length) {
    console.log(`no video files found: ${torrentFound.name}`);
    return;
  }

-  repository.updateTorrent({
-    infoHash: record.infoHash,
-    provider: NAME,
-    title: torrentFound.name,
-    imdbId: imdbId,
-    uploadDate: record.uploadDate,
-    seeders: torrentFound.seeders,
-    files: files
-  })
+  repository.createTorrent(torrent)
+      .then(() => files.forEach(file => repository.createFile(file)));
+  console.log(`Created entry for ${torrentFound.name}`);
+}
+
+async function findTorrent(record) {
+  return findTorrentInSource(record)
+      .catch((error) => findTorrentViaBing(record));
+}
+
+async function findTorrentInSource(record) {
+  let page = 0;
+  let torrentFound;
+  while (!torrentFound && page < 5) {
+    const torrents = await pirata.search(record.title.replace(/[\W\s]+/, ' '), { page: page });
+    torrentFound = torrents.filter(torrent => torrent.magnetLink.toLowerCase().includes(record.infoHash))[0];
+    page = torrents.length === 0 ? 1000 : page + 1;
+  }
+  if (!torrentFound) {
+    return Promise.reject(new Error(`Failed to find torrent ${record.title}`));
+  }
+  return Promise.resolve(torrentFound);
+}
+
+async function findTorrentViaBing(record) {
+  return bing.web(`${record.infoHash}`)
+      .then((results) => results
+          .find(result => result.description.includes('Direct download via magnet link') || result.description.includes('Get this torrent')))
+      .then((result) => {
+        if (!result) {
+          throw new Error(`Failed to find torrent ${record.title}`);
+        }
+        return result.link.match(/torrent\/(\w+)\//)[1];
+      })
+      .then((torrentId) => pirata.torrent(torrentId))
 }

 function downloadDump(dump) {