small cleanup

horrible_subs_mapping.json

@@ -185,7 +185,7 @@
   },
   "Bokura wa Minna Kawaisou": {
     "showId": "46",
-    "kitsu_id": "8536"
+    "kitsu_id": "8094"
   },
   "Bonjour Sweet Love Patisserie": {
     "showId": "47",
@@ -385,7 +385,7 @@
   },
   "Denpa Kyoushi": {
     "showId": "96",
-    "kitsu_id": "8791"
+    "kitsu_id": "10235"
   },
   "Detective Conan": {
     "showId": "97",
@@ -764,7 +764,7 @@
   },
   "Highschool DxD BorN": {
     "showId": "190",
-    "kitsu_id": "11598"
+    "kitsu_id": "8514"
   },
   "Hiiro no Kakera": {
     "showId": "191",
@@ -1036,7 +1036,7 @@
   },
   "Kyoukai no Rinne": {
     "showId": "258",
-    "kitsu_id": "8461"
+    "kitsu_id": [ "10018", "11366", "12561" ]
   },
   "Kyoukai Senjou no Horizon S2": {
     "showId": "259",
@@ -1596,7 +1596,7 @@
   },
   "Rokujouma no Shinryakusha": {
     "showId": "398",
-    "kitsu_id": "9063"
+    "kitsu_id": "8336"
   },
   "Rowdy Sumo Wrestler Matsutaro": {
     "showId": "399",
@@ -1740,7 +1740,7 @@
   },
   "Seraph of the End S2": {
     "showId": "434",
-    "kitsu_id": "10881"
+    "kitsu_id": "10082"
   },
   "Servant x Service": {
     "showId": "435",
@@ -2380,7 +2380,7 @@
   },
   "Mobile Suit Gundam Thunderbolt": {
     "showId": "602",
-    "kitsu_id": "11476"
+    "kitsu_id": ["11476", "12755"]
   },
   "Durarara!!": {
     "showId": "603",
@@ -2696,7 +2696,7 @@
   },
   "Big Order": {
     "showId": "688",
-    "kitsu_id": "10672"
+    "kitsu_id": "11455"
   },
   "Cardfight!! Vanguard G Stride Gate": {
     "showId": "689",
@@ -2712,7 +2712,7 @@
   },
   "Yowamushi Pedal Re RIDE": {
     "showId": "691",
-    "kitsu_id": "7693"
+    "kitsu_id": "8657"
   },
   "Yowamushi Pedal Re ROAD": {
     "showId": "691",
@@ -2900,7 +2900,7 @@
   },
   "Mob Pyscho 100 OVA": {
     "showId": "741",
-    "kitsu_id": "42416"
+    "kitsu_id": "41073"
   },
   "Mahou Shoujo Naria Girls": {
     "showId": "742",
@@ -3020,7 +3020,7 @@
   },
   "Nobunaga no Shinobi": {
     "showId": "773",
-    "kitsu_id": "11871"
+    "kitsu_id": [ "11871", "13303", "14207" ]
   },
   "Zutto Mae kara Suki deshita": {
     "showId": "774",
@@ -3152,7 +3152,7 @@
   },
   "Shelter": {
     "showId": "807",
-    "kitsu_id": "9215"
+    "imdb_id": "tt6443118"
   },
   "3-Nen D-Gumi Glass no Kamen": {
     "showId": "808",
@@ -3380,7 +3380,7 @@
   },
   "THE iDOLM@STER CINDERELLA GIRLS Theater (TV)": {
     "showId": "866",
-    "kitsu_id": "12671"
+    "kitsu_id": [ "12671","13895", "41377",  "42229" ]
   },
   "THE iDOLM@STER CINDERELLA GIRLS Theater (Web)": {
     "showId": "866",
@@ -3812,7 +3812,7 @@
   },
   "Yuki Yuna wa Yusha de Aru - Washio Sumi no Shou": {
     "showId": "976",
-    "kitsu_id": "12678"
+    "kitsu_id": [ "12678", "14029" ]
   },
   "ID-0": {
     "showId": "978",
@@ -3860,7 +3860,7 @@
   },
   "Houseki no Kuni": {
     "showId": "989",
-    "kitsu_id": "8774"
+    "kitsu_id": "13521"
   },
   "Kekkai Sensen & Beyond": {
     "showId": "990",
@@ -3868,7 +3868,7 @@
   },
   "Hozuki no Reitetsu S2": {
     "showId": "991",
-    "kitsu_id": "13226"
+    "kitsu_id": [ "13226", "14157" ]
   },
   "Cardfight!! Vanguard G Z": {
     "showId": "992",
@@ -4080,7 +4080,7 @@
   },
   "Beatless": {
     "showId": "1047",
-    "kitsu_id": "13939"
+    "kitsu_id": [ "13939", "41407" ]
   },
   "Zoku Touken Ranbu - Hanamaru": {
     "showId": "1048",
@@ -4140,7 +4140,7 @@
   },
   "Souten no Ken Re-Genesis": {
     "showId": "1062",
-    "kitsu_id": "13983"
+    "kitsu_id": [ "13983", "41953" ]
   },
   "Gundam Build Divers": {
     "showId": "1063",
@@ -4256,7 +4256,7 @@
   },
   "Chi's Sweet Adventure S2": {
     "showId": "1089",
-    "kitsu_id": "12154"
+    "kitsu_id": "13906"
   },
   "Golden Kamuy": {
     "showId": "1090",
@@ -4372,7 +4372,7 @@
   },
   "Ongaku Shoujo": {
     "showId": "1119",
-    "kitsu_id": "10422"
+    "kitsu_id": "41066"
   },
   "Yami Shibai - Japanese Ghost Stories 6": {
     "showId": "1121",
@@ -4864,7 +4864,7 @@
   },
   "Fairy Gone": {
     "showId": "1250",
-    "kitsu_id": "42130"
+    "kitsu_id": [ "42130", "42358" ]
   },
   "Shoumetsu Toshi": {
     "showId": "1251",

index.js

@@ -1,7 +1,6 @@
 require('dotenv').config();
 const express = require("express");
 const server = express();
-const { init } = require('./lib/torrent');
 const { connect } = require('./lib/repository');
 const tpbDump = require('./scrapers/piratebay_dump');
 const horribleSubsScraper = require('./scrapers/horiblesubs_scraper');
@@ -19,7 +18,6 @@ server.post('/scrape', function(req, res) {
 
 server.listen(7000, async function () {
   await connect();
-  await init();
   console.log('Scraper started');
   scrape();
 });

lib/torrent.js

@@ -1,49 +1,18 @@
 const torrentStream = require('torrent-stream');
-const cheerio = require('cheerio');
 const needle = require('needle');
 const parseTorrent = require('parse-torrent');
-const Tracker = require("peer-search/tracker");
 const { retrieveTorrentFiles } = require('./cache');
 
 const MAX_PEER_CONNECTIONS = process.env.MAX_PEER_CONNECTIONS || 20;
 const EXTENSIONS = ["3g2", "3gp", "avi", "flv", "mkv", "mov", "mp2", "mp4", "mpe", "mpeg", "mpg", "mpv", "webm", "wmv"];
-let TRACKERS = [];
-
-module.exports.init = async function() {
-  TRACKERS = await getTrackerList();
-};
 
 module.exports.torrentFiles = function(torrent) {
   return filesFromTorrentFile(torrent)
-      .catch(() => filesFromKat(torrent.infoHash))
       .catch(() => filesFromTorrentStream(torrent))
       .catch(() => filesFromCache(torrent.infoHash))
       .then((files) => files.filter((file) => isVideo(file)));
 };
 
-module.exports.currentSeeders = function (torrent) {
-  if (!torrent.infoHash) {
-    return Promise.reject(new Error("no infoHash"));
-  }
-  return new Promise((resolve) =>
-    Promise.all(TRACKERS.map((tracker) => new Promise((resolve) => {
-          const t = new Tracker(tracker, { }, torrent.infoHash);
-          t.run();
-          t.on("info", (inf) => resolve([inf.seeders, inf.leechers]));
-          setTimeout(() => resolve([0, 0]), 1000);
-        }))
-    ).then((results) => resolve(results.reduce((seeders, next) => seeders + next[0], 0)))
-  );
-};
-
-// async function filesFromBtSeeds(infoHash) {
-//   const url = `https://www.btseed.net/show/${infoHash}`;
-//   return needle('get', url, { open_timeout: 2000 })
-//       .then((response) => response.body)
-//       .then((body) => body.match(/<script id="__NEXT_DATA__"[^>]+>(.*?)<\/script>/)[1])
-//       .then((match) => JSON.parse(match).props.pageProps.result.torrent.files)
-// }
-
 function filesFromCache(infoHash) {
   return retrieveTorrentFiles(infoHash)
       .then((files) => files.map((file) => ({
@@ -54,38 +23,6 @@ function filesFromCache(infoHash) {
       })));
 }
 
-function filesFromKat(infoHash) {
-  if (!infoHash) {
-    return Promise.reject(new Error("no infoHash"));
-  }
-  const url = `https://kat.rip/torrent/${infoHash}.html`;
-  return needle('get', url, { open_timeout: 2000 })
-      .then((response) => {
-        if (!response.body || response.statusCode !== 200) {
-          throw new Error('torrent not found in kat')
-        }
-        return response.body
-      })
-      .then((body) => {
-        const $ = cheerio.load(body);
-        const files = [];
-
-        $('table[id=\'ul_top\'] tr').each((index, row) => {
-          files.push({
-            fileIndex: index,
-            name: $(row).find('td[class=\'torFileName\']').text().replace(/.*\//, ''),
-            path: $(row).find('td[class=\'torFileName\']').text(),
-            size: convertToBytes($(row).find('td[class=\'torFileSize\']').text())
-          });
-        });
-
-        if (!files[files.length - 1].size) {
-          throw new Error('not full file list')
-        }
-        return files;
-      })
-}
-
 async function filesFromTorrentFile(torrent) {
   if (!torrent.torrentLink) {
     return Promise.reject(new Error("no torrentLink"));
@@ -137,19 +74,6 @@ function isVideo(title) {
   return EXTENSIONS.includes(title.path.match(/\.(\w{2,4})$/)[1]);
 }
 
-function convertToBytes(sizeString) {
-  if (!sizeString) {
-    return;
-  }
-  const prefix = sizeString.match(/\w+$/)[0].toLowerCase();
-  let multiplier = 1;
-  if (prefix === 'gb') multiplier = 1024 * 1024 * 1024;
-  else if (prefix === 'mb') multiplier = 1024 * 1024;
-  else if (prefix === 'kb') multiplier = 1024;
-
-  return Math.floor(parseFloat(sizeString) * multiplier);
-}
-
 function dynamicTimeout(torrent) {
   if (torrent.seeders < 5) {
     return 5000;
@@ -166,16 +90,3 @@ function dynamicTimeout(torrent) {
   }
 }
 
-function getTrackerList() {
-  return needle('get', 'https://torrents.me/tracker-list/', { open_timeout: 2000, follow_max: 2 })
-      .then((response) => {
-        if (!response.body || response.statusCode !== 200) {
-          throw new Error('tracker list not found')
-        }
-        return response.body
-      })
-      .then((body) => cheerio.load(body))
-      .then(($) => $('div[class="small-12 columns"] pre').text())
-      .then((text) => text.replace(/"/g, '').trim().split('\n'))
-}
-

package-lock.json

@@ -1714,7 +1714,7 @@
       }
     },
     "parse-torrent-title": {
-      "version": "git://github.com/TheBeastLT/parse-torrent-title.git#b85e60d555fac758b4827e0e130e723d66502afc",
+      "version": "git://github.com/TheBeastLT/parse-torrent-title.git#9f6abe9c606cc48d78962fe9496f2c34f1f7ea5c",
       "from": "git://github.com/TheBeastLT/parse-torrent-title.git#master"
     },
     "parseurl": {

scrapers/api/thepiratebay.js

@@ -4,9 +4,9 @@ const moment = require('moment');
 
 const defaultProxies = [
     'https://thepiratebay.org',
-  'https://piratebays.life',
+  'https://piratebays.icu',
-  'https://piratebays.icu/',
+  'https://piratebays.cool',
-  'https://piratebays.cool'];
+  'https://piratebays.life'];
 const dumpUrl = '/static/dump/csv/';
 const defaultTimeout = 30000;