small cleanup

2024-12-20 03:29:51 +00:00 · 2020-01-05 18:59:30 +01:00
parent d19d544859
commit 643917939b
5 changed files with 25 additions and 116 deletions
--- a/horrible_subs_mapping.json
+++ b/horrible_subs_mapping.json
@@ -185,7 +185,7 @@
  },
  "Bokura wa Minna Kawaisou": {
    "showId": "46",
-    "kitsu_id": "8536"
+    "kitsu_id": "8094"
  },
  "Bonjour Sweet Love Patisserie": {
    "showId": "47",
@@ -385,7 +385,7 @@
  },
  "Denpa Kyoushi": {
    "showId": "96",
-    "kitsu_id": "8791"
+    "kitsu_id": "10235"
  },
  "Detective Conan": {
    "showId": "97",
@@ -764,7 +764,7 @@
  },
  "Highschool DxD BorN": {
    "showId": "190",
-    "kitsu_id": "11598"
+    "kitsu_id": "8514"
  },
  "Hiiro no Kakera": {
    "showId": "191",
@@ -1036,7 +1036,7 @@
  },
  "Kyoukai no Rinne": {
    "showId": "258",
-    "kitsu_id": "8461"
+    "kitsu_id": [ "10018", "11366", "12561" ]
  },
  "Kyoukai Senjou no Horizon S2": {
    "showId": "259",
@@ -1596,7 +1596,7 @@
  },
  "Rokujouma no Shinryakusha": {
    "showId": "398",
-    "kitsu_id": "9063"
+    "kitsu_id": "8336"
  },
  "Rowdy Sumo Wrestler Matsutaro": {
    "showId": "399",
@@ -1740,7 +1740,7 @@
  },
  "Seraph of the End S2": {
    "showId": "434",
-    "kitsu_id": "10881"
+    "kitsu_id": "10082"
  },
  "Servant x Service": {
    "showId": "435",
@@ -2380,7 +2380,7 @@
  },
  "Mobile Suit Gundam Thunderbolt": {
    "showId": "602",
-    "kitsu_id": "11476"
+    "kitsu_id": ["11476", "12755"]
  },
  "Durarara!!": {
    "showId": "603",
@@ -2696,7 +2696,7 @@
  },
  "Big Order": {
    "showId": "688",
-    "kitsu_id": "10672"
+    "kitsu_id": "11455"
  },
  "Cardfight!! Vanguard G Stride Gate": {
    "showId": "689",
@@ -2712,7 +2712,7 @@
  },
  "Yowamushi Pedal Re RIDE": {
    "showId": "691",
-    "kitsu_id": "7693"
+    "kitsu_id": "8657"
  },
  "Yowamushi Pedal Re ROAD": {
    "showId": "691",
@@ -2900,7 +2900,7 @@
  },
  "Mob Pyscho 100 OVA": {
    "showId": "741",
-    "kitsu_id": "42416"
+    "kitsu_id": "41073"
  },
  "Mahou Shoujo Naria Girls": {
    "showId": "742",
@@ -3020,7 +3020,7 @@
  },
  "Nobunaga no Shinobi": {
    "showId": "773",
-    "kitsu_id": "11871"
+    "kitsu_id": [ "11871", "13303", "14207" ]
  },
  "Zutto Mae kara Suki deshita": {
    "showId": "774",
@@ -3152,7 +3152,7 @@
  },
  "Shelter": {
    "showId": "807",
-    "kitsu_id": "9215"
+    "imdb_id": "tt6443118"
  },
  "3-Nen D-Gumi Glass no Kamen": {
    "showId": "808",
@@ -3380,7 +3380,7 @@
  },
  "THE iDOLM@STER CINDERELLA GIRLS Theater (TV)": {
    "showId": "866",
-    "kitsu_id": "12671"
+    "kitsu_id": [ "12671","13895", "41377",  "42229" ]
  },
  "THE iDOLM@STER CINDERELLA GIRLS Theater (Web)": {
    "showId": "866",
@@ -3812,7 +3812,7 @@
  },
  "Yuki Yuna wa Yusha de Aru - Washio Sumi no Shou": {
    "showId": "976",
-    "kitsu_id": "12678"
+    "kitsu_id": [ "12678", "14029" ]
  },
  "ID-0": {
    "showId": "978",
@@ -3860,7 +3860,7 @@
  },
  "Houseki no Kuni": {
    "showId": "989",
-    "kitsu_id": "8774"
+    "kitsu_id": "13521"
  },
  "Kekkai Sensen & Beyond": {
    "showId": "990",
@@ -3868,7 +3868,7 @@
  },
  "Hozuki no Reitetsu S2": {
    "showId": "991",
-    "kitsu_id": "13226"
+    "kitsu_id": [ "13226", "14157" ]
  },
  "Cardfight!! Vanguard G Z": {
    "showId": "992",
@@ -4080,7 +4080,7 @@
  },
  "Beatless": {
    "showId": "1047",
-    "kitsu_id": "13939"
+    "kitsu_id": [ "13939", "41407" ]
  },
  "Zoku Touken Ranbu - Hanamaru": {
    "showId": "1048",
@@ -4140,7 +4140,7 @@
  },
  "Souten no Ken Re-Genesis": {
    "showId": "1062",
-    "kitsu_id": "13983"
+    "kitsu_id": [ "13983", "41953" ]
  },
  "Gundam Build Divers": {
    "showId": "1063",
@@ -4256,7 +4256,7 @@
  },
  "Chi's Sweet Adventure S2": {
    "showId": "1089",
-    "kitsu_id": "12154"
+    "kitsu_id": "13906"
  },
  "Golden Kamuy": {
    "showId": "1090",
@@ -4372,7 +4372,7 @@
  },
  "Ongaku Shoujo": {
    "showId": "1119",
-    "kitsu_id": "10422"
+    "kitsu_id": "41066"
  },
  "Yami Shibai - Japanese Ghost Stories 6": {
    "showId": "1121",
@@ -4864,7 +4864,7 @@
  },
  "Fairy Gone": {
    "showId": "1250",
-    "kitsu_id": "42130"
+    "kitsu_id": [ "42130", "42358" ]
  },
  "Shoumetsu Toshi": {
    "showId": "1251",
--- a/index.js
+++ b/index.js
@@ -1,7 +1,6 @@
 require('dotenv').config();
 const express = require("express");
 const server = express();
-const { init } = require('./lib/torrent');
 const { connect } = require('./lib/repository');
 const tpbDump = require('./scrapers/piratebay_dump');
 const horribleSubsScraper = require('./scrapers/horiblesubs_scraper');
@@ -19,7 +18,6 @@ server.post('/scrape', function(req, res) {

 server.listen(7000, async function () {
  await connect();
-  await init();
  console.log('Scraper started');
  scrape();
 });
--- a/lib/torrent.js
+++ b/lib/torrent.js
@@ -1,49 +1,18 @@
 const torrentStream = require('torrent-stream');
-const cheerio = require('cheerio');
 const needle = require('needle');
 const parseTorrent = require('parse-torrent');
-const Tracker = require("peer-search/tracker");
 const { retrieveTorrentFiles } = require('./cache');

 const MAX_PEER_CONNECTIONS = process.env.MAX_PEER_CONNECTIONS || 20;
 const EXTENSIONS = ["3g2", "3gp", "avi", "flv", "mkv", "mov", "mp2", "mp4", "mpe", "mpeg", "mpg", "mpv", "webm", "wmv"];
-let TRACKERS = [];
-
-module.exports.init = async function() {
-  TRACKERS = await getTrackerList();
-};

 module.exports.torrentFiles = function(torrent) {
  return filesFromTorrentFile(torrent)
-      .catch(() => filesFromKat(torrent.infoHash))
      .catch(() => filesFromTorrentStream(torrent))
      .catch(() => filesFromCache(torrent.infoHash))
      .then((files) => files.filter((file) => isVideo(file)));
 };

-module.exports.currentSeeders = function (torrent) {
-  if (!torrent.infoHash) {
-    return Promise.reject(new Error("no infoHash"));
-  }
-  return new Promise((resolve) =>
-    Promise.all(TRACKERS.map((tracker) => new Promise((resolve) => {
-          const t = new Tracker(tracker, { }, torrent.infoHash);
-          t.run();
-          t.on("info", (inf) => resolve([inf.seeders, inf.leechers]));
-          setTimeout(() => resolve([0, 0]), 1000);
-        }))
-    ).then((results) => resolve(results.reduce((seeders, next) => seeders + next[0], 0)))
-  );
-};
-
-// async function filesFromBtSeeds(infoHash) {
-//   const url = `https://www.btseed.net/show/${infoHash}`;
-//   return needle('get', url, { open_timeout: 2000 })
-//       .then((response) => response.body)
-//       .then((body) => body.match(/<script id="__NEXT_DATA__"[^>]+>(.*?)<\/script>/)[1])
-//       .then((match) => JSON.parse(match).props.pageProps.result.torrent.files)
-// }
-
 function filesFromCache(infoHash) {
  return retrieveTorrentFiles(infoHash)
      .then((files) => files.map((file) => ({
@@ -54,38 +23,6 @@ function filesFromCache(infoHash) {
      })));
 }

-function filesFromKat(infoHash) {
-  if (!infoHash) {
-    return Promise.reject(new Error("no infoHash"));
-  }
-  const url = `https://kat.rip/torrent/${infoHash}.html`;
-  return needle('get', url, { open_timeout: 2000 })
-      .then((response) => {
-        if (!response.body || response.statusCode !== 200) {
-          throw new Error('torrent not found in kat')
-        }
-        return response.body
-      })
-      .then((body) => {
-        const $ = cheerio.load(body);
-        const files = [];
-
-        $('table[id=\'ul_top\'] tr').each((index, row) => {
-          files.push({
-            fileIndex: index,
-            name: $(row).find('td[class=\'torFileName\']').text().replace(/.*\//, ''),
-            path: $(row).find('td[class=\'torFileName\']').text(),
-            size: convertToBytes($(row).find('td[class=\'torFileSize\']').text())
-          });
-        });
-
-        if (!files[files.length - 1].size) {
-          throw new Error('not full file list')
-        }
-        return files;
-      })
-}
-
 async function filesFromTorrentFile(torrent) {
  if (!torrent.torrentLink) {
    return Promise.reject(new Error("no torrentLink"));
@@ -137,19 +74,6 @@ function isVideo(title) {
  return EXTENSIONS.includes(title.path.match(/\.(\w{2,4})$/)[1]);
 }

-function convertToBytes(sizeString) {
-  if (!sizeString) {
-    return;
-  }
-  const prefix = sizeString.match(/\w+$/)[0].toLowerCase();
-  let multiplier = 1;
-  if (prefix === 'gb') multiplier = 1024 * 1024 * 1024;
-  else if (prefix === 'mb') multiplier = 1024 * 1024;
-  else if (prefix === 'kb') multiplier = 1024;
-
-  return Math.floor(parseFloat(sizeString) * multiplier);
-}
-
 function dynamicTimeout(torrent) {
  if (torrent.seeders < 5) {
    return 5000;
@@ -166,16 +90,3 @@ function dynamicTimeout(torrent) {
  }
 }

-function getTrackerList() {
-  return needle('get', 'https://torrents.me/tracker-list/', { open_timeout: 2000, follow_max: 2 })
-      .then((response) => {
-        if (!response.body || response.statusCode !== 200) {
-          throw new Error('tracker list not found')
-        }
-        return response.body
-      })
-      .then((body) => cheerio.load(body))
-      .then(($) => $('div[class="small-12 columns"] pre').text())
-      .then((text) => text.replace(/"/g, '').trim().split('\n'))
-}
-
--- a/package-lock.json
+++ b/package-lock.json
@@ -1714,7 +1714,7 @@
      }
    },
    "parse-torrent-title": {
-      "version": "git://github.com/TheBeastLT/parse-torrent-title.git#b85e60d555fac758b4827e0e130e723d66502afc",
+      "version": "git://github.com/TheBeastLT/parse-torrent-title.git#9f6abe9c606cc48d78962fe9496f2c34f1f7ea5c",
      "from": "git://github.com/TheBeastLT/parse-torrent-title.git#master"
    },
    "parseurl": {
--- a/scrapers/api/thepiratebay.js
+++ b/scrapers/api/thepiratebay.js
@@ -4,9 +4,9 @@ const moment = require('moment');

 const defaultProxies = [
    'https://thepiratebay.org',
-  'https://piratebays.life',
-  'https://piratebays.icu/',
-  'https://piratebays.cool'];
+  'https://piratebays.icu',
+  'https://piratebays.cool',
+  'https://piratebays.life'];
 const dumpUrl = '/static/dump/csv/';
 const defaultTimeout = 30000;