Initial commit

lib/metadata.js

@@ -0,0 +1,94 @@
+const _ = require('lodash');
+const needle = require('needle');
+const nameToImdb = require('name-to-imdb');
+
+const CINEMETA_URL = 'https://v3-cinemeta.strem.io';
+
+function getMetadata(imdbId, type) {
+  return needle('get', `${CINEMETA_URL}/meta/${type}/${imdbId}.json`, { open_timeout: 1000 })
+      .then((response) => response.body)
+      .then((body) => {
+        if (body && body.meta && body.meta.name) {
+          return {
+            title: body.meta.name,
+            year: body.meta.year,
+            genres: body.meta.genres,
+            episodeCount: body.meta.videos && _.chain(body.meta.videos)
+                .countBy('season')
+                .toPairs()
+                .filter((pair) => pair[0] !== '0')
+                .sortBy((pair) => parseInt(pair[0], 10))
+                .map((pair) => pair[1])
+                .value()
+          };
+        } else {
+          console.log(`failed cinemeta query: Empty Body`);
+          throw new Error('failed cinemeta query');
+        }
+      });
+}
+
+function escapeTitle(title, hyphenEscape = true) {
+  return title.toLowerCase()
+  .normalize('NFKD') // normalize non-ASCII characters
+  .replace(/[\u0300-\u036F]/g, '')
+  .replace(/&/g, 'and')
+  .replace(hyphenEscape ? /[.,_+ -]+/g : /[.,_+ ]+/g, ' ') // replace dots, commas or underscores with spaces
+  .replace(/[^\w- ()]/gi, '') // remove all non-alphanumeric chars
+  .trim();
+}
+
+const hardcodedTitles = {
+  'tt0388629': 'one piece',
+  'tt0182629': 'rurouni kenshin',
+  'tt2098220': 'hunter x hunter 2011',
+  'tt1409055': 'dragon ball kai',
+  'tt7441658': 'black clover tv'
+};
+
+async function seriesMetadata(id) {
+  const idInfo = id.split(':');
+  const imdbId = idInfo[0];
+  const season = parseInt(idInfo[1], 10);
+  const episode = parseInt(idInfo[2], 10);
+
+  const metadata = await getMetadata(imdbId, 'series');
+  const title = escapeTitle(metadata.title);
+  const hasEpisodeCount = metadata.episodeCount && metadata.episodeCount.length >= season;
+
+  return {
+    imdb: imdbId,
+    title: hardcodedTitles[imdbId] || title,
+    season: season,
+    episode: episode,
+    absoluteEpisode: hasEpisodeCount && metadata.episodeCount.slice(0, season - 1).reduce((a, b) => a + b, episode),
+    genres: metadata.genres,
+    isAnime: !metadata.genres.length || metadata.genres.includes('Animation')
+  };
+}
+
+async function movieMetadata(id) {
+  const metadata = await getMetadata(id, 'movie');
+
+  return {
+    imdb: id,
+    title: escapeTitle(metadata.title),
+    year: metadata.year,
+    genres: metadata.genres,
+    isAnime: !metadata.genres.length || metadata.genres.includes('Animation')
+  };
+}
+
+async function getImdbId(info) {
+  return new Promise((resolve, reject) => {
+    nameToImdb(info, function(err, res) {
+      if (res) {
+        resolve(res);
+      } else {
+        reject(err || new Error('failed imdbId search'));
+      }
+    });
+  });
+}
+
+module.exports = { movieMetadata, seriesMetadata, getImdbId };

lib/repository.js

@@ -0,0 +1,84 @@
+const { Sequelize }= require('sequelize');
+
+const POSTGRES_URI = process.env.POSTGRES_URI || 'postgres://torrentio:postgres@localhost:5432/torrentio';
+
+const database = new Sequelize(POSTGRES_URI, { logging: false });
+
+const Provider = database.define('provider', {
+  name: { type: Sequelize.STRING(16), primaryKey: true},
+  lastScraped: { type: Sequelize.DATE }
+});
+
+const Torrent = database.define('torrent', {
+  infoHash: { type: Sequelize.STRING(64), primaryKey: true },
+  provider: { type: Sequelize.STRING(16), allowNull: false },
+  title: { type: Sequelize.STRING(128), allowNull: false },
+  imdbId: { type: Sequelize.STRING(12) },
+  uploadDate: { type: Sequelize.DATE, allowNull: false },
+  seeders: { type: Sequelize.SMALLINT },
+  files: { type: Sequelize.ARRAY(Sequelize.TEXT) },
+});
+
+const SkipTorrent = database.define('skip_torrent', {
+  infoHash: {type: Sequelize.STRING(64), primaryKey: true},
+});
+
+const FailedImdbTorrent = database.define('failed_imdb_torrent', {
+  infoHash: {type: Sequelize.STRING(64), primaryKey: true},
+});
+
+function connect() {
+  return database.sync({ alter: true });
+}
+
+function getProvider(provider) {
+  return Provider.findOrCreate({ where: { name: provider.name }, defaults: provider });
+}
+
+function updateProvider(provider) {
+  return Provider.update(provider);
+}
+
+function getTorrent(torrent) {
+  return Torrent.findByPk(torrent.infoHash)
+      .then((result) =>{
+        if (!result) {
+          throw new Error(`torrent not found: ${torrent.infoHash}`);
+        }
+        return result.dataValues;
+      })
+}
+
+function updateTorrent(torrent) {
+  return Torrent.upsert(torrent);
+}
+
+function getSkipTorrent(torrent) {
+  return SkipTorrent.findByPk(torrent.infoHash)
+      .then((result) =>{
+        if (!result) {
+          return getFailedImdbTorrent(torrent);
+        }
+        return result.dataValues;
+      })
+}
+
+function createSkipTorrent(torrent) {
+  return SkipTorrent.upsert({ infoHash: torrent.infoHash });
+}
+
+function getFailedImdbTorrent(torrent) {
+  return FailedImdbTorrent.findByPk(torrent.infoHash)
+      .then((result) =>{
+        if (!result) {
+          throw new Error(`torrent not found: ${torrent.infoHash}`);
+        }
+        return result.dataValues;
+      })
+}
+
+function createFailedImdbTorrent(torrent) {
+  return FailedImdbTorrent.upsert({ infoHash: torrent.infoHash });
+}
+
+module.exports = { connect, getProvider, updateProvider, getTorrent, updateTorrent, getSkipTorrent, createSkipTorrent, createFailedImdbTorrent };

lib/torrent.js

@@ -0,0 +1,109 @@
+const torrentStream = require('torrent-stream');
+const cheerio = require('cheerio');
+const needle = require('needle');
+const parseTorrent = require('parse-torrent');
+const cloudscraper = require('cloudscraper');
+
+const MAX_PEER_CONNECTIONS = process.env.MAX_PEER_CONNECTIONS || 20;
+const EXTENSIONS = ["3g2", "3gp", "avi", "flv", "mkv", "mov", "mp2", "mp4", "mpe", "mpeg", "mpg", "mpv", "webm", "wmv"];
+
+module.exports.torrentFiles = function(torrent) {
+  return filesFromKat(torrent.infoHash)
+      .catch(() => filesFromTorrentStream(torrent))
+      .then((files) => files
+          .filter((file) => isVideo(file))
+          .map((file) => `${file.fileIndex}@@${file.path}`));
+};
+
+// async function filesFromBtSeeds(infoHash) {
+//   const url = `https://www.btseed.net/show/${infoHash}`;
+//   return needle('get', url, { open_timeout: 2000 })
+//       .then((response) => response.body)
+//       .then((body) => body.match(/<script id="__NEXT_DATA__"[^>]+>(.*?)<\/script>/)[1])
+//       .then((match) => JSON.parse(match).props.pageProps.result.torrent.files)
+// }
+
+function filesFromKat(infoHash) {
+  const url = `http://kat.rip/torrent/${infoHash}.html`;
+  return needle('get', url, { open_timeout: 2000 })
+      .then((response) => {
+        if (!response.body) {
+          throw new Error('torrent not found in kat')
+        }
+        return response.body
+      })
+      .then((body) => {
+        const $ = cheerio.load(body);
+        const files = [];
+
+        $('table[id=\'ul_top\'] tr').each((index, row) => {
+          files.push({
+            fileIndex: index,
+            path: $(row).find('td[class=\'torFileName\']').text(),
+            size: convertToBytes($(row).find('td[class=\'torFileSize\']').text())
+          });
+        });
+        return files;
+      })
+}
+
+async function filesFromTorrentStream(torrent) {
+  return new Promise((resolve, rejected) => {
+    const engine = new torrentStream(torrent.infoHash, { connections: MAX_PEER_CONNECTIONS });
+
+    engine.ready(() => {
+      const files = engine.files
+          .map((file, fileId) => ({
+            fileIndex: fileId,
+            name: file.name,
+            path: file.path.replace(/^[^\/]+\//, ''),
+            size: file.length
+          }));
+
+      engine.destroy();
+      resolve(files);
+    });
+    setTimeout(() => {
+      engine.destroy();
+      rejected(new Error('No available connections for torrent!'));
+    }, dynamicTimeout(torrent));
+  });
+}
+
+function isVideo(title) {
+  return EXTENSIONS.includes(title.path.match(/\.(\w{2,4})$/)[1]);
+}
+
+
+function convertToBytes(sizeString) {
+  if (!sizeString) {
+    return;
+  }
+  const prefix = sizeString.match(/\w+$/)[0].toLowerCase();
+  let multiplier = 1;
+  if (prefix === 'gb') multiplier = 1024 * 1024 * 1024;
+  else if (prefix === 'mb') multiplier = 1024 * 1024;
+  else if (prefix === 'kb') multiplier = 1024;
+
+  return Math.floor(parseFloat(sizeString) * multiplier);
+}
+
+
+function dynamicTimeout(torrent) {
+  if (torrent.seeders < 5) {
+    return 2000;
+  } else if (torrent.seeders < 10) {
+    return 3000;
+  } else if (torrent.seeders < 20) {
+    return 4000;
+  } else if (torrent.seeders < 30) {
+    return 5000;
+  } else if (torrent.seeders < 50) {
+    return 7000;
+  } else if (torrent.seeders < 100) {
+    return 10000;
+  } else {
+    return 15000;
+  }
+}
+