Initial commit

.gitignore

@@ -0,0 +1,2 @@
+/.idea
+/node_modules

README.md

@@ -0,0 +1 @@
+# Torrentio Scraper

docker-compose.yml

@@ -0,0 +1,16 @@
+version: '3'
+services:
+  database:
+    image: postgres
+    volumes:
+    - db-data:/var/lib/postgresql/data
+    ports:
+    - 5432:5432
+    environment:
+      POSTGRES_USER: torrentio
+      POSTGRES_PASSWORD: postgres
+      POSTGRES_DB: torrentio
+
+volumes:
+  db-data:
+    driver: local

index.js

@@ -0,0 +1,24 @@
+const express = require("express");
+const server = express();
+const { connect } = require('./lib/repository');
+const tpbDump = require('./scrapers/piratebay_dump');
+const horribleSubs = require('./scrapers/api/horriblesubs');
+
+const providers = [tpbDump];
+
+async function scrape() {
+  providers.forEach((provider) => provider.scrape());
+}
+
+server.post('/scrape', function(req, res) {
+  scrape();
+  res.send(200);
+});
+
+server.listen(7000, async function () {
+  await connect();
+  console.log('Scraper started');
+  const shows = await horribleSubs.allShows();
+  console.log(shows)
+  //scrape();
+});

lib/metadata.js

@@ -0,0 +1,94 @@
+const _ = require('lodash');
+const needle = require('needle');
+const nameToImdb = require('name-to-imdb');
+
+const CINEMETA_URL = 'https://v3-cinemeta.strem.io';
+
+function getMetadata(imdbId, type) {
+  return needle('get', `${CINEMETA_URL}/meta/${type}/${imdbId}.json`, { open_timeout: 1000 })
+      .then((response) => response.body)
+      .then((body) => {
+        if (body && body.meta && body.meta.name) {
+          return {
+            title: body.meta.name,
+            year: body.meta.year,
+            genres: body.meta.genres,
+            episodeCount: body.meta.videos && _.chain(body.meta.videos)
+                .countBy('season')
+                .toPairs()
+                .filter((pair) => pair[0] !== '0')
+                .sortBy((pair) => parseInt(pair[0], 10))
+                .map((pair) => pair[1])
+                .value()
+          };
+        } else {
+          console.log(`failed cinemeta query: Empty Body`);
+          throw new Error('failed cinemeta query');
+        }
+      });
+}
+
+function escapeTitle(title, hyphenEscape = true) {
+  return title.toLowerCase()
+  .normalize('NFKD') // normalize non-ASCII characters
+  .replace(/[\u0300-\u036F]/g, '')
+  .replace(/&/g, 'and')
+  .replace(hyphenEscape ? /[.,_+ -]+/g : /[.,_+ ]+/g, ' ') // replace dots, commas or underscores with spaces
+  .replace(/[^\w- ()]/gi, '') // remove all non-alphanumeric chars
+  .trim();
+}
+
+const hardcodedTitles = {
+  'tt0388629': 'one piece',
+  'tt0182629': 'rurouni kenshin',
+  'tt2098220': 'hunter x hunter 2011',
+  'tt1409055': 'dragon ball kai',
+  'tt7441658': 'black clover tv'
+};
+
+async function seriesMetadata(id) {
+  const idInfo = id.split(':');
+  const imdbId = idInfo[0];
+  const season = parseInt(idInfo[1], 10);
+  const episode = parseInt(idInfo[2], 10);
+
+  const metadata = await getMetadata(imdbId, 'series');
+  const title = escapeTitle(metadata.title);
+  const hasEpisodeCount = metadata.episodeCount && metadata.episodeCount.length >= season;
+
+  return {
+    imdb: imdbId,
+    title: hardcodedTitles[imdbId] || title,
+    season: season,
+    episode: episode,
+    absoluteEpisode: hasEpisodeCount && metadata.episodeCount.slice(0, season - 1).reduce((a, b) => a + b, episode),
+    genres: metadata.genres,
+    isAnime: !metadata.genres.length || metadata.genres.includes('Animation')
+  };
+}
+
+async function movieMetadata(id) {
+  const metadata = await getMetadata(id, 'movie');
+
+  return {
+    imdb: id,
+    title: escapeTitle(metadata.title),
+    year: metadata.year,
+    genres: metadata.genres,
+    isAnime: !metadata.genres.length || metadata.genres.includes('Animation')
+  };
+}
+
+async function getImdbId(info) {
+  return new Promise((resolve, reject) => {
+    nameToImdb(info, function(err, res) {
+      if (res) {
+        resolve(res);
+      } else {
+        reject(err || new Error('failed imdbId search'));
+      }
+    });
+  });
+}
+
+module.exports = { movieMetadata, seriesMetadata, getImdbId };

lib/repository.js

@@ -0,0 +1,84 @@
+const { Sequelize }= require('sequelize');
+
+const POSTGRES_URI = process.env.POSTGRES_URI || 'postgres://torrentio:postgres@localhost:5432/torrentio';
+
+const database = new Sequelize(POSTGRES_URI, { logging: false });
+
+const Provider = database.define('provider', {
+  name: { type: Sequelize.STRING(16), primaryKey: true},
+  lastScraped: { type: Sequelize.DATE }
+});
+
+const Torrent = database.define('torrent', {
+  infoHash: { type: Sequelize.STRING(64), primaryKey: true },
+  provider: { type: Sequelize.STRING(16), allowNull: false },
+  title: { type: Sequelize.STRING(128), allowNull: false },
+  imdbId: { type: Sequelize.STRING(12) },
+  uploadDate: { type: Sequelize.DATE, allowNull: false },
+  seeders: { type: Sequelize.SMALLINT },
+  files: { type: Sequelize.ARRAY(Sequelize.TEXT) },
+});
+
+const SkipTorrent = database.define('skip_torrent', {
+  infoHash: {type: Sequelize.STRING(64), primaryKey: true},
+});
+
+const FailedImdbTorrent = database.define('failed_imdb_torrent', {
+  infoHash: {type: Sequelize.STRING(64), primaryKey: true},
+});
+
+function connect() {
+  return database.sync({ alter: true });
+}
+
+function getProvider(provider) {
+  return Provider.findOrCreate({ where: { name: provider.name }, defaults: provider });
+}
+
+function updateProvider(provider) {
+  return Provider.update(provider);
+}
+
+function getTorrent(torrent) {
+  return Torrent.findByPk(torrent.infoHash)
+      .then((result) =>{
+        if (!result) {
+          throw new Error(`torrent not found: ${torrent.infoHash}`);
+        }
+        return result.dataValues;
+      })
+}
+
+function updateTorrent(torrent) {
+  return Torrent.upsert(torrent);
+}
+
+function getSkipTorrent(torrent) {
+  return SkipTorrent.findByPk(torrent.infoHash)
+      .then((result) =>{
+        if (!result) {
+          return getFailedImdbTorrent(torrent);
+        }
+        return result.dataValues;
+      })
+}
+
+function createSkipTorrent(torrent) {
+  return SkipTorrent.upsert({ infoHash: torrent.infoHash });
+}
+
+function getFailedImdbTorrent(torrent) {
+  return FailedImdbTorrent.findByPk(torrent.infoHash)
+      .then((result) =>{
+        if (!result) {
+          throw new Error(`torrent not found: ${torrent.infoHash}`);
+        }
+        return result.dataValues;
+      })
+}
+
+function createFailedImdbTorrent(torrent) {
+  return FailedImdbTorrent.upsert({ infoHash: torrent.infoHash });
+}
+
+module.exports = { connect, getProvider, updateProvider, getTorrent, updateTorrent, getSkipTorrent, createSkipTorrent, createFailedImdbTorrent };

lib/torrent.js

@@ -0,0 +1,109 @@
+const torrentStream = require('torrent-stream');
+const cheerio = require('cheerio');
+const needle = require('needle');
+const parseTorrent = require('parse-torrent');
+const cloudscraper = require('cloudscraper');
+
+const MAX_PEER_CONNECTIONS = process.env.MAX_PEER_CONNECTIONS || 20;
+const EXTENSIONS = ["3g2", "3gp", "avi", "flv", "mkv", "mov", "mp2", "mp4", "mpe", "mpeg", "mpg", "mpv", "webm", "wmv"];
+
+module.exports.torrentFiles = function(torrent) {
+  return filesFromKat(torrent.infoHash)
+      .catch(() => filesFromTorrentStream(torrent))
+      .then((files) => files
+          .filter((file) => isVideo(file))
+          .map((file) => `${file.fileIndex}@@${file.path}`));
+};
+
+// async function filesFromBtSeeds(infoHash) {
+//   const url = `https://www.btseed.net/show/${infoHash}`;
+//   return needle('get', url, { open_timeout: 2000 })
+//       .then((response) => response.body)
+//       .then((body) => body.match(/<script id="__NEXT_DATA__"[^>]+>(.*?)<\/script>/)[1])
+//       .then((match) => JSON.parse(match).props.pageProps.result.torrent.files)
+// }
+
+function filesFromKat(infoHash) {
+  const url = `http://kat.rip/torrent/${infoHash}.html`;
+  return needle('get', url, { open_timeout: 2000 })
+      .then((response) => {
+        if (!response.body) {
+          throw new Error('torrent not found in kat')
+        }
+        return response.body
+      })
+      .then((body) => {
+        const $ = cheerio.load(body);
+        const files = [];
+
+        $('table[id=\'ul_top\'] tr').each((index, row) => {
+          files.push({
+            fileIndex: index,
+            path: $(row).find('td[class=\'torFileName\']').text(),
+            size: convertToBytes($(row).find('td[class=\'torFileSize\']').text())
+          });
+        });
+        return files;
+      })
+}
+
+async function filesFromTorrentStream(torrent) {
+  return new Promise((resolve, rejected) => {
+    const engine = new torrentStream(torrent.infoHash, { connections: MAX_PEER_CONNECTIONS });
+
+    engine.ready(() => {
+      const files = engine.files
+          .map((file, fileId) => ({
+            fileIndex: fileId,
+            name: file.name,
+            path: file.path.replace(/^[^\/]+\//, ''),
+            size: file.length
+          }));
+
+      engine.destroy();
+      resolve(files);
+    });
+    setTimeout(() => {
+      engine.destroy();
+      rejected(new Error('No available connections for torrent!'));
+    }, dynamicTimeout(torrent));
+  });
+}
+
+function isVideo(title) {
+  return EXTENSIONS.includes(title.path.match(/\.(\w{2,4})$/)[1]);
+}
+
+
+function convertToBytes(sizeString) {
+  if (!sizeString) {
+    return;
+  }
+  const prefix = sizeString.match(/\w+$/)[0].toLowerCase();
+  let multiplier = 1;
+  if (prefix === 'gb') multiplier = 1024 * 1024 * 1024;
+  else if (prefix === 'mb') multiplier = 1024 * 1024;
+  else if (prefix === 'kb') multiplier = 1024;
+
+  return Math.floor(parseFloat(sizeString) * multiplier);
+}
+
+
+function dynamicTimeout(torrent) {
+  if (torrent.seeders < 5) {
+    return 2000;
+  } else if (torrent.seeders < 10) {
+    return 3000;
+  } else if (torrent.seeders < 20) {
+    return 4000;
+  } else if (torrent.seeders < 30) {
+    return 5000;
+  } else if (torrent.seeders < 50) {
+    return 7000;
+  } else if (torrent.seeders < 100) {
+    return 10000;
+  } else {
+    return 15000;
+  }
+}
+

package.json

@@ -0,0 +1,35 @@
+{
+  "name": "stremio-torrention",
+  "version": "1.0.0",
+  "main": "index.js",
+  "engines": {
+    "npm": "6.x",
+    "node": "10.x"
+  },
+  "scripts": {
+    "start": "node index.js"
+  },
+  "author": "TheBeastLT <pauliox@beyond.lt>",
+  "license": "MIT",
+  "dependencies": {
+    "bottleneck": "^2.16.2",
+    "cheerio": "^0.22.0",
+    "cloudscraper": "^3.0.0",
+    "express": "^4.16.4",
+    "imdb": "^1.1.0",
+    "is-video": "^1.0.1",
+    "line-by-line": "^0.1.6",
+    "lodash": "^4.17.11",
+    "magnet-uri": "^5.1.7",
+    "moment": "^2.24.0",
+    "name-to-imdb": "^2.3.0",
+    "needle": "^2.2.4",
+    "node-gzip": "^1.1.2",
+    "parse-torrent": "^6.1.2",
+    "parse-torrent-title": "git://github.com/TheBeastLT/parse-torrent-title.git#master",
+    "pg": "^7.8.2",
+    "pg-hstore": "^2.3.2",
+    "sequelize": "^4.43.0",
+    "torrent-stream": "^1.1.0"
+  }
+}

scrapers/api/horriblesubs.js

@@ -0,0 +1,42 @@
+const cheerio = require('cheerio');
+const needle = require('needle');
+const moment = require('moment');
+
+const defaultUrl = 'https://horriblesubs.info';
+const defaultTimeout = 5000;
+
+function _getContent(url, config = {},) {
+  const baseUrl = config.proxyUrl || defaultUrl;
+  const timeout = config.timeout || defaultTimeout;
+
+  return needle('get', `${baseUrl}${url}`, { open_timeout: timeout, follow: 2 })
+      .then((response) => response.body)
+      .then((body) => cheerio.load(body))
+}
+
+function _getAnimeId(showInfo) {
+  return _getContent(showInfo.url).then($ => {
+    const text = $('div.entry-content').find('script').html();
+    showInfo.id = text.match(/var hs_showid = (\d+)/)[1];
+    return showInfo
+  })
+}
+
+function allShows(config = {}) {
+  return _getContent('/shows', config)
+      .then(($) =>  $('div[class=\'ind-show\']')
+          .map((index, element) => $(element).children('a'))
+          .map((index, element) => ({
+            title: element.attr('title'),
+            url: `${config.proxyUrl || defaultUrl}${element.attr('href')}`
+          })).get());
+}
+
+function showData(showInfo) {
+  return _getAnimeId(showInfo)
+      .then((showInfo) => )
+
+}
+
+module.exports = { allShows };
+

scrapers/api/thepiratebay.js

@@ -0,0 +1,179 @@
+const cheerio = require('cheerio');
+const needle = require('needle');
+const moment = require('moment');
+
+const defaultProxies = ['https://pirateproxy.sh', 'https://thepiratebay.org'];
+const dumpUrl = '/static/dump/csv/';
+const defaultTimeout = 5000;
+
+const errors = {
+  REQUEST_ERROR: { code: 'REQUEST_ERROR' },
+  PARSER_ERROR: { code: 'PARSER_ERROR' }
+};
+
+Categories = {
+  AUDIO: {
+    ALL: 100,
+    MUSIC: 101,
+    AUDIO_BOOKS: 102,
+    SOUND_CLIPS: 103,
+    FLAC: 104,
+    OTHER: 199
+  },
+  VIDEO: {
+    ALL: 200,
+    MOVIES: 201,
+    MOVIES_DVDR: 202,
+    MUSIC_VIDEOS: 203,
+    MOVIE_CLIPS: 204,
+    TV_SHOWS: 205,
+    HANDHELD: 206,
+    MOVIES_HD: 207,
+    TV_SHOWS_HD: 208,
+    MOVIES_3D: 209,
+    OTHER: 299
+  },
+  APPS: {
+    ALL: 300,
+    WINDOWS: 301,
+    MAC: 302,
+    UNIX: 303,
+    HANDHELD: 304,
+    IOS: 305,
+    ANDROID: 306,
+    OTHER_OS: 399
+  },
+  GAMES: {
+    ALL: 400,
+    PC: 401,
+    MAC: 402,
+    PSx: 403,
+    XBOX360: 404,
+    Wii: 405,
+    HANDHELD: 406,
+    IOS: 407,
+    ANDROID: 408,
+    OTHER: 499
+  },
+  PORN: {
+    ALL: 500,
+    MOVIES: 501,
+    MOVIES_DVDR: 502,
+    PICTURES: 503,
+    GAMES: 504,
+    MOVIES_HD: 505,
+    MOVIE_CLIPS: 506,
+    OTHER: 599
+  },
+  OTHER: {
+    ALL: 600,
+    E_BOOKS: 601,
+    COMICS: 602,
+    PICTURES: 603,
+    COVERS: 604,
+    PHYSIBLES: 605,
+    OTHER: 699
+  }
+};
+
+function search(keyword, config = {}, retries = 2) {
+  if (!keyword || retries === 0) {
+    return Promise.reject(new Error(`Failed ${keyword} search`));
+  }
+  const proxyList = config.proxyList || defaultProxies;
+  const page = config.page || 0;
+  const category = config.cat || 0;
+
+  return raceFirstSuccessful(proxyList
+      .map((proxyUrl) => singleRequest(`${proxyUrl}/search/${keyword}/${page}/99/${category}`, config)))
+      .then((body) => parseBody(body))
+      .catch(() => search(keyword, config, retries - 1));
+}
+
+function dumps(config = {}, retries = 2) {
+  if (retries === 0) {
+    return Promise.reject(new Error(`Failed dump search`));
+  }
+  const proxyList = config.proxyList || defaultProxies;
+
+  return raceFirstSuccessful(proxyList
+      .map((proxyUrl) => singleRequest(`${proxyUrl}${dumpUrl}`, config)
+          .then((body) => body.match(/(<a href="[^"]+">[^<]+<\/a>.+\d)/g)
+              .map((group) => ({
+                url: `${proxyUrl}${dumpUrl}` + group.match(/<a href="([^"]+)">/)[1],
+                updatedAt: moment(group.match(/\s+([\w-]+\s+[\d:]+)\s+\d+$/)[1], 'DD-MMM-YYYY HH:mm').toDate()
+              })))))
+      .catch(() => dumps(config, retries - 1));
+}
+
+function singleRequest(requestUrl, config = {}) {
+  const timeout = config.timeout || defaultTimeout;
+
+  return new Promise(((resolve, reject) => {
+    needle.get(requestUrl,
+        { open_timeout: timeout, follow: 2 },
+        (err, res, body) => {
+          if (err || !body) {
+            reject(err || errors.REQUEST_ERROR);
+          } else if (body.includes('Access Denied') && !body.includes('<title>The Pirate Bay')) {
+            console.log(`Access Denied: ${url}`);
+            reject(new Error(`Access Denied: ${url}`));
+          } else if (body.includes('502: Bad gateway') ||
+            body.includes('403 Forbidden') ||
+            body.includes('Database maintenance') ||
+            body.includes('Origin DNS error') ||
+            !body.includes('<title>The Pirate Bay')) {
+            reject(errors.REQUEST_ERROR);
+          }
+
+          resolve(body);
+        });
+  }));
+}
+
+function parseBody(body) {
+  return new Promise((resolve, reject) => {
+    const $ = cheerio.load(body);
+
+    if (!$) {
+      reject(new Error(errors.PARSER_ERROR));
+    }
+
+    const torrents = [];
+
+    $('table[id=\'searchResult\'] tr').each(function() {
+      const name = $(this).find('.detLink').text();
+      if (!name) {
+        return;
+      }
+      torrents.push({
+        name: name,
+        seeders: parseInt($(this).find('td[align=\'right\']').eq(0).text(), 10),
+        leechers: parseInt($(this).find('td[align=\'right\']').eq(1).text(), 10),
+        magnetLink: $(this).find('a[title=\'Download this torrent using magnet\']').attr('href'),
+        category: parseInt($(this).find('a[title=\'More from this category\']').eq(0).attr('href').match(/\d+$/)[0], 10),
+        subcategory: parseInt($(this).find('a[title=\'More from this category\']').eq(1).attr('href').match(/\d+$/)[0], 10)
+      });
+    });
+    resolve(torrents);
+  });
+}
+
+function raceFirstSuccessful(promises) {
+  return Promise.all(promises.map((p) => {
+    // If a request fails, count that as a resolution so it will keep
+    // waiting for other possible successes. If a request succeeds,
+    // treat it as a rejection so Promise.all immediately bails out.
+    return p.then(
+        (val) => Promise.reject(val),
+        (err) => Promise.resolve(err)
+    );
+  })).then(
+      // If '.all' resolved, we've just got an array of errors.
+      (errors) => Promise.reject(errors),
+      // If '.all' rejected, we've got the result we wanted.
+      (val) => Promise.resolve(val)
+  );
+}
+
+module.exports = { search, dumps, Categories };

scrapers/piratebay_dump.js

@@ -0,0 +1,167 @@
+const moment = require('moment');
+const needle = require('needle');
+const Bottleneck = require('bottleneck');
+const { ungzip } = require('node-gzip');
+const LineByLineReader = require('line-by-line');
+const fs = require('fs');
+const { parse } = require('parse-torrent-title');
+const pirata = require('./api/thepiratebay');
+const { torrentFiles } = require('../lib/torrent');
+const repository = require('../lib/repository');
+const { getImdbId } = require('../lib/metadata');
+
+const NAME = 'thepiratebay';
+const CSV_FILE_PATH = '/tmp/tpb_dump.csv';
+
+const limiter = new Bottleneck({maxConcurrent: 40});
+
+async function scrape() {
+  const title = 'Я'
+      + '+(2014)_1280x720-raroch.mp4'
+      .replace(/^"|"$/g, '')
+      .normalize('NFKD') // normalize non-ASCII characters
+      .replace(/[\u0300-\u036F]/g, '')
+      .replace(/&\w{2,6};/g, ' ')
+      .replace(/\s+/g, ' ')
+      .replace(/[\W\s]+/, ' ');
+  const titleInfo = parse(title);
+  const imdbId = await getImdbId({
+    name: titleInfo.title.toLowerCase(),
+    year: titleInfo.year
+  });
+
+
+  const lastScraped = await repository.getProvider({ name: NAME });
+  const lastDump = await pirata.dumps().then((dumps) => dumps.sort((a, b) => b.updatedAt - a.updatedAt)[0]);
+
+  if (!lastScraped.lastScraped || lastScraped.lastScraped < lastDump.updatedAt) {
+    console.log(`starting to scrape tpb dump: ${JSON.stringify(lastDump)}`);
+    //await downloadDump(lastDump);
+
+    const lr = new LineByLineReader(CSV_FILE_PATH);
+    lr.on('line', (line) => {
+      if (line.includes("#ADDED")) {
+        return;
+      }
+      const row = line.match(/(?<=^|;)(".*"|[^;]+)(?=;|$)/g);
+      const torrent = {
+        uploadDate: moment(row[0], 'YYYY-MMM-DD HH:mm:ss').toDate(),
+        infoHash: Buffer.from(row[1], 'base64').toString('hex'),
+        title: row[2]
+            .replace(/^"|"$/g, '')
+            .replace(/&\w{2,6};/g, ' ')
+            .replace(/\s+/g, ' '),
+        size: parseInt(row[3], 10)
+      };
+
+      if (!limiter.empty()) {
+        lr.pause()
+      }
+
+      limiter.schedule(() => processTorrentRecord(torrent)
+            .catch((error) => console.log(`failed ${torrent.title} due: ${error}`)))
+            .then(() => limiter.empty())
+            .then((empty) => empty && lr.resume());
+    });
+    lr.on('error', (err) => {
+        console.log(err);
+    });
+    lr.on('end', () => {
+        fs.unlink(CSV_FILE_PATH);
+      updateProvider({ name: NAME, lastScraped: lastDump.updatedAt.toDate() });
+        console.log(`finished to scrape tpb dump: ${JSON.stringify(lastDump)}!`);
+    });
+  }
+}
+const allowedCategories = [
+  pirata.Categories.VIDEO.MOVIES,
+  pirata.Categories.VIDEO.MOVIES_HD,
+  pirata.Categories.VIDEO.MOVIES_DVDR,
+  pirata.Categories.VIDEO.MOVIES_3D,
+  pirata.Categories.VIDEO.TV_SHOWS,
+  pirata.Categories.VIDEO.TV_SHOWS_HD
+];
+const seriesCategories = [
+  pirata.Categories.VIDEO.TV_SHOWS,
+  pirata.Categories.VIDEO.TV_SHOWS_HD
+];
+async function processTorrentRecord(record) {
+  const persisted = await repository.getSkipTorrent(record)
+      .catch(() => repository.getTorrent(record)).catch(() => undefined);
+  if (persisted) {
+     return;
+  }
+
+  let page = 0;
+  let torrentFound;
+  while (!torrentFound && page < 5) {
+    const torrents = await pirata.search(record.title.replace(/[\W\s]+/, ' '), { page: page });
+    torrentFound = torrents.
+    filter(torrent => torrent.magnetLink.toLowerCase().includes(record.infoHash))[0];
+    page = torrents.length === 0 ? 1000 : page + 1;
+  }
+
+  if (!torrentFound) {
+    console.log(`not found: ${JSON.stringify(record)}`);
+    repository.createSkipTorrent(record);
+    return;
+  }
+  if (!allowedCategories.includes(torrentFound.subcategory)) {
+    console.log(`wrong category: ${torrentFound.name}`);
+    repository.createSkipTorrent(record);
+    return;
+  }
+
+  const type = seriesCategories.includes(torrentFound.subcategory) ? 'series' : 'movie';
+  console.log(`imdbId search: ${torrentFound.name}`);
+  const titleInfo = parse(torrentFound.name);
+  const imdbId = await getImdbId({
+    name: titleInfo.title.toLowerCase(),
+    year: titleInfo.year,
+    type: type
+  }).catch(() => undefined);
+
+  if (!imdbId) {
+    console.log(`imdbId not found: ${torrentFound.name}`);
+    repository.createFailedImdbTorrent(record);
+    return;
+  }
+
+  if (type === 'movie' || titleInfo.episode) {
+    repository.updateTorrent({
+      infoHash: record.infoHash,
+      provider: NAME,
+      title: torrentFound.name,
+      imdbId: imdbId,
+      uploadDate: record.uploadDate,
+      seeders: torrentFound.seeders,
+    });
+    return;
+  }
+
+  const files = await torrentFiles(record).catch(() => []);
+  if (!files || !files.length) {
+    console.log(`no video files found: ${torrentFound.name}`);
+    return;
+  }
+
+  repository.updateTorrent({
+    infoHash: record.infoHash,
+    provider: NAME,
+    title: torrentFound.name,
+    imdbId: imdbId,
+    uploadDate: record.uploadDate,
+    seeders: torrentFound.seeders,
+    files: files
+  })
+}
+
+function downloadDump(dump) {
+  console.log('downloading dump file...');
+  return needle('get', dump.url, { open_timeout: 2000, output: '/home/paulius/Downloads/tpb_dump.gz' })
+      .then((response) => response.body)
+      .then((body) => { console.log('unzipping dump file...'); return ungzip(body); })
+      .then((unzipped) => { console.log('writing dump file...'); return fs.promises.writeFile(CSV_FILE_PATH, unzipped); })
+}
+
+module.exports = { scrape };