From 2acc9a7f2d72b6895bbc043be180cd06872f34b2 Mon Sep 17 00:00:00 2001 From: TheBeastLT Date: Wed, 11 Mar 2020 15:05:09 +0100 Subject: [PATCH] updates addon to a working state --- addon/addon.js | 95 ++++++++++++++++++++++++++++++++----- addon/index.js | 6 +++ addon/lib/repository.js | 2 +- addon/lib/streamInfo.js | 61 ++++++++++++++++++++++++ addon/lib/types.js | 5 ++ addon/package-lock.json | 2 +- package-lock.json | 2 +- scraper/index.js | 14 +++--- scraper/lib/torrentFiles.js | 4 +- 9 files changed, 168 insertions(+), 23 deletions(-) create mode 100644 addon/index.js create mode 100644 addon/lib/streamInfo.js create mode 100644 addon/lib/types.js diff --git a/addon/addon.js b/addon/addon.js index 56a23df..51e2903 100644 --- a/addon/addon.js +++ b/addon/addon.js @@ -1,55 +1,126 @@ const { addonBuilder } = require('stremio-addon-sdk'); +const titleParser = require('parse-torrent-title'); +const { toStreamInfo } = require('./lib/streamInfo'); const { cacheWrapStream } = require('./lib/cache'); +const repository = require('./lib/repository'); const CACHE_MAX_AGE = process.env.CACHE_MAX_AGE || 24 * 60; // 24 hours in seconds const CACHE_MAX_AGE_EMPTY = 4 * 60; // 4 hours in seconds const STALE_REVALIDATE_AGE = 4 * 60; // 4 hours const STALE_ERROR_AGE = 7 * 24 * 60; // 7 days -const EMPTY_OBJECT = {}; const builder = new addonBuilder({ id: 'com.stremio.torrentio.addon', version: '1.0.0', name: 'Torrentio', - description: 'Provides torrent stream from scraped torrent providers. Currently support ThePirateBay, 1337x, RARBG, KickassTorrents, HorribleSubs.', + description: 'Provides torrent stream from scraped torrent providers. ' + + 'Currently supports ThePirateBay, 1337x, RARBG, KickassTorrents, HorribleSubs.', catalogs: [], resources: ['stream'], types: ['movie', 'series'], - idPrefixes: ['tt'], + idPrefixes: ['tt', 'kitsu'], background: `https://i.imgur.com/t8wVwcg.jpg`, - logo: `https://i.imgur.com/dPa2clS.png`, + logo: `https://i.imgur.com/GwxAcDV.png`, }); builder.defineStreamHandler((args) => { - if (!args.id.match(/tt\d+/i)) { + if (!args.id.match(/tt\d+/i) && args.id.match(/kitsu:\d+/i)) { return Promise.resolve({ streams: [] }); } const handlers = { - series: () => seriesStreamHandler(args), - movie: () => movieStreamHandler(args), + series: () => seriesRecordsHandler(args), + movie: () => movieRecordsHandler(args), fallback: () => Promise.reject('not supported type') }; return cacheWrapStream(args.id, handlers[args.type] || handlers.fallback) - .then((streams) => ({ + .then(records => filterRecordsBySeeders(records)) + .then(records => sortRecordsByVideoQuality(records)) + .then(records => records.map(record => toStreamInfo(record))) + .then(streams => ({ streams: streams, cacheMaxAge: streams.length ? CACHE_MAX_AGE : CACHE_MAX_AGE_EMPTY, staleRevalidate: STALE_REVALIDATE_AGE, staleError: STALE_ERROR_AGE })) - .catch((error) => { + .catch(error => { console.log(`Failed request ${args.id}: ${error}`); throw error; }); }); -async function seriesStreamHandler(args) { - +async function seriesRecordsHandler(args) { + if (args.id.match(/tt\d+/)) { + const parts = args.id.split(':'); + const imdbId = parts[0]; + const season = parseInt(parts[1], 10); + const episode = parseInt(parts[2], 10); + return repository.getImdbIdSeriesEntries(imdbId, season, episode); + } else if (args.id.match(/kitsu:\d+/i)) { + const parts = args.id.split(':'); + const kitsuId = parts[1]; + const episode = parseInt(parts[2], 10); + return repository.getKitsuIdSeriesEntries(kitsuId, episode); + } + return Promise.reject(`Unsupported id type: ${args.id}`); } -async function movieStreamHandler(args) { +async function movieRecordsHandler(args) { + if (args.id.match(/tt\d+/)) { + return repository.getImdbIdMovieEntries(args.id); + } else if (args.id.match(/kitsu:\d+/i)) { + return repository.getKitsuIdMovieEntries(args.id.replace('kitsu:', ''), episode); + } + return Promise.reject(`Unsupported id type: ${args.id}`); +} +const HEALTHY_SEEDERS = 5; +const SEEDED_SEEDERS = 1; +const MIN_HEALTHY_COUNT = 10; +const MAX_UNHEALTHY_COUNT = 5; + +function filterRecordsBySeeders(records) { + const sortedRecords = records + .sort((a, b) => b.torrent.seeders - a.torrent.seeders || b.torrent.uploadDate - a.torrent.uploadDate); + const healthy = sortedRecords.filter(record => record.torrent.seeders >= HEALTHY_SEEDERS); + const seeded = sortedRecords.filter(record => record.torrent.seeders >= SEEDED_SEEDERS); + + if (healthy.length >= MIN_HEALTHY_COUNT) { + return healthy; + } else if (seeded.length >= MAX_UNHEALTHY_COUNT) { + return seeded.slice(0, MIN_HEALTHY_COUNT); + } + return sortedRecords.slice(0, MAX_UNHEALTHY_COUNT); +} + +function sortRecordsByVideoQuality(records) { + const qualityMap = records + .reduce((map, record) => { + const parsedFile = titleParser.parse(record.title); + const parsedTorrent = titleParser.parse(record.torrent.title); + const quality = parsedFile.resolution || parsedTorrent.resolution || parsedFile.source || parsedTorrent.source; + map[quality] = (map[quality] || []).concat(record); + return map; + }, {}); + const sortedQualities = Object.keys(qualityMap) + .sort((a, b) => { + const aQuality = a === '4k' ? '2160p' : a === 'undefined' ? undefined : a; + const bQuality = b === '4k' ? '2160p' : b === 'undefined' ? undefined : b; + const aResolution = aQuality && aQuality.match(/\d+p/) && parseInt(aQuality, 10); + const bResolution = bQuality && bQuality.match(/\d+p/) && parseInt(bQuality, 10); + if (aResolution && bResolution) { + return bResolution - aResolution; // higher resolution first; + } else if (aResolution) { + return -1; + } else if (bResolution) { + return 1; + } + return a < b ? -1 : b < a ? 1 : 0; + }); + return sortedQualities + .map(quality => qualityMap[quality]) + .reduce((a, b) => a.concat(b), []); } module.exports = builder.getInterface(); diff --git a/addon/index.js b/addon/index.js new file mode 100644 index 0000000..9e43810 --- /dev/null +++ b/addon/index.js @@ -0,0 +1,6 @@ +const { serveHTTP } = require('stremio-addon-sdk'); +const addonInterface = require('./addon'); + +const PORT = process.env.PORT || 7000; + +serveHTTP(addonInterface, { port: PORT, cacheMaxAge: 86400 }); diff --git a/addon/lib/repository.js b/addon/lib/repository.js index 02ffc69..730f02b 100644 --- a/addon/lib/repository.js +++ b/addon/lib/repository.js @@ -40,7 +40,7 @@ const File = database.define('file', ); Torrent.hasMany(File, { foreignKey: 'infoHash', constraints: false }); -File.belongsTo(Torrent, { constraints: false }); +File.belongsTo(Torrent, { foreignKey: 'infoHash', constraints: false }); function getImdbIdMovieEntries(imdbId) { return File.findAll({ diff --git a/addon/lib/streamInfo.js b/addon/lib/streamInfo.js new file mode 100644 index 0000000..c7bc786 --- /dev/null +++ b/addon/lib/streamInfo.js @@ -0,0 +1,61 @@ +const titleParser = require('parse-torrent-title'); +const { Type } = require('./types'); + +const ADDON_NAME = 'Torrentio'; + +function toStreamInfo(record) { + if (record.torrent.type === Type.MOVIE) { + return movieStream(record); + } + return seriesStream(record); +} + +function movieStream(record) { + const titleInfo = titleParser.parse(record.title); + const title = joinDetailParts( + [ + joinDetailParts([titleInfo.title, titleInfo.year, titleInfo.language]), + joinDetailParts([titleInfo.resolution, titleInfo.source], '📺 '), + joinDetailParts([record.torrent.seeders], '👤 ') + ], + '', + '\n' + ); + + return { + name: `${ADDON_NAME}\n${record.torrent.provider}`, + title: title, + infoHash: record.infoHash, + }; +} + +function seriesStream(record) { + const tInfo = titleParser.parse(record.title); + const eInfo = titleParser.parse(record.torrent.title); + const sameInfo = tInfo.season === eInfo.season && tInfo.episode && eInfo.episode === tInfo.episode; + const title = joinDetailParts( + [ + joinDetailParts([record.torrent.title.replace(/[, ]+/g, ' ')]), + joinDetailParts([!sameInfo && record.title.replace(/[, ]+/g, ' ') || undefined]), + joinDetailParts([tInfo.resolution || eInfo.resolution, tInfo.source || eInfo.source], '📺 '), + joinDetailParts([record.torrent.seeders], '👤 ') + ], + '', + '\n' + ); + + return { + name: `${ADDON_NAME}\n${record.torrent.provider}`, + title: title, + infoHash: record.infoHash, + fileIdx: record.fileIndex, + }; +} + +function joinDetailParts(parts, prefix = '', delimiter = ' ') { + const filtered = parts.filter((part) => part !== undefined && part !== null).join(delimiter); + + return filtered.length > 0 ? `${prefix}${filtered}` : undefined; +} + +module.exports = { toStreamInfo }; diff --git a/addon/lib/types.js b/addon/lib/types.js new file mode 100644 index 0000000..2f37a8e --- /dev/null +++ b/addon/lib/types.js @@ -0,0 +1,5 @@ +exports.Type = { + MOVIE: 'movie', + SERIES: 'series', + ANIME: 'anime' +}; \ No newline at end of file diff --git a/addon/package-lock.json b/addon/package-lock.json index 77af9b6..a0db8a5 100644 --- a/addon/package-lock.json +++ b/addon/package-lock.json @@ -1306,7 +1306,7 @@ } }, "parse-torrent-title": { - "version": "git://github.com/TheBeastLT/parse-torrent-title.git#ddd5037820289d35e600baec9d8e730935d261af", + "version": "git://github.com/TheBeastLT/parse-torrent-title.git#90c60ab3edab3a40843160ba1bdf4995c4b5cc56", "from": "git://github.com/TheBeastLT/parse-torrent-title.git#master", "requires": { "moment": "^2.24.0" diff --git a/package-lock.json b/package-lock.json index 9ad3b0b..ebf8e29 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1392,7 +1392,7 @@ } }, "parse-torrent-title": { - "version": "git://github.com/TheBeastLT/parse-torrent-title.git#ddd5037820289d35e600baec9d8e730935d261af", + "version": "git://github.com/TheBeastLT/parse-torrent-title.git#90c60ab3edab3a40843160ba1bdf4995c4b5cc56", "from": "git://github.com/TheBeastLT/parse-torrent-title.git#master", "requires": { "moment": "^2.24.0" diff --git a/scraper/index.js b/scraper/index.js index 1cb6790..6b46d92 100644 --- a/scraper/index.js +++ b/scraper/index.js @@ -8,15 +8,17 @@ const horribleSubsScraper = require('./scrapers/horriblesubs/horriblesubs_scrape const leetxScraper = require('./scrapers/1337x/1337x_scraper'); const kickassScraper = require('./scrapers/kickass/kickass_scraper'); const rarbgScraper = require('./scrapers/rarbg/rarbg_scraper'); +const rarbgDumpScraper = require('./scrapers/rarbg/rarbg_dump_scraper'); const thepiratebayDumpScraper = require('./scrapers/thepiratebay/thepiratebay_dump_scraper'); const thepiratebayUnofficialDumpScraper = require('./scrapers/thepiratebay/thepiratebay_unofficial_dump_scraper'); const PROVIDERS = [ - horribleSubsScraper, - rarbgScraper, - thepiratebayScraper, - kickassScraper, - leetxScraper + // horribleSubsScraper, + // rarbgScraper, + // thepiratebayScraper, + // kickassScraper, + // leetxScraper + rarbgDumpScraper ]; const SCRAPE_CRON = process.env.SCRAPE_CRON || '* * 0/4 * * *'; @@ -34,7 +36,7 @@ server.get('/', function (req, res) { server.listen(process.env.PORT || 7000, async function () { await connect(); - schedule.scheduleJob(SCRAPE_CRON, () => scrape()); + // schedule.scheduleJob(SCRAPE_CRON, () => scrape()); console.log('Scraper started'); scrape(); }); \ No newline at end of file diff --git a/scraper/lib/torrentFiles.js b/scraper/lib/torrentFiles.js index 201fddc..acf5b66 100644 --- a/scraper/lib/torrentFiles.js +++ b/scraper/lib/torrentFiles.js @@ -156,8 +156,8 @@ async function decomposeEpisodes(torrent, files, metadata = { episodeCount: [] } } else if (files.every(file => (!file.season || !metadata.episodeCount[file.season - 1]) && file.date)) { decomposeDateEpisodeFiles(torrent, files, metadata); } else if (files.filter(file => !file.isMovie && file.episodes).every(file => !file.season && file.episodes) || - files.some(file => file.season && file.episodes && file.episodes - .every(ep => metadata.episodeCount[file.season - 1] < ep))) { + files.filter(file => file.season && file.episodes && file.episodes + .every(ep => metadata.episodeCount[file.season - 1] < ep)).length > Math.ceil(files.length / 5)) { decomposeAbsoluteEpisodeFiles(torrent, files, metadata); }