From b43da7a882314014b56daf3bb0deea27056dc22d Mon Sep 17 00:00:00 2001 From: TheBeastLT Date: Tue, 17 Oct 2023 20:37:05 +0300 Subject: [PATCH] add (disabled) possibility to return stream subtitles for non debrid streams --- addon/lib/extension.js | 2 +- addon/lib/languages.js | 11 ++++- addon/lib/repository.js | 23 +++++++++ addon/lib/streamInfo.js | 27 +++++++++-- addon/lib/subtitles.js | 101 ++++++++++++++++++++++++++++++++++++++++ addon/package-lock.json | 72 ++++++++++++++-------------- addon/package.json | 6 +-- 7 files changed, 198 insertions(+), 44 deletions(-) create mode 100644 addon/lib/subtitles.js diff --git a/addon/lib/extension.js b/addon/lib/extension.js index e701b73..57eb70c 100644 --- a/addon/lib/extension.js +++ b/addon/lib/extension.js @@ -71,4 +71,4 @@ function isExtension(filename, extensions) { return extensionMatch && extensions.includes(extensionMatch[1].toLowerCase()); } -module.exports = { isVideo, isSubtitle, isDisk, isArchive } \ No newline at end of file +module.exports = { isVideo, isSubtitle, isDisk, isArchive, isExtension } \ No newline at end of file diff --git a/addon/lib/languages.js b/addon/lib/languages.js index 5a97cff..097f962 100644 --- a/addon/lib/languages.js +++ b/addon/lib/languages.js @@ -21,11 +21,15 @@ const languageMapping = { 'tamil': '🇮🇳', 'polish': '🇵🇱', 'lithuanian': '🇱🇹', + 'latvian': '🇱🇻', + 'estonian': '🇪🇪', 'czech': '🇨🇿', 'slovakian': '🇸🇰', 'slovenian': '🇸🇮', 'hungarian': '🇭🇺', 'romanian': '🇷🇴', + 'bulgarian': '🇧🇬', + 'serbian': '🇷🇸 ', 'croatian': '🇭🇷', 'ukrainian': '🇺🇦', 'greek': '🇬🇷', @@ -65,4 +69,9 @@ function containsLanguage(stream, languages) { return languages.map(lang => languageMapping[lang]).some(lang => stream.title.includes(lang)); } -module.exports = { mapLanguages, containsLanguage, LanguageOptions } \ No newline at end of file +function languageFromCode(code) { + const entry = Object.entries(languageMapping).find(entry => entry[1] === code); + return entry && entry[0]; +} + +module.exports = { mapLanguages, containsLanguage, languageFromCode, LanguageOptions } \ No newline at end of file diff --git a/addon/lib/repository.js b/addon/lib/repository.js index 245b4a8..9325bd2 100644 --- a/addon/lib/repository.js +++ b/addon/lib/repository.js @@ -41,8 +41,31 @@ const File = database.define('file', }, ); +const Subtitle = database.define('subtitle', + { + infoHash: { + type: Sequelize.STRING(64), + allowNull: false, + references: { model: Torrent, key: 'infoHash' }, + onDelete: 'CASCADE' + }, + fileIndex: { type: Sequelize.INTEGER, allowNull: false }, + fileId: { + type: Sequelize.BIGINT, + allowNull: true, + references: { model: File, key: 'id' }, + onDelete: 'SET NULL' + }, + title: { type: Sequelize.STRING(512), allowNull: false }, + size: { type: Sequelize.BIGINT, allowNull: false }, + }, + { timestamps: false } +); + Torrent.hasMany(File, { foreignKey: 'infoHash', constraints: false }); File.belongsTo(Torrent, { foreignKey: 'infoHash', constraints: false }); +File.hasMany(Subtitle, { foreignKey: 'fileId', constraints: false }); +Subtitle.belongsTo(File, { foreignKey: 'fileId', constraints: false }); function getTorrent(infoHash) { return Torrent.findOne({ where: { infoHash: infoHash } }); diff --git a/addon/lib/streamInfo.js b/addon/lib/streamInfo.js index f067f08..b498e4f 100644 --- a/addon/lib/streamInfo.js +++ b/addon/lib/streamInfo.js @@ -2,6 +2,7 @@ const titleParser = require('parse-torrent-title'); const { Type } = require('./types'); const { mapLanguages } = require('./languages'); const { enrichStreamSources, getSources } = require('./magnetHelper'); +const { getSubtitles } = require("./subtitles"); const ADDON_NAME = 'Torrentio'; const SIZE_DELTA = 0.02; @@ -39,9 +40,8 @@ function toStreamInfo(record) { '\n' ); const bingeGroupParts = getBingeGroupParts(record, sameInfo, quality, torrentInfo, fileInfo); - const behaviorHints = { - bingeGroup: joinDetailParts(bingeGroupParts, "torrentio|", "|") - }; + const bingeGroup = joinDetailParts(bingeGroupParts, "torrentio|", "|") + const behaviorHints = bingeGroup ? { bingeGroup } : undefined; return cleanOutputObject({ name: name, @@ -49,7 +49,8 @@ function toStreamInfo(record) { infoHash: record.infoHash, fileIdx: record.fileIndex, behaviorHints: behaviorHints, - sources: getSources(record.torrent.trackers, record.infoHash) + sources: getSources(record.torrent.trackers, record.infoHash), + subtitles: getSubtitles(record) }); } @@ -110,7 +111,23 @@ function applyStaticInfo(streams) { } function enrichStaticInfo(stream) { - return enrichStreamSources(stream); + return enrichSubtitles(enrichStreamSources({ ...stream })); +} + +function enrichSubtitles(stream) { + if (stream.subtitles?.length) { + stream.subtitles = stream.subtitles.map(subtitle =>{ + if (subtitle.url) { + return subtitle; + } + return { + id: `${subtitle.fileIndex}`, + lang: subtitle.lang, + url: `http://localhost:11470/${subtitle.infoHash}/${subtitle.fileIndex}/${subtitle.title.split('/').pop()}` + }; + }); + } + return stream; } function getBingeGroupParts(record, sameInfo, quality, torrentInfo, fileInfo) { diff --git a/addon/lib/subtitles.js b/addon/lib/subtitles.js new file mode 100644 index 0000000..dac87c5 --- /dev/null +++ b/addon/lib/subtitles.js @@ -0,0 +1,101 @@ +const { parse } = require('parse-torrent-title'); +const { isExtension } = require("./extension"); +const { Providers } = require("./filter"); +const { languageFromCode } = require("./languages"); + +const languageMapping = { + 'english': 'eng', + 'japanese': 'jpn', + 'russian': 'rus', + 'italian': 'ita', + 'portuguese': 'por', + 'spanish': 'spa', + 'latino': 'lat', + 'korean': 'kor', + 'chinese': 'zho', + 'taiwanese': 'zht', + 'french': 'fre', + 'german': 'ger', + 'dutch': 'dut', + 'hindi': 'hin ', + 'telugu': 'tel', + 'tamil': 'tam', + 'polish': 'pol', + 'lithuanian': 'lit', + 'latvian': 'lav', + 'estonian': 'est', + 'czech': 'cze', + 'slovakian': 'slo', + 'slovenian': 'slv', + 'hungarian': 'hun', + 'romanian': 'rum', + 'bulgarian': 'bul', + 'serbian': 'scc', + 'croatian': 'hrv', + 'ukrainian': 'ukr', + 'greek': 'ell', + 'danish': 'dan', + 'finnish': 'fin', + 'swedish': 'swe', + 'norwegian': 'nor', + 'turkish': 'tur', + 'arabic': 'ara', + 'persian': 'per', + 'hebrew': 'heb', + 'vietnamese': 'vie', + 'indonesian': 'ind', + 'thai': 'tha' +} + +const ignoreSet = new Set(['dubbed', 'multi audio', 'multi subs', 'dual audio']); +const allowedExtensions = ['srt', 'vtt', 'ass', 'ssa']; + +function getSubtitles(record) { + if (!record.subtitles || !record.subtitles.length) { + return null; + } + return record.subtitles + .filter(subtitle => isExtension(subtitle.title, allowedExtensions)) + .sort((a, b) => b.size - a.size) + .map(subtitle => ({ + infoHash: subtitle.infoHash, + fileIndex: subtitle.fileIndex, + title: subtitle.title, + lang: parseLanguage(subtitle.title, record), + })); +} + +function parseLanguage(title, record) { + const subtitlePathParts = title.split('/'); + const subtitleFileName = subtitlePathParts.pop(); + const subtitleTitleNoExt = title.replace(/\.\w{2,5}$/, ''); + const videoFileName = record.title.split('/').pop().replace(/\.\w{2,5}$/, ''); + const fileNameLanguage = getSingleLanguage(subtitleFileName.replace(videoFileName, '')); + if (fileNameLanguage) { + return fileNameLanguage; + } + const videoTitleNoExt = record.title.replace(/\.\w{2,5}$/, ''); + if (subtitleTitleNoExt === record.title || subtitleTitleNoExt === videoTitleNoExt) { + const provider = Providers.options.find(provider => provider.label === record.torrent.provider); + return provider?.foreign && languageFromCode(provider.foreign) || 'eng'; + } + const folderName = subtitlePathParts.join('/'); + const folderNameLanguage = getSingleLanguage(folderName.replace(videoFileName, '')); + if (folderNameLanguage) { + return folderNameLanguage + } + return getFileNameLanguageCode(subtitleFileName) || 'Unknown'; +} + +function getSingleLanguage(title) { + const parsedInfo = parse(title); + const languages = (parsedInfo.languages || []).filter(language => !ignoreSet.has(language)); + return languages.length === 1 ? languageMapping[languages[0]] : undefined; +} + +function getFileNameLanguageCode(fileName) { + const match = fileName.match(/(?:(?:^|[._ ])([A-Za-z][a-z]{1,2})|\[([a-z]{2,3})])\.\w{3,4}$/); + return match && match[1].toLowerCase(); +} + +module.exports = { getSubtitles } \ No newline at end of file diff --git a/addon/package-lock.json b/addon/package-lock.json index 56b072a..0e5cc72 100644 --- a/addon/package-lock.json +++ b/addon/package-lock.json @@ -21,15 +21,15 @@ "name-to-imdb": "^3.0.4", "named-queue": "^2.2.1", "offcloud-api": "^1.0.0", - "parse-torrent-title": "git://github.com/TheBeastLT/parse-torrent-title.git#0019426237d273b5214df1a6df81232d7f2ae3fb", + "parse-torrent-title": "git://github.com/TheBeastLT/parse-torrent-title.git#aeae91e512741910bc479ac57dcb0ce78bad3ee0", "pg": "^8.10.0", "premiumize-api": "^1.0.3", "real-debrid-api": "git://github.com/TheBeastLT/node-real-debrid.git#d1f7eaa8593b947edbfbc8a92a176448b48ef445", "request-ip": "^3.3.0", "sequelize": "^6.31.1", "stremio-addon-sdk": "^1.6.10", - "ua-parser-js": "^1.0.32", - "user-agents": "^1.0.1234" + "ua-parser-js": "^1.0.36", + "user-agents": "^1.0.1444" } }, "node_modules/@putdotio/api-client": { @@ -574,9 +574,9 @@ } }, "node_modules/dot-json": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/dot-json/-/dot-json-1.2.2.tgz", - "integrity": "sha512-AKL+GsO4wSEU4LU+fAk/PqN4nQ6PB1vT3HpMiZous9xCzK5S0kh4DzfUY0EfU67jsIXLlu0ty71659N9Nmg+Tw==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/dot-json/-/dot-json-1.3.0.tgz", + "integrity": "sha512-Pu11Prog/Yjf2lBICow82/DSV46n3a2XT1Rqt/CeuhkO1fuacF7xydYhI0SwQx2Ue0jCyLtQzgKPFEO6ewv+bQ==", "dependencies": { "detect-indent": "~6.0.0", "docopt": "~0.6.2", @@ -1439,8 +1439,8 @@ }, "node_modules/parse-torrent-title": { "version": "1.3.0", - "resolved": "git+ssh://git@github.com/TheBeastLT/parse-torrent-title.git#0019426237d273b5214df1a6df81232d7f2ae3fb", - "integrity": "sha512-Ay79gA/yYof3TU+8ddeL3kBnt0Mw9oVe2RXyrkN51Xj1PeiqJws7JaGnSuB3bsSsJ3weFBnH9qeXEhhC+9WVPw==", + "resolved": "git+ssh://git@github.com/TheBeastLT/parse-torrent-title.git#aeae91e512741910bc479ac57dcb0ce78bad3ee0", + "integrity": "sha512-Mys327xq6sHDWM4QVOMIZ7PVOY1WxoDI8ps4vjchwoyD26ezY2OQ7a1FsnZiaLBJTH9uCbLp/I4sKwsBcDinXQ==", "license": "MIT", "dependencies": { "moment": "^2.24.0" @@ -2270,9 +2270,9 @@ } }, "node_modules/ua-parser-js": { - "version": "1.0.33", - "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-1.0.33.tgz", - "integrity": "sha512-RqshF7TPTE0XLYAqmjlu5cLLuGdKrNu9O1KLA/qp39QtbZwuzwv1dT46DZSopoUMsYgXpB3Cv8a03FI8b74oFQ==", + "version": "1.0.36", + "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-1.0.36.tgz", + "integrity": "sha512-znuyCIXzl8ciS3+y3fHJI/2OhQIXbXw9MWC/o3qwyR+RGppjZHrM27CGFSKCJXi2Kctiz537iOu2KnXs1lMQhw==", "funding": [ { "type": "opencollective", @@ -2281,6 +2281,10 @@ { "type": "paypal", "url": "https://paypal.me/faisalman" + }, + { + "type": "github", + "url": "https://github.com/sponsors/faisalman" } ], "engines": { @@ -2288,9 +2292,9 @@ } }, "node_modules/underscore": { - "version": "1.13.1", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.1.tgz", - "integrity": "sha512-hzSoAVtJF+3ZtiFX0VgfFPHEDRm7Y/QPjGyNo4TVdnDTdft3tr8hEkD25a1jC+TjTuE7tkHGKkhwCgs9dgBB2g==" + "version": "1.13.6", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.6.tgz", + "integrity": "sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==" }, "node_modules/underscore-keypath": { "version": "0.0.22", @@ -2322,11 +2326,11 @@ "integrity": "sha512-HXgFDgDommxn5/bIv0cnQZsPhHDA90NPHD6+c/v21U5+Sx5hoP8+dP9IZXBU1gIfvdRfhG8cel9QNPeionfcCQ==" }, "node_modules/user-agents": { - "version": "1.0.1295", - "resolved": "https://registry.npmjs.org/user-agents/-/user-agents-1.0.1295.tgz", - "integrity": "sha512-9Rgw4X4FeHZ1d11X9zASRgLYlPKeBINZNRsitoL/RwNEXuNTnDiImMfqa5/+OoGBNRJ1kqxmH3bJL8Fu4m0VIg==", + "version": "1.0.1444", + "resolved": "https://registry.npmjs.org/user-agents/-/user-agents-1.0.1444.tgz", + "integrity": "sha512-6WXJ0RZuUKgif1rW5FN02HnpoJ8EzH6COQoXCiVStZEVPz+YnAx3iA48etY3ZD4UwueYN9ALC7j4ayHvYEh7tA==", "dependencies": { - "dot-json": "^1.2.2", + "dot-json": "^1.3.0", "lodash.clonedeep": "^4.5.0" } }, @@ -2873,9 +2877,9 @@ "integrity": "sha512-NqTbaYeE4gA/wU1hdKFdU+AFahpDOpgGLzHP42k6H6DKExJd0A55KEVWYhL9FEmHmgeLvEU2vuKXDuU+4yToOw==" }, "dot-json": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/dot-json/-/dot-json-1.2.2.tgz", - "integrity": "sha512-AKL+GsO4wSEU4LU+fAk/PqN4nQ6PB1vT3HpMiZous9xCzK5S0kh4DzfUY0EfU67jsIXLlu0ty71659N9Nmg+Tw==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/dot-json/-/dot-json-1.3.0.tgz", + "integrity": "sha512-Pu11Prog/Yjf2lBICow82/DSV46n3a2XT1Rqt/CeuhkO1fuacF7xydYhI0SwQx2Ue0jCyLtQzgKPFEO6ewv+bQ==", "requires": { "detect-indent": "~6.0.0", "docopt": "~0.6.2", @@ -3515,9 +3519,9 @@ "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==" }, "parse-torrent-title": { - "version": "git+ssh://git@github.com/TheBeastLT/parse-torrent-title.git#0019426237d273b5214df1a6df81232d7f2ae3fb", - "integrity": "sha512-Ay79gA/yYof3TU+8ddeL3kBnt0Mw9oVe2RXyrkN51Xj1PeiqJws7JaGnSuB3bsSsJ3weFBnH9qeXEhhC+9WVPw==", - "from": "parse-torrent-title@git://github.com/TheBeastLT/parse-torrent-title.git#0019426237d273b5214df1a6df81232d7f2ae3fb", + "version": "git+ssh://git@github.com/TheBeastLT/parse-torrent-title.git#aeae91e512741910bc479ac57dcb0ce78bad3ee0", + "integrity": "sha512-Mys327xq6sHDWM4QVOMIZ7PVOY1WxoDI8ps4vjchwoyD26ezY2OQ7a1FsnZiaLBJTH9uCbLp/I4sKwsBcDinXQ==", + "from": "parse-torrent-title@git://github.com/TheBeastLT/parse-torrent-title.git#aeae91e512741910bc479ac57dcb0ce78bad3ee0", "requires": { "moment": "^2.24.0" } @@ -4149,14 +4153,14 @@ } }, "ua-parser-js": { - "version": "1.0.33", - "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-1.0.33.tgz", - "integrity": "sha512-RqshF7TPTE0XLYAqmjlu5cLLuGdKrNu9O1KLA/qp39QtbZwuzwv1dT46DZSopoUMsYgXpB3Cv8a03FI8b74oFQ==" + "version": "1.0.36", + "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-1.0.36.tgz", + "integrity": "sha512-znuyCIXzl8ciS3+y3fHJI/2OhQIXbXw9MWC/o3qwyR+RGppjZHrM27CGFSKCJXi2Kctiz537iOu2KnXs1lMQhw==" }, "underscore": { - "version": "1.13.1", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.1.tgz", - "integrity": "sha512-hzSoAVtJF+3ZtiFX0VgfFPHEDRm7Y/QPjGyNo4TVdnDTdft3tr8hEkD25a1jC+TjTuE7tkHGKkhwCgs9dgBB2g==" + "version": "1.13.6", + "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.6.tgz", + "integrity": "sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==" }, "underscore-keypath": { "version": "0.0.22", @@ -4185,11 +4189,11 @@ "integrity": "sha512-HXgFDgDommxn5/bIv0cnQZsPhHDA90NPHD6+c/v21U5+Sx5hoP8+dP9IZXBU1gIfvdRfhG8cel9QNPeionfcCQ==" }, "user-agents": { - "version": "1.0.1295", - "resolved": "https://registry.npmjs.org/user-agents/-/user-agents-1.0.1295.tgz", - "integrity": "sha512-9Rgw4X4FeHZ1d11X9zASRgLYlPKeBINZNRsitoL/RwNEXuNTnDiImMfqa5/+OoGBNRJ1kqxmH3bJL8Fu4m0VIg==", + "version": "1.0.1444", + "resolved": "https://registry.npmjs.org/user-agents/-/user-agents-1.0.1444.tgz", + "integrity": "sha512-6WXJ0RZuUKgif1rW5FN02HnpoJ8EzH6COQoXCiVStZEVPz+YnAx3iA48etY3ZD4UwueYN9ALC7j4ayHvYEh7tA==", "requires": { - "dot-json": "^1.2.2", + "dot-json": "^1.3.0", "lodash.clonedeep": "^4.5.0" } }, diff --git a/addon/package.json b/addon/package.json index eba660b..790e94f 100644 --- a/addon/package.json +++ b/addon/package.json @@ -20,14 +20,14 @@ "name-to-imdb": "^3.0.4", "named-queue": "^2.2.1", "offcloud-api": "^1.0.0", - "parse-torrent-title": "git://github.com/TheBeastLT/parse-torrent-title.git#0019426237d273b5214df1a6df81232d7f2ae3fb", + "parse-torrent-title": "git://github.com/TheBeastLT/parse-torrent-title.git#aeae91e512741910bc479ac57dcb0ce78bad3ee0", "pg": "^8.10.0", "premiumize-api": "^1.0.3", "real-debrid-api": "git://github.com/TheBeastLT/node-real-debrid.git#d1f7eaa8593b947edbfbc8a92a176448b48ef445", "request-ip": "^3.3.0", "sequelize": "^6.31.1", "stremio-addon-sdk": "^1.6.10", - "ua-parser-js": "^1.0.32", - "user-agents": "^1.0.1234" + "ua-parser-js": "^1.0.36", + "user-agents": "^1.0.1444" } }