[addon] adds debrid meta catalogs for RD and AD

addon/addon.js

@@ -1,12 +1,12 @@
 const Bottleneck = require('bottleneck');
 const { addonBuilder } = require('stremio-addon-sdk');
 const { Type } = require('./lib/types');
-const { manifest, DefaultProviders } = require('./lib/manifest');
+const { dummyManifest, DefaultProviders } = require('./lib/manifest');
 const { cacheWrapStream } = require('./lib/cache');
 const { toStreamInfo } = require('./lib/streamInfo');
 const repository = require('./lib/repository');
 const applySorting = require('./lib/sort');
-const { applyMochs } = require('./moch/moch');
+const { applyMochs, getMochCatalog, getMochItemMeta } = require('./moch/moch');
 
 const CACHE_MAX_AGE = process.env.CACHE_MAX_AGE || 4 * 60 * 60; // 4 hours in seconds
 const CACHE_MAX_AGE_EMPTY = 30 * 60; // 30 minutes
@@ -14,7 +14,7 @@ const STALE_REVALIDATE_AGE = 4 * 60 * 60; // 4 hours
 const STALE_ERROR_AGE = 7 * 24 * 60 * 60; // 7 days
 
 const defaultProviders = DefaultProviders.map(provider => provider.toLowerCase());
-const builder = new addonBuilder(manifest());
+const builder = new addonBuilder(dummyManifest());
 const limiter = new Bottleneck({
   maxConcurrent: process.env.LIMIT_MAX_CONCURRENT || 20,
   highWater: process.env.LIMIT_QUEUE_SIZE || 100,
@@ -45,6 +45,34 @@ builder.defineStreamHandler((args) => {
       });
 });
 
+builder.defineCatalogHandler((args) => {
+  const mochKey = args.id.replace("torrentio-", '');
+  console.log(`Incoming catalog ${args.id} request with skip=${args.extra.skip || 0}`)
+  return getMochCatalog(mochKey, args.extra)
+      .then(metas => ({
+        metas: metas,
+        cacheMaxAge: 0
+      }))
+      .catch(error => {
+        console.log(`Failed retrieving catalog ${args.id}: `, error);
+        throw Promise.reject(error);
+      });
+})
+
+builder.defineMetaHandler((args) => {
+  const [mochKey, metaId] = args.id.split(':');
+  console.log(`Incoming debrid meta ${args.id} request`)
+  return getMochItemMeta(mochKey, metaId, args.extra)
+      .then(meta => ({
+        meta: meta,
+        cacheMaxAge: CACHE_MAX_AGE
+      }))
+      .catch(error => {
+        console.log(`Failed retrieving catalog meta ${args.id}: `, error);
+        throw Promise.reject(error);
+      });
+})
+
 async function streamHandler(args) {
   if (args.type === Type.MOVIE) {
     return movieRecordsHandler(args);

addon/lib/manifest.js

@@ -1,4 +1,5 @@
 const { MochOptions } = require('../moch/moch');
+const { Type } = require('./types');
 
 const Providers = [
   'YTS',
@@ -12,6 +13,7 @@ const Providers = [
   'NyaaPantsu'
 ];
 const DefaultProviders = Providers
+const CatalogMochs = [MochOptions.realdebrid, MochOptions.alldebrid];
 
 function manifest(config = {}) {
   const providersList = config.providers && config.providers.map(provider => getProvider(provider)) || DefaultProviders;
@@ -26,15 +28,14 @@ function manifest(config = {}) {
   const mochsDesc = enabledMochs ? ` and ${enabledMochs} enabled ` : '';
   return {
     id: 'com.stremio.torrentio.addon',
-    version: '0.0.6',
+    version: '0.0.7',
     name: 'Torrentio',
     description: 'Provides torrent streams from scraped torrent providers.'
         + ` Currently supports ${enabledProvidersDesc}${mochsDesc}.`
         + ` To configure providers, ${possibleMochs} support and other settings visit https://torrentio.strem.fun`,
-    catalogs: [],
-    resources: ['stream'],
-    types: ['movie', 'series'],
-    idPrefixes: ['tt', 'kitsu'],
+    catalogs: getCatalogs(config),
+    resources: getResources(config),
+    types: [Type.MOVIE, Type.SERIES, Type.OTHER],
     background: `https://i.ibb.co/VtSfFP9/t8wVwcg.jpg`,
     logo: `https://i.ibb.co/w4BnkC9/GwxAcDV.png`,
     behaviorHints: {
@@ -44,8 +45,42 @@ function manifest(config = {}) {
   }
 }
 
+function dummyManifest() {
+  const manifestDefault = manifest();
+  manifestDefault.catalogs = [{ id: 'dummy', type: Type.OTHER }];
+  manifestDefault.resources = ['stream', 'meta'];
+  return manifestDefault;
+}
+
 function getProvider(configProvider) {
   return Providers.find(provider => provider.toLowerCase() === configProvider);
 }
 
-module.exports = { manifest, Providers, DefaultProviders };
+function getCatalogs(config) {
+  return CatalogMochs
+      .filter(moch => config[moch.key])
+      .map(moch => ({
+        id: `torrentio-${moch.key}`,
+        name: `${moch.name}`,
+        type: 'other',
+      }));
+}
+
+function getResources(config) {
+  const streamResource = {
+    name: 'stream',
+    types: [Type.MOVIE, Type.SERIES],
+    idPrefixes: ['tt', 'kitsu']
+  };
+  const metaResource = {
+    name: 'meta',
+    types: [Type.OTHER],
+    idPrefixes: CatalogMochs.filter(moch => config[moch.key]).map(moch => moch.key)
+  };
+  if (CatalogMochs.filter(moch => config[moch.key]).length) {
+    return [streamResource, metaResource];
+  }
+  return [streamResource];
+}
+
+module.exports = { manifest, dummyManifest, Providers, DefaultProviders };

addon/lib/requestHelper.js

@@ -15,7 +15,7 @@ function getRandomProxy() {
   if (PROXY_HOSTS && PROXY_HOSTS.length && PROXY_USERNAME && PROXY_PASSWORD) {
     const index = new Date().getHours() % PROXY_HOSTS.length;
     const proxyHost = PROXY_HOSTS[index];
-    console.log(`${new Date()} Using ${proxyHost} proxy`);
+    console.log(`${new Date().toISOString()} Using ${proxyHost} proxy`);
     return `https://${PROXY_USERNAME}:${PROXY_PASSWORD}@${proxyHost}:${PROXY_PORT}`;
   }
   console.warn('No proxy configured!');
@@ -28,7 +28,7 @@ function getProxyAgent(proxy) {
 
 function blacklistProxy(proxy) {
   const proxyHost = proxy.replace(/.*@/, '');
-  console.warn(`${new Date()} Blacklisting ${proxyHost}`);
+  console.warn(`${new Date().toISOString()} Blacklisting ${proxyHost}`);
   if (PROXY_HOSTS && PROXY_HOSTS.indexOf(proxyHost) > -1) {
     PROXY_HOSTS.splice(PROXY_HOSTS.indexOf(proxyHost), 1);
   }

addon/lib/types.js

@@ -1,5 +1,6 @@
 exports.Type = {
   MOVIE: 'movie',
   SERIES: 'series',
-  ANIME: 'anime'
+  ANIME: 'anime',
+  OTHER: 'other'
 };

addon/moch/alldebrid.js

@@ -1,9 +1,12 @@
 const AllDebridClient = require('all-debrid-api');
+const { Type } = require('../lib/types');
 const { isVideo, isArchive } = require('../lib/extension');
 const StaticResponse = require('./static');
 const { getRandomProxy, getProxyAgent, getRandomUserAgent } = require('../lib/requestHelper');
 const { cacheWrapProxy, cacheUserAgent } = require('../lib/cache');
 
+const KEY = 'alldebrid';
+
 async function getCachedStreams(streams, apiKey) {
   const options = await getDefaultOptions(apiKey);
   const AD = new AllDebridClient(apiKey, options);
@@ -28,10 +31,61 @@ async function getCachedStreams(streams, apiKey) {
       }, {})
 }
 
+async function getCatalog(apiKey, offset = 0) {
+  if (offset > 0) {
+    return [];
+  }
+  const options = await getDefaultOptions(apiKey);
+  const AD = new AllDebridClient(apiKey, options);
+  return AD.magnet.status()
+      .then(response => response.data.magnets)
+      .then(torrents => (torrents || [])
+          .filter(torrent => statusReady(torrent.statusCode))
+          .map(torrent => ({
+            id: `${KEY}:${torrent.id}`,
+            type: Type.OTHER,
+            name: torrent.filename
+          })));
+}
+
+async function getItemMeta(itemId, apiKey) {
+  const options = await getDefaultOptions(apiKey);
+  const AD = new AllDebridClient(apiKey, options);
+  return AD.magnet.status(itemId)
+      .then(response => response.data.magnets)
+      .then(torrent => ({
+        id: `${KEY}:${torrent.id}`,
+        type: Type.OTHER,
+        name: torrent.filename,
+        videos: torrent.links
+            .filter(file => isVideo(file.filename))
+            .map((file, index) => ({
+              id: `${KEY}:${torrent.id}:${index}`,
+              title: file.filename,
+              released: new Date(torrent.uploadDate * 1000).toISOString(),
+              streams: [
+                { url: `${apiKey}/${torrent.hash.toLowerCase()}/${encodeURIComponent(file.filename)}/${index}` }
+              ]
+            }))
+      }))
+}
+
 async function resolve({ ip, apiKey, infoHash, cachedEntryInfo, fileIndex }) {
   console.log(`Unrestricting AllDebrid ${infoHash} [${fileIndex}]`);
   const options = await getDefaultOptions(apiKey, ip);
   const AD = new AllDebridClient(apiKey, options);
+
+  return _resolve(AD, infoHash, cachedEntryInfo, fileIndex)
+      .catch(error => {
+        if (errorExpiredSubscriptionError(error)) {
+          console.log(`Access denied to AllDebrid ${infoHash} [${fileIndex}]`);
+          return StaticResponse.FAILED_ACCESS;
+        }
+        return Promise.reject(`Failed AllDebrid adding torrent ${error}`);
+      });
+}
+
+async function _resolve(AD, infoHash, cachedEntryInfo, fileIndex) {
   const torrent = await _createOrFindTorrent(AD, infoHash);
   if (torrent && statusReady(torrent.statusCode)) {
     return _unrestrictLink(AD, torrent, cachedEntryInfo, fileIndex);
@@ -41,10 +95,8 @@ async function resolve({ ip, apiKey, infoHash, cachedEntryInfo, fileIndex }) {
   } else if (torrent && statusHandledError(torrent.statusCode)) {
     console.log(`Retrying downloading to AllDebrid ${infoHash} [${fileIndex}]...`);
     return _retryCreateTorrent(AD, infoHash, cachedEntryInfo, fileIndex);
-  } else if (torrent && errorExpiredSubscriptionError(torrent)) {
-    console.log(`Access denied to AllDebrid ${infoHash} [${fileIndex}]`);
-    return StaticResponse.FAILED_ACCESS;
   }
+
   return Promise.reject(`Failed AllDebrid adding torrent ${torrent}`);
 }
 
@@ -126,4 +178,4 @@ function errorExpiredSubscriptionError(error) {
   return ['MUST_BE_PREMIUM', 'MAGNET_MUST_BE_PREMIUM', 'FREE_TRIAL_LIMIT_REACHED'].includes(error.code);
 }
 
-module.exports = { getCachedStreams, resolve };
+module.exports = { getCachedStreams, resolve, getCatalog, getItemMeta };

addon/moch/moch.js

@@ -88,6 +88,31 @@ async function resolve(parameters) {
   }));
 }
 
+async function getMochCatalog(mochKey, config) {
+  const moch = MOCHS[mochKey];
+  if (!moch) {
+    return Promise.reject(`Not a valid moch provider: ${mochKey}`);
+  }
+
+  return moch.instance.getCatalog(config[moch.key], config.skip);
+}
+
+async function getMochItemMeta(mochKey, itemId, config) {
+  const moch = MOCHS[mochKey];
+  if (!moch) {
+    return Promise.reject(`Not a valid moch provider: ${mochKey}`);
+  }
+
+  return moch.instance.getItemMeta(itemId, config[moch.key])
+      .then(meta => {
+        meta.videos
+            .map(video => video.streams)
+            .reduce((a, b) => a.concat(b), [])
+            .forEach(stream => stream.url = `${RESOLVER_HOST}/${moch.key}/${stream.url}`)
+        return meta;
+      });
+}
+
 function populateCachedLinks(streams, mochResult) {
   streams
       .filter(stream => stream.infoHash)
@@ -120,4 +145,4 @@ function populateDownloadLinks(streams, mochResults) {
   return streams;
 }
 
-module.exports = { applyMochs, resolve, MochOptions: MOCHS }
+module.exports = { applyMochs, getMochCatalog, getMochItemMeta, resolve, MochOptions: MOCHS }

addon/moch/realdebrid.js

@@ -1,5 +1,6 @@
 const RealDebridClient = require('real-debrid-api');
 const { encode } = require('magnet-uri');
+const { Type } = require('../lib/types');
 const { isVideo, isArchive } = require('../lib/extension');
 const delay = require('./delay');
 const StaticResponse = require('./static');
@@ -7,6 +8,8 @@ const { getRandomProxy, getProxyAgent, getRandomUserAgent, blacklistProxy } = re
 const { cacheWrapProxy, cacheUserAgent, uncacheProxy } = require('../lib/cache');
 
 const MIN_SIZE = 15728640; // 15 MB
+const CATALOG_MAX_PAGE = 5;
+const KEY = "realdebrid"
 
 async function getCachedStreams(streams, apiKey) {
   const hashes = streams.map(stream => stream.infoHash);
@@ -65,6 +68,48 @@ function _getCachedFileIds(fileIndex, hosterResults) {
   return cachedTorrents.length && cachedTorrents[0] || [];
 }
 
+async function getCatalog(apiKey, offset = 0) {
+  if (offset > 0) {
+    return [];
+  }
+  const options = await getDefaultOptions(apiKey);
+  const RD = new RealDebridClient(apiKey, options);
+  let page = 1;
+  return RD.torrents.get(page - 1, page)
+      .then(torrents => torrents && torrents.length === 50 && page < CATALOG_MAX_PAGE
+          ? RD.torrents.get(page, page = page + 1).then(nextTorrents => torrents.concat(nextTorrents)).catch(() => [])
+          : torrents)
+      .then(torrents => (torrents || [])
+          .filter(torrent => statusReady(torrent.status))
+          .map(torrent => ({
+            id: `${KEY}:${torrent.id}`,
+            type: Type.OTHER,
+            name: torrent.filename
+          })));
+}
+
+async function getItemMeta(itemId, apiKey) {
+  const options = await getDefaultOptions(apiKey);
+  const RD = new RealDebridClient(apiKey, options);
+  return _getTorrentInfo(RD, itemId)
+      .then(torrent => ({
+        id: `${KEY}:${torrent.id}`,
+        type: Type.OTHER,
+        name: torrent.filename,
+        videos: torrent.files
+            .filter(file => file.selected)
+            .filter(file => isVideo(file.path))
+            .map(file => ({
+              id: `${KEY}:${torrent.id}:${file.id}`,
+              title: file.path,
+              released: torrent.added,
+              streams: [
+                { url: `${apiKey}/${torrent.hash.toLowerCase()}/null/${file.id - 1}` }
+              ]
+            }))
+      }))
+}
+
 async function resolve({ apiKey, infoHash, cachedEntryInfo, fileIndex }) {
   console.log(`Unrestricting RealDebrid ${infoHash} [${fileIndex}]`);
   const options = await getDefaultOptions(apiKey);
@@ -215,4 +260,4 @@ async function getDefaultOptions(id) {
   return { timeout: 30000, agent: agent, headers: { 'User-Agent': userAgent } };
 }
 
-module.exports = { getCachedStreams, resolve };
+module.exports = { getCachedStreams, resolve, getCatalog, getItemMeta };

addon/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "stremio-torrentio",
-  "version": "1.0.6",
+  "version": "1.0.7",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {

addon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "stremio-torrentio",
-  "version": "1.0.6",
+  "version": "1.0.7",
   "main": "addon.js",
   "scripts": {
     "start": "node index.js"

addon/serverless.js

@@ -1,12 +1,13 @@
 const rateLimit = require('express-rate-limit');
 const { getRouter } = require('stremio-addon-sdk');
 const addonInterface = require('./addon');
+const qs = require('querystring')
 const { manifest } = require('./lib/manifest');
 const parseConfiguration = require('./lib/configuration');
 const landingTemplate = require('./lib/landingTemplate');
 const moch = require('./moch/moch');
 
-const router = getRouter(addonInterface);
+const router = getRouter({ ...addonInterface, manifest: manifest() });
 const limiter = rateLimit({
   windowMs: 10 * 1000, // 10 seconds
   max: 10, // limit each IP to 10 requests per windowMs
@@ -27,16 +28,17 @@ router.get('/:configuration?/configure', (req, res) => {
   res.end(landingHTML);
 });
 
-router.get('/:configuration/manifest.json', (req, res) => {
-  const configValues = parseConfiguration(req.params.configuration);
+router.get('/:configuration?/manifest.json', (req, res) => {
+  const configValues = parseConfiguration(req.params.configuration || '');
   const manifestBuf = JSON.stringify(manifest(configValues));
   res.setHeader('Content-Type', 'application/json; charset=utf-8');
   res.end(manifestBuf)
 });
 
-router.get('/:configuration/:resource/:type/:id.json', (req, res, next) => {
+router.get('/:configuration/:resource/:type/:id/:extra?.json', (req, res, next) => {
   const { configuration, resource, type, id } = req.params;
-  const configValues = parseConfiguration(configuration);
+  const extra = req.params.extra ? qs.parse(req.url.split('/').pop().slice(0, -5)) : {}
+  const configValues = { ...extra, ...parseConfiguration(configuration) };
   addonInterface.get(resource, type, id, configValues)
       .then(resp => {
         const cacheHeaders = {