add torrentio catalogs addon

catalogs/.dockerignore

@@ -0,0 +1,3 @@
+node_modules
+npm-debug.log
+**/.env

catalogs/Dockerfile

@@ -0,0 +1,12 @@
+FROM node:16-alpine
+
+RUN apk update && apk upgrade && \
+    apk add --no-cache git
+
+WORKDIR /home/node/app
+
+COPY package*.json ./
+RUN npm ci --only-production
+COPY . .
+
+CMD [ "node", "index.js" ]

catalogs/addon.js

@@ -0,0 +1,85 @@
+const Bottleneck = require('bottleneck');
+const moment = require('moment')
+const { addonBuilder } = require('stremio-addon-sdk');
+const { createManifest, genres } = require('./lib/manifest');
+const { getMetas } = require('./lib/metadata');
+const { cacheWrapCatalog, cacheWrapIds } = require('./lib/cache');
+const repository = require('./lib/repository');
+
+
+const CACHE_MAX_AGE = process.env.CACHE_MAX_AGE || 4 * 60 * 60; // 4 hours in seconds
+const STALE_REVALIDATE_AGE = 4 * 60 * 60; // 4 hours
+const STALE_ERROR_AGE = 7 * 24 * 60 * 60; // 7 days
+
+const manifest = createManifest();
+const builder = new addonBuilder(manifest);
+const limiter = new Bottleneck({
+  maxConcurrent: process.env.LIMIT_MAX_CONCURRENT || 20,
+  highWater: process.env.LIMIT_QUEUE_SIZE || 50,
+  strategy: Bottleneck.strategy.OVERFLOW
+});
+
+
+builder.defineCatalogHandler((args) => {
+  const offset = parseInt(args.extra.skip || '0', 10);
+  const genre = args.extra.genre || genres[3];
+  const catalog = manifest.catalogs.find(c => c.id === args.id);
+  console.log(`Incoming catalog ${args.id} request with genre=${genre} and skip=${offset}`)
+  if (!catalog) {
+    return Promise.reject(`No catalog found for with id: ${args.id}`)
+  }
+
+  const cacheKey = `${args.id}|${genre}|${offset}`
+  return limiter.schedule(() => cacheWrapCatalog(cacheKey, () => getCatalog(catalog, genre, offset)))
+      .then(metas => ({
+        metas: metas,
+        cacheMaxAge: CACHE_MAX_AGE,
+        staleRevalidate: STALE_REVALIDATE_AGE,
+        staleError: STALE_ERROR_AGE
+      }))
+      .catch(error => Promise.reject(`Failed retrieving catalog ${args.id}: ${JSON.stringify(error)}`));
+})
+
+async function getCursor(catalog, genre, offset) {
+  if (offset === 0) {
+    return undefined;
+  }
+  const previousCacheKey = `${catalog.id}|${genre}|${offset - catalog.pageSize}`;
+  return cacheWrapCatalog(previousCacheKey, () => Promise.reject("cursor not found"))
+      .then(metas => metas[metas.length - 1])
+      .then(meta => meta.id.replace('kitsu:', ''))
+}
+
+async function getCatalog(catalog, genre, offset) {
+  const cursor = await getCursor(catalog, genre, offset)
+  const startDate = getStartDate(genre)?.toISOString();
+  const endDate = getEndDate(genre)?.toISOString();
+  const cacheKey = `${catalog.id}|${genre}`
+
+  return cacheWrapIds(cacheKey, () => repository.getIds(catalog.type, startDate, endDate))
+      .then(ids => ids.slice(ids.indexOf(cursor) + 1))
+      .then(ids => getMetas(ids, catalog.type))
+      .then(metas => metas.slice(0, catalog.pageSize));
+}
+
+function getStartDate(genre) {
+  switch (genre) {
+    case genres[0]: return moment().utc().startOf('day');
+    case genres[1]: return moment().utc().subtract(1, 'day').startOf('day');
+    case genres[2]: return moment().utc().subtract(7, 'day').startOf('isoWeek');
+    case genres[3]: return moment().utc().subtract(30, 'day').startOf('month');
+    default: return undefined;
+  }
+}
+
+function getEndDate( genre) {
+  switch (genre) {
+    case genres[0]: return moment().utc();
+    case genres[1]: return moment().utc().subtract(1, 'day').endOf('day');
+    case genres[2]: return moment().utc().subtract(7, 'day').endOf('isoWeek');
+    case genres[3]: return moment().utc().subtract(30, 'day').endOf('month');
+    default: return undefined;
+  }
+}
+
+module.exports = builder.getInterface();

catalogs/index.js

@@ -0,0 +1,9 @@
+const express = require('express');
+const serverless = require('./serverless');
+
+const app = express();
+
+app.use((req, res, next) => serverless(req, res, next));
+app.listen(process.env.PORT || 7000, () => {
+  console.log(`Started addon at: http://localhost:${process.env.PORT || 7000}`);
+});

catalogs/lib/cache.js

@@ -0,0 +1,48 @@
+const cacheManager = require('cache-manager');
+const mangodbStore = require('cache-manager-mongodb');
+
+const CATALOG_TTL = process.env.STREAM_TTL || 12 * 60 * 60; // 12 hours
+
+const MONGO_URI = process.env.MONGODB_URI;
+
+const remoteCache = initiateRemoteCache();
+
+function initiateRemoteCache() {
+  if (MONGO_URI) {
+    return cacheManager.caching({
+      store: mangodbStore,
+      uri: MONGO_URI,
+      options: {
+        collection: 'torrentio_catalog_collection',
+        useNewUrlParser: true,
+        useUnifiedTopology: false,
+        ttl: CATALOG_TTL
+      },
+      ttl: CATALOG_TTL,
+      ignoreCacheErrors: true
+    });
+  } else {
+    return cacheManager.caching({
+      store: 'memory',
+      ttl: CATALOG_TTL
+    });
+  }
+}
+
+function cacheWrap(cache, key, method, options) {
+  if (!cache) {
+    return method();
+  }
+  return cache.wrap(key, method, options);
+}
+
+function cacheWrapCatalog(key, method) {
+  return cacheWrap(remoteCache, key, method, { ttl: CATALOG_TTL });
+}
+
+function cacheWrapIds(key, method) {
+  return cacheWrap(remoteCache, `ids|${key}`, method, { ttl: CATALOG_TTL });
+}
+
+module.exports = { cacheWrapCatalog, cacheWrapIds };
+

catalogs/lib/landingTemplate.js

@@ -0,0 +1,276 @@
+const STYLESHEET = `
+* {
+   box-sizing: border-box;
+}
+
+body,
+html {
+   margin: 0;
+   padding: 0;
+   width: 100%;
+   height: 100%
+}
+
+html {
+   background-size: auto 100%;
+   background-size: cover;
+   background-position: center center;
+   background-repeat: repeat-y;
+}
+
+body {
+   display: flex;
+   background-color: transparent;
+   font-family: 'Open Sans', Arial, sans-serif;
+   color: white;
+}
+
+h1 {
+   font-size: 4.5vh;
+   font-weight: 700;
+}
+
+h2 {
+   font-size: 2.2vh;
+   font-weight: normal;
+   font-style: italic;
+   opacity: 0.8;
+}
+
+h3 {
+   font-size: 2.2vh;
+}
+
+h1,
+h2,
+h3,
+p,
+label {
+   margin: 0;
+   text-shadow: 0 0 1vh rgba(0, 0, 0, 0.15);
+}
+
+p {
+   font-size: 1.75vh;
+}
+
+ul {
+   font-size: 1.75vh;
+   margin: 0;
+   margin-top: 1vh;
+   padding-left: 3vh;
+}
+
+a {
+   color: green
+}
+
+a.install-link {
+   text-decoration: none
+}
+
+button {
+   border: 0;
+   outline: 0;
+   color: white;
+   background: #8A5AAB;
+   padding: 1.2vh 3.5vh;
+   margin: auto;
+   text-align: center;
+   font-family: 'Open Sans', Arial, sans-serif;
+   font-size: 2.2vh;
+   font-weight: 600;
+   cursor: pointer;
+   display: block;
+   box-shadow: 0 0.5vh 1vh rgba(0, 0, 0, 0.2);
+   transition: box-shadow 0.1s ease-in-out;
+}
+
+button:hover {
+   box-shadow: none;
+}
+
+button:active {
+   box-shadow: 0 0 0 0.5vh white inset;
+}
+
+#addon {
+   width: 90vh;
+   margin: auto;
+   padding-left: 10%;
+   padding-right: 10%;
+   background: rgba(0, 0, 0, 0.60);
+}
+
+.logo {
+   height: 14vh;
+   width: 14vh;
+   margin: auto;
+   margin-bottom: 3vh;
+}
+
+.logo img {
+   width: 100%;
+}
+
+.name, .version {
+   display: inline-block;
+   vertical-align: top;
+}
+
+.name {
+   line-height: 5vh;
+}
+
+.version {
+   position: absolute;
+   line-height: 5vh;
+   margin-left: 1vh;
+   opacity: 0.8;
+}
+
+.contact {
+   position: absolute;
+   left: 0;
+   bottom: 4vh;
+   width: 100%;
+   text-align: center;
+}
+
+.contact a {
+   font-size: 1.4vh;
+   font-style: italic;
+}
+
+.separator {
+   margin-bottom: 4vh;
+}
+
+.label {
+  font-size: 2.2vh;
+  font-weight: 600;
+  padding: 0;
+  line-height: inherit;
+}
+
+.btn-group, .multiselect-container {
+  width: 100%;
+}
+
+.btn {
+  text-align: left;
+}
+
+.multiselect-container {
+  border: 0;
+  border-radius: 0;
+}
+
+.input, .btn {
+  height: 3.8vh;
+  width: 100%;
+  margin: auto;
+  margin-bottom: 10px;
+  padding: 6px 12px;
+  border: 0;
+  border-radius: 0;
+  outline: 0;
+  color: #333;
+  background-color: rgb(255, 255, 255);
+  box-shadow: 0 0.5vh 1vh rgba(0, 0, 0, 0.2);
+}
+`;
+const { Providers } = require('../../addon/lib/filter');
+
+function landingTemplate(manifest, config = {}) {
+  const providers = config.providers || [];
+
+  const background = manifest.background || 'https://dl.strem.io/addon-background.jpg';
+  const logo = manifest.logo || 'https://dl.strem.io/addon-logo.png';
+  const contactHTML = manifest.contactEmail ?
+      `<div class="contact">
+         <p>Contact ${manifest.name} creator:</p>
+         <a href="mailto:${manifest.contactEmail}">${manifest.contactEmail}</a>
+      </div>` : '<div class="separator"></div>';
+  const providersHTML = Providers.options
+      .map(provider => `<option value="${provider.key}">${provider.foreign || ''}${provider.label}</option>`)
+      .join('\n');
+  const stylizedTypes = manifest.types
+      .map(t => t[0].toUpperCase() + t.slice(1) + (t !== 'series' ? 's' : ''));
+
+  return `
+   <!DOCTYPE html>
+   <html style="background-image: url(${background});">
+
+   <head>
+      <meta charset="utf-8">
+      <title>${manifest.name} - Stremio Addon</title>
+      <link rel="shortcut icon" href="${logo}" type="image/x-icon">
+      <link href="https://fonts.googleapis.com/css?family=Open+Sans:400,600,700&display=swap" rel="stylesheet">
+      <script src="https://code.jquery.com/jquery-3.5.1.slim.min.js"></script>
+      <script src="https://cdn.jsdelivr.net/npm/popper.js@1.16.1/dist/umd/popper.min.js"></script>
+      <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.5.2/js/bootstrap.min.js"></script>
+      <link href="https://stackpath.bootstrapcdn.com/bootstrap/4.5.2/css/bootstrap.min.css" rel="stylesheet" >
+      <script src="https://cdnjs.cloudflare.com/ajax/libs/bootstrap-multiselect/0.9.15/js/bootstrap-multiselect.min.js"></script>
+      <link href="https://cdnjs.cloudflare.com/ajax/libs/bootstrap-multiselect/0.9.15/css/bootstrap-multiselect.css" rel="stylesheet"/>
+      <style>${STYLESHEET}</style>
+   </head>
+
+	<body>
+      <div id="addon">
+         <div class="logo">
+            <img src="${logo}">
+         </div>
+         <h1 class="name">${manifest.name}</h1>
+         <h2 class="version">${manifest.version || '0.0.0'}</h2>
+         <h2 class="description">${manifest.description || ''}</h2>
+
+         <div class="separator"></div>
+
+         <h3 class="gives">This addon has more :</h3>
+         <ul>
+            ${stylizedTypes.map(t => `<li>${t}</li>`).join('')}
+         </ul>
+
+         <div class="separator"></div>
+         
+         <label class="label" for="iProviders">Providers:</label>
+         <select id="iProviders" class="input" name="providers[]" multiple="multiple">
+            ${providersHTML}
+         </select>
+         
+         <div class="separator"></div>
+
+         <a id="installLink" class="install-link" href="#">
+            <button name="Install">INSTALL</button>
+         </a>
+         ${contactHTML}
+      </div>
+      <script type="text/javascript">
+          $(document).ready(function() {
+              $('#iProviders').multiselect({ 
+                  nonSelectedText: 'All providers',
+                  onChange: () => generateInstallLink()
+              });
+              $('#iProviders').multiselect('select', [${providers.map(provider => '"' + provider + '"')}]);
+              generateInstallLink();
+          });
+          
+          function generateInstallLink() {
+              const providersValue = $('#iProviders').val().join(',') || '';
+              const providers = providersValue.length && providersValue;
+              const configurationValue = [
+                    ['${Providers.key}', providers],
+                  ]
+                  .filter(([_, value]) => value.length)
+                  .map(([key, value]) => key + '=' + value).join('|');
+              const configuration = configurationValue && configurationValue.length ? '/' + configurationValue : '';
+              installLink.href = 'stremio://' + window.location.host + configuration + '/manifest.json';
+          }
+      </script>
+	</body>
+
+	</html>`
+}
+
+module.exports = landingTemplate;

catalogs/lib/manifest.js

@@ -0,0 +1,54 @@
+const { Type } = require('../../addon/lib/types');
+
+const genres = [
+    'Today',
+    'Yesterday',
+    'Last Week',
+    'Last Month',
+    'All Time'
+]
+
+function createManifest() {
+  return {
+    id: 'com.stremio.torrentio.catalog.addon',
+    version: '1.0.0',
+    name: 'Torrentio Catalogs',
+    description: 'Provides catalogs for movies/series/anime based on top seeded torrents',
+    logo: `https://i.ibb.co/w4BnkC9/GwxAcDV.png`,
+    background: `https://i.ibb.co/VtSfFP9/t8wVwcg.jpg`,
+    types: [Type.MOVIE, Type.SERIES, Type.ANIME],
+    resources: ['catalog'],
+    catalogs: [
+      {
+        id: 'top-movies',
+        type: Type.MOVIE,
+        name: "Top seeded movies",
+        pageSize: 20,
+        extra: [{ name: 'genre', options: genres }, { name: 'skip' }],
+        genres: genres
+      },
+      {
+        id: 'top-series',
+        type: Type.SERIES,
+        name: "Top seeded series",
+        pageSize: 20,
+        extra: [{ name: 'genre', options: genres }, { name: 'skip' }],
+        genres: genres
+      },
+      {
+        id: 'top-anime',
+        type: Type.ANIME,
+        name: "Top seeded anime",
+        pageSize: 20,
+        extra: [{ name: 'genre', options: genres }, { name: 'skip' }],
+        genres: genres
+      }
+    ],
+    behaviorHints: {
+      configurable: false, // @TODO might enable configuration to configure providers
+      configurationRequired: false
+    }
+  };
+}
+
+module.exports = { createManifest, genres };

catalogs/lib/metadata.js

@@ -0,0 +1,35 @@
+const axios = require('axios');
+const { Type } = require('../../addon/lib/types');
+
+const CINEMETA_URL = 'https://v3-cinemeta.strem.io';
+const KITSU_URL = 'https://anime-kitsu.strem.fun';
+const TIMEOUT = 30000;
+const MAX_SIZE = 50;
+
+function getMetas(ids, type) {
+  if (!ids.length || !type) {
+    return [];
+  }
+
+  return _requestMetadata(ids, type)
+      .catch((error) => {
+        throw new Error(`failed metadata ${type} query due: ${error.message}`);
+      });
+}
+
+function _requestMetadata(ids, type) {
+  const url = _getUrl(ids, type);
+  return axios.get(url, { timeout: TIMEOUT })
+      .then(response => response?.data?.metas || response?.data?.metasDetailed || [])
+      .then(metas => metas.filter(meta => meta));
+}
+
+function _getUrl(ids, type) {
+  const joinedIds = ids.slice(0, MAX_SIZE).join(',');
+  if (type === Type.ANIME) {
+    return `${KITSU_URL}/catalog/${type}/kitsu-anime-list/lastVideosIds=${joinedIds}.json`
+  }
+  return `${CINEMETA_URL}/catalog/${type}/last-videos/lastVideosIds=${joinedIds}.json`
+}
+
+module.exports = { getMetas };

catalogs/lib/repository.js

@@ -0,0 +1,29 @@
+const { Sequelize, QueryTypes } = require('sequelize');
+const { Type } = require('../../addon/lib/types');
+
+const DATABASE_URI = process.env.DATABASE_URI;
+
+const database = new Sequelize(DATABASE_URI, { logging: false });
+
+async function getIds(type, startDate, endDate) {
+  const idName = type === Type.ANIME ? 'kitsuId' : 'imdbId';
+  const episodeCondition = type === Type.SERIES
+      ? 'AND files."imdbSeason" IS NOT NULL AND files."imdbEpisode" IS NOT NULL'
+      : '';
+  const dateCondition = startDate && endDate
+      ? `AND "uploadDate" BETWEEN '${startDate}' AND '${endDate}'`
+      : ''
+  const query = `SELECT files."${idName}"
+        FROM (SELECT torrents."infoHash", torrents.seeders FROM torrents
+                WHERE seeders > 0 AND type = '${type}' ${dateCondition}
+              ) as torrents
+        JOIN files ON torrents."infoHash" = files."infoHash"
+        WHERE files."${idName}" IS NOT NULL ${episodeCondition}
+        GROUP BY files."${idName}"
+        ORDER BY max(torrents.seeders) DESC
+        LIMIT 5000`
+  const results = await database.query(query, { type: QueryTypes.SELECT });
+  return results.map(result => `${result.imdbId || result.kitsuId}`);
+}
+
+module.exports = { getIds };

catalogs/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "stremio-torrentio-catalogs",
+  "version": "1.0.0",
+  "main": "index.js",
+  "scripts": {
+    "start": "node index.js"
+  },
+  "engines": {
+    "node": "16.x"
+  },
+  "author": "TheBeastLT <pauliox@beyond.lt>",
+  "license": "MIT",
+  "dependencies": {
+    "axios": "^0.21.4",
+    "bottleneck": "^2.19.5",
+    "cache-manager": "^3.4.4",
+    "cache-manager-mongodb": "^0.3.0",
+    "moment": "^2.29.1",
+    "pg": "^8.7.1",
+    "pg-hstore": "^2.3.4",
+    "request-ip": "^2.1.3",
+    "sequelize": "^4.43.0",
+    "stremio-addon-sdk": "^1.6.1"
+  }
+}

catalogs/serverless.js

@@ -0,0 +1,69 @@
+const { getRouter } = require('stremio-addon-sdk');
+const addonInterface = require('./addon');
+const qs = require('querystring')
+const { parseConfiguration } = require('../addon/lib/configuration');
+const { createManifest } = require('./lib/manifest');
+const landingTemplate = require('./lib/landingTemplate');
+
+const router = getRouter(addonInterface);
+
+// router.get('/', (_, res) => {
+//   res.redirect('/configure')
+//   res.end();
+// });
+//
+// router.get('/:configuration?/configure', (req, res) => {
+//   const configValues = parseConfiguration(req.params.configuration || '');
+//   const landingHTML = landingTemplate(createManifest(configValues), configValues);
+//   res.setHeader('content-type', 'text/html');
+//   res.end(landingHTML);
+// });
+
+router.get('/:configuration?/manifest.json', (req, res) => {
+  const configValues = parseConfiguration(req.params.configuration || '');
+  const manifestBuf = JSON.stringify(createManifest(configValues));
+  res.setHeader('Content-Type', 'application/json; charset=utf-8');
+  res.end(manifestBuf)
+});
+
+router.get('/:configuration/:resource/:type/:id/:extra?.json', (req, res, next) => {
+  const { configuration, resource, type, id } = req.params;
+  const extra = req.params.extra ? qs.parse(req.url.split('/').pop().slice(0, -5)) : {}
+  const configValues = { ...extra, ...parseConfiguration(configuration) };
+  addonInterface.get(resource, type, id, configValues)
+      .then(resp => {
+        const cacheHeaders = {
+          cacheMaxAge: 'max-age',
+          staleRevalidate: 'stale-while-revalidate',
+          staleError: 'stale-if-error'
+        };
+        const cacheControl = Object.keys(cacheHeaders)
+            .map(prop => resp[prop] && cacheHeaders[prop] + '=' + resp[prop])
+            .filter(val => !!val).join(', ');
+
+        res.setHeader('Cache-Control', `${cacheControl}, public`);
+        res.setHeader('Content-Type', 'application/json; charset=utf-8');
+        res.end(JSON.stringify(resp));
+      })
+      .catch(err => {
+        if (err.noHandler) {
+          if (next) {
+            next()
+          } else {
+            res.writeHead(404);
+            res.end(JSON.stringify({ err: 'not found' }));
+          }
+        } else {
+          console.error(err);
+          res.writeHead(500);
+          res.end(JSON.stringify({ err: 'handler error' }));
+        }
+      });
+});
+
+module.exports = function (req, res) {
+  router(req, res, function () {
+    res.statusCode = 404;
+    res.end();
+  });
+};