From 3bf13e68baea29e15683a6079970d4099e879740 Mon Sep 17 00:00:00 2001 From: TheBeastLT Date: Sun, 28 Jan 2024 02:12:41 +0200 Subject: [PATCH 01/12] try use p-limit instead of bottleneck --- addon/addon.js | 6 ++++-- addon/package-lock.json | 26 ++++++++++++++++++++++++++ addon/package.json | 1 + 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/addon/addon.js b/addon/addon.js index a017a4e..93dec7d 100644 --- a/addon/addon.js +++ b/addon/addon.js @@ -10,6 +10,7 @@ import applyFilters from './lib/filter.js'; import { applyMochs, getMochCatalog, getMochItemMeta } from './moch/moch.js'; import StaticLinks from './moch/static.js'; import { createNamedQueue } from "./lib/namedQueue.js"; +import pLimit from "p-limit"; const CACHE_MAX_AGE = parseInt(process.env.CACHE_MAX_AGE) || 60 * 60; // 1 hour in seconds const CACHE_MAX_AGE_EMPTY = 60; // 60 seconds @@ -24,6 +25,7 @@ const limiter = new Bottleneck({ highWater: process.env.LIMIT_QUEUE_SIZE || 100, strategy: Bottleneck.strategy.OVERFLOW }); +const newLimiter = pLimit(40) const limiterOptions = { expiration: 2 * 60 * 1000 } builder.defineStreamHandler((args) => { @@ -69,14 +71,14 @@ builder.defineMetaHandler((args) => { }) async function resolveStreams(args) { - return cacheWrapStream(args.id, () => limiter.schedule(limiterOptions, () => streamHandler(args) + return cacheWrapStream(args.id, () => newLimiter(() => streamHandler(args) .then(records => records .sort((a, b) => b.torrent.seeders - a.torrent.seeders || b.torrent.uploadDate - a.torrent.uploadDate) .map(record => toStreamInfo(record))))); } async function streamHandler(args) { - console.log(`Current stats: `, limiter.counts()) + console.log(`Pending count: ${newLimiter.pendingCount}, active count: ${newLimiter.activeCount}`, ) if (args.type === Type.MOVIE) { return movieRecordsHandler(args); } else if (args.type === Type.SERIES) { diff --git a/addon/package-lock.json b/addon/package-lock.json index b19be54..e58e9b2 100644 --- a/addon/package-lock.json +++ b/addon/package-lock.json @@ -22,6 +22,7 @@ "name-to-imdb": "^3.0.4", "named-queue": "^2.2.1", "offcloud-api": "^1.0.2", + "p-limit": "^5.0.0", "parse-torrent-title": "git://github.com/TheBeastLT/parse-torrent-title.git#022408972c2a040f846331a912a6a8487746a654", "pg": "^8.10.0", "premiumize-api": "^1.0.3", @@ -1428,6 +1429,20 @@ "node": ">=0.10.0" } }, + "node_modules/p-limit": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-5.0.0.tgz", + "integrity": "sha512-/Eaoq+QyLSiXQ4lyYV23f14mZRQcXnxfHrN0vCai+ak9G0pp9iEQukIIZq5NccEvwRB8PUnZT0KsOoDCINS1qQ==", + "dependencies": { + "yocto-queue": "^1.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/packet-reader": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", @@ -2499,6 +2514,17 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + }, + "node_modules/yocto-queue": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.0.0.tgz", + "integrity": "sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==", + "engines": { + "node": ">=12.20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } } } } diff --git a/addon/package.json b/addon/package.json index 16c707f..6b52a93 100644 --- a/addon/package.json +++ b/addon/package.json @@ -22,6 +22,7 @@ "name-to-imdb": "^3.0.4", "named-queue": "^2.2.1", "offcloud-api": "^1.0.2", + "p-limit": "^5.0.0", "parse-torrent-title": "git://github.com/TheBeastLT/parse-torrent-title.git#022408972c2a040f846331a912a6a8487746a654", "pg": "^8.10.0", "premiumize-api": "^1.0.3", From 81218bbc8d0e9026886f664b5b2aae6eb465c296 Mon Sep 17 00:00:00 2001 From: TheBeastLT Date: Sun, 28 Jan 2024 21:44:54 +0200 Subject: [PATCH 02/12] increase mongo connection pool --- addon/lib/cache.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addon/lib/cache.js b/addon/lib/cache.js index 7f46d0a..6cf193d 100644 --- a/addon/lib/cache.js +++ b/addon/lib/cache.js @@ -30,7 +30,7 @@ function initiateRemoteCache() { options: { collection: 'torrentio_addon_collection', socketTimeoutMS: 30000, - poolSize: 150, + poolSize: 300, useNewUrlParser: true, useUnifiedTopology: false, ttl: STREAM_EMPTY_TTL From cc0d8cbdabcff010b394ac98809fd5be95581270 Mon Sep 17 00:00:00 2001 From: TheBeastLT Date: Tue, 30 Jan 2024 08:49:58 +0200 Subject: [PATCH 03/12] bump node base image to 21 --- addon/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addon/Dockerfile b/addon/Dockerfile index 9aaf66a..14dd96e 100644 --- a/addon/Dockerfile +++ b/addon/Dockerfile @@ -1,4 +1,4 @@ -FROM node:16-alpine +FROM node:21-alpine RUN apk update && apk upgrade && \ apk add --no-cache git From ae955e6634f53ac022209c87df7f87d8c7648a7d Mon Sep 17 00:00:00 2001 From: TheBeastLT Date: Tue, 30 Jan 2024 08:50:12 +0200 Subject: [PATCH 04/12] update deploy workflow --- .github/workflows/deploy_addon.yml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy_addon.yml b/.github/workflows/deploy_addon.yml index 1efdd28..f7f369b 100644 --- a/.github/workflows/deploy_addon.yml +++ b/.github/workflows/deploy_addon.yml @@ -9,6 +9,13 @@ on: jobs: build: runs-on: ubuntu-latest + strategy: + matrix: + include: + - ssh-host: ${{ secrets.SSH_HOST }} + ssh-key: ${{ secrets.SSH_KEY }} + - ssh-host: ${{ secrets.SSH_HOST_2 }} + ssh-key: ${{ secrets.SSH_KEY_2 }} steps: - uses: actions/checkout@v1 @@ -21,10 +28,10 @@ jobs: - name: Upload Docker image uses: appleboy/scp-action@master with: - host: ${{ secrets.SSH_HOST }} + host: ${{ matrix.ssh-host }} username: ${{ secrets.SSH_USERNAME }} port: ${{ secrets.SSH_PORT }} - key: ${{ secrets.SSH_KEY }} + key: ${{ matrix.ssh-key }} source: torrentio_addon_latest.tar target: /tmp/docker overwrite: true @@ -32,13 +39,13 @@ jobs: - name: Deploy Docker image uses: appleboy/ssh-action@master with: - host: ${{ secrets.SSH_HOST }} + host: ${{ matrix.ssh-host }} username: ${{ secrets.SSH_USERNAME }} port: ${{ secrets.SSH_PORT }} - key: ${{ secrets.SSH_KEY }} + key: ${{ matrix.ssh-key }} script: | docker load -i /tmp/docker/torrentio_addon_latest.tar docker stop torrentio-addon docker rm torrentio-addon - docker run -p 80:7000 -d --name torrentio-addon --restart always -e MONGODB_URI=${{ secrets.MONGODB_URI }} -e DATABASE_URI=${{ secrets.DATABASE_URI }} -e PROXY_HOSTS=${{ secrets.PROXY_HOSTS }} -e PROXY_USERNAME=${{ secrets.PROXY_USERNAME }} -e PROXY_PASSWORD=${{ secrets.PROXY_PASSWORD }} -e METRICS_USER=${{ secrets.METRICS_USER }} -e METRICS_PASSWORD=${{ secrets.METRICS_PASSWORD }} torrentio-addon:latest + docker run -p 7000:7000 -d --name torrentio-addon --restart always -e MONGODB_URI=${{ secrets.MONGODB_URI }} -e DATABASE_URI=${{ secrets.DATABASE_URI }} -e PROXY_HOSTS=${{ secrets.PROXY_HOSTS }} -e PROXY_USERNAME=${{ secrets.PROXY_USERNAME }} -e PROXY_PASSWORD=${{ secrets.PROXY_PASSWORD }} -e METRICS_USER=${{ secrets.METRICS_USER }} -e METRICS_PASSWORD=${{ secrets.METRICS_PASSWORD }} torrentio-addon:latest docker image prune -f From 85d9f3d406449299e192120f0e064c672cbdcc34 Mon Sep 17 00:00:00 2001 From: TheBeastLT Date: Tue, 30 Jan 2024 08:55:17 +0200 Subject: [PATCH 05/12] update deploy workflow --- .github/workflows/deploy_addon.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/deploy_addon.yml b/.github/workflows/deploy_addon.yml index f7f369b..972111c 100644 --- a/.github/workflows/deploy_addon.yml +++ b/.github/workflows/deploy_addon.yml @@ -12,10 +12,10 @@ jobs: strategy: matrix: include: - - ssh-host: ${{ secrets.SSH_HOST }} - ssh-key: ${{ secrets.SSH_KEY }} - - ssh-host: ${{ secrets.SSH_HOST_2 }} - ssh-key: ${{ secrets.SSH_KEY_2 }} + - ssh-host: SSH_HOST + ssh-key: SSH_KEY + - ssh-host: SSH_HOST_2 + ssh-key: SSH_KEY_2 steps: - uses: actions/checkout@v1 @@ -28,10 +28,10 @@ jobs: - name: Upload Docker image uses: appleboy/scp-action@master with: - host: ${{ matrix.ssh-host }} + host: ${{ secrets[matrix.matrix.ssh-host] }} username: ${{ secrets.SSH_USERNAME }} port: ${{ secrets.SSH_PORT }} - key: ${{ matrix.ssh-key }} + key: ${{ secrets[matrix.matrix.ssh-key] }} source: torrentio_addon_latest.tar target: /tmp/docker overwrite: true @@ -39,10 +39,10 @@ jobs: - name: Deploy Docker image uses: appleboy/ssh-action@master with: - host: ${{ matrix.ssh-host }} + host: ${{ secrets[matrix.matrix.ssh-host] }} username: ${{ secrets.SSH_USERNAME }} port: ${{ secrets.SSH_PORT }} - key: ${{ matrix.ssh-key }} + key: ${{ secrets[matrix.matrix.ssh-key] }} script: | docker load -i /tmp/docker/torrentio_addon_latest.tar docker stop torrentio-addon From cca0331caada707e0f40fbad31bff4244139e300 Mon Sep 17 00:00:00 2001 From: TheBeastLT Date: Tue, 30 Jan 2024 08:56:27 +0200 Subject: [PATCH 06/12] remove bottleneck --- addon/addon.js | 7 ------- 1 file changed, 7 deletions(-) diff --git a/addon/addon.js b/addon/addon.js index 93dec7d..d2d7e0e 100644 --- a/addon/addon.js +++ b/addon/addon.js @@ -1,4 +1,3 @@ -import Bottleneck from 'bottleneck'; import { addonBuilder } from 'stremio-addon-sdk'; import { Type } from './lib/types.js'; import { dummyManifest } from './lib/manifest.js'; @@ -20,13 +19,7 @@ const STALE_ERROR_AGE = 7 * 24 * 60 * 60; // 7 days const builder = new addonBuilder(dummyManifest()); const requestQueue = createNamedQueue(Infinity); -const limiter = new Bottleneck({ - maxConcurrent: process.env.LIMIT_MAX_CONCURRENT || 40, - highWater: process.env.LIMIT_QUEUE_SIZE || 100, - strategy: Bottleneck.strategy.OVERFLOW -}); const newLimiter = pLimit(40) -const limiterOptions = { expiration: 2 * 60 * 1000 } builder.defineStreamHandler((args) => { if (!args.id.match(/tt\d+/i) && !args.id.match(/kitsu:\d+/i)) { From fb8148a737610c7334cfc2e78891ccec950b1ae0 Mon Sep 17 00:00:00 2001 From: TheBeastLT Date: Tue, 30 Jan 2024 08:58:28 +0200 Subject: [PATCH 07/12] update deploy workflow --- .github/workflows/deploy_addon.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy_addon.yml b/.github/workflows/deploy_addon.yml index 972111c..a71786e 100644 --- a/.github/workflows/deploy_addon.yml +++ b/.github/workflows/deploy_addon.yml @@ -5,6 +5,7 @@ on: branches: master paths: - 'addon/**' + - '.github/**' jobs: build: @@ -28,10 +29,10 @@ jobs: - name: Upload Docker image uses: appleboy/scp-action@master with: - host: ${{ secrets[matrix.matrix.ssh-host] }} + host: ${{ secrets[matrix.ssh-host] }} username: ${{ secrets.SSH_USERNAME }} port: ${{ secrets.SSH_PORT }} - key: ${{ secrets[matrix.matrix.ssh-key] }} + key: ${{ secrets[matrix.ssh-key] }} source: torrentio_addon_latest.tar target: /tmp/docker overwrite: true @@ -39,10 +40,10 @@ jobs: - name: Deploy Docker image uses: appleboy/ssh-action@master with: - host: ${{ secrets[matrix.matrix.ssh-host] }} + host: ${{ secrets[matrix.ssh-host] }} username: ${{ secrets.SSH_USERNAME }} port: ${{ secrets.SSH_PORT }} - key: ${{ secrets[matrix.matrix.ssh-key] }} + key: ${{ secrets[matrix.ssh-key] }} script: | docker load -i /tmp/docker/torrentio_addon_latest.tar docker stop torrentio-addon From aaa48f585cb6f2dffa5158da1df9e8869a8e6b25 Mon Sep 17 00:00:00 2001 From: TheBeastLT Date: Tue, 30 Jan 2024 09:04:57 +0200 Subject: [PATCH 08/12] update deploy workflow --- .github/workflows/deploy_addon.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy_addon.yml b/.github/workflows/deploy_addon.yml index a71786e..8663727 100644 --- a/.github/workflows/deploy_addon.yml +++ b/.github/workflows/deploy_addon.yml @@ -48,5 +48,5 @@ jobs: docker load -i /tmp/docker/torrentio_addon_latest.tar docker stop torrentio-addon docker rm torrentio-addon - docker run -p 7000:7000 -d --name torrentio-addon --restart always -e MONGODB_URI=${{ secrets.MONGODB_URI }} -e DATABASE_URI=${{ secrets.DATABASE_URI }} -e PROXY_HOSTS=${{ secrets.PROXY_HOSTS }} -e PROXY_USERNAME=${{ secrets.PROXY_USERNAME }} -e PROXY_PASSWORD=${{ secrets.PROXY_PASSWORD }} -e METRICS_USER=${{ secrets.METRICS_USER }} -e METRICS_PASSWORD=${{ secrets.METRICS_PASSWORD }} torrentio-addon:latest + docker run -p 7050:7000 -d --name torrentio-addon --restart always -e MONGODB_URI=${{ secrets.MONGODB_URI }} -e DATABASE_URI=${{ secrets.DATABASE_URI }} -e PROXY_HOSTS=${{ secrets.PROXY_HOSTS }} -e PROXY_USERNAME=${{ secrets.PROXY_USERNAME }} -e PROXY_PASSWORD=${{ secrets.PROXY_PASSWORD }} -e METRICS_USER=${{ secrets.METRICS_USER }} -e METRICS_PASSWORD=${{ secrets.METRICS_PASSWORD }} torrentio-addon:latest docker image prune -f From 11e86c3cdcf7bffbf9833f6875f95bc51b34a904 Mon Sep 17 00:00:00 2001 From: TheBeastLT Date: Tue, 30 Jan 2024 09:19:57 +0200 Subject: [PATCH 09/12] increase RD timeout --- addon/addon.js | 2 +- addon/moch/realdebrid.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/addon/addon.js b/addon/addon.js index d2d7e0e..fafe095 100644 --- a/addon/addon.js +++ b/addon/addon.js @@ -71,7 +71,7 @@ async function resolveStreams(args) { } async function streamHandler(args) { - console.log(`Pending count: ${newLimiter.pendingCount}, active count: ${newLimiter.activeCount}`, ) + // console.log(`Pending count: ${newLimiter.pendingCount}, active count: ${newLimiter.activeCount}`, ) if (args.type === Type.MOVIE) { return movieRecordsHandler(args); } else if (args.type === Type.SERIES) { diff --git a/addon/moch/realdebrid.js b/addon/moch/realdebrid.js index 6aaa7a3..54ea857 100644 --- a/addon/moch/realdebrid.js +++ b/addon/moch/realdebrid.js @@ -395,5 +395,5 @@ function infringingFile(error) { } async function getDefaultOptions(ip) { - return { ip, timeout: 10000 }; + return { ip, timeout: 15000 }; } From ebf2934c6c07d8d2ffa06b90f2b01797f07da643 Mon Sep 17 00:00:00 2001 From: TheBeastLT Date: Wed, 31 Jan 2024 09:24:31 +0200 Subject: [PATCH 10/12] cache resolve url in remote mongo and reduce cache time --- addon/lib/cache.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/addon/lib/cache.js b/addon/lib/cache.js index 6cf193d..72a2c3a 100644 --- a/addon/lib/cache.js +++ b/addon/lib/cache.js @@ -9,6 +9,7 @@ const RESOLVED_URL_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|resolved`; const STREAM_TTL = process.env.STREAM_TTL || 24 * 60 * 60; // 24 hours const STREAM_EMPTY_TTL = process.env.STREAM_EMPTY_TTL || 60; // 1 minute +const RESOLVED_URL_TTL = 3 * 60 * 60; // 3 hours const AVAILABILITY_TTL = 8 * 60 * 60; // 8 hours const AVAILABILITY_EMPTY_TTL = 30 * 60; // 30 minutes const MESSAGE_VIDEO_URL_TTL = 60; // 1 minutes @@ -30,7 +31,7 @@ function initiateRemoteCache() { options: { collection: 'torrentio_addon_collection', socketTimeoutMS: 30000, - poolSize: 300, + poolSize: 200, useNewUrlParser: true, useUnifiedTopology: false, ttl: STREAM_EMPTY_TTL @@ -68,8 +69,8 @@ export function cacheWrapStream(id, method) { } export function cacheWrapResolvedUrl(id, method) { - return cacheWrap(memoryCache, `${RESOLVED_URL_KEY_PREFIX}:${id}`, method, { - ttl: (url) => isStaticUrl(url) ? MESSAGE_VIDEO_URL_TTL : STREAM_TTL + return cacheWrap(remoteCache, `${RESOLVED_URL_KEY_PREFIX}:${id}`, method, { + ttl: (url) => isStaticUrl(url) ? MESSAGE_VIDEO_URL_TTL : RESOLVED_URL_TTL }); } From 79b83e52e46d21ccd32125537aa9b872ca3dfd9d Mon Sep 17 00:00:00 2001 From: TheBeastLT Date: Wed, 31 Jan 2024 09:25:33 +0200 Subject: [PATCH 11/12] reduce psql connection pool a bit --- addon/addon.js | 2 +- addon/lib/repository.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/addon/addon.js b/addon/addon.js index fafe095..0492989 100644 --- a/addon/addon.js +++ b/addon/addon.js @@ -19,7 +19,7 @@ const STALE_ERROR_AGE = 7 * 24 * 60 * 60; // 7 days const builder = new addonBuilder(dummyManifest()); const requestQueue = createNamedQueue(Infinity); -const newLimiter = pLimit(40) +const newLimiter = pLimit(30) builder.defineStreamHandler((args) => { if (!args.id.match(/tt\d+/i) && !args.id.match(/kitsu:\d+/i)) { diff --git a/addon/lib/repository.js b/addon/lib/repository.js index 9447514..71f5da2 100644 --- a/addon/lib/repository.js +++ b/addon/lib/repository.js @@ -3,7 +3,7 @@ const Op = Sequelize.Op; const DATABASE_URI = process.env.DATABASE_URI; -const database = new Sequelize(DATABASE_URI, { logging: false, pool: { max: 50, min: 5, idle: 60 * 60 * 1000 } }); +const database = new Sequelize(DATABASE_URI, { logging: true, pool: { max: 30, min: 5, idle: 20 * 60 * 1000 } }); const Torrent = database.define('torrent', { From fb267b8057dcc36612755f9a22855fb26934ec55 Mon Sep 17 00:00:00 2001 From: TheBeastLT Date: Wed, 31 Jan 2024 09:30:59 +0200 Subject: [PATCH 12/12] remove psql logging --- addon/lib/repository.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addon/lib/repository.js b/addon/lib/repository.js index 71f5da2..e1a6121 100644 --- a/addon/lib/repository.js +++ b/addon/lib/repository.js @@ -3,7 +3,7 @@ const Op = Sequelize.Op; const DATABASE_URI = process.env.DATABASE_URI; -const database = new Sequelize(DATABASE_URI, { logging: true, pool: { max: 30, min: 5, idle: 20 * 60 * 1000 } }); +const database = new Sequelize(DATABASE_URI, { logging: false, pool: { max: 30, min: 5, idle: 20 * 60 * 1000 } }); const Torrent = database.define('torrent', {