From 8a4eb4cdfecc68f0d85f4dc86f50e1cd4e6401ee Mon Sep 17 00:00:00 2001 From: TheBeastLT Date: Fri, 19 Jan 2024 10:30:13 +0200 Subject: [PATCH 1/7] add limiter options --- addon/addon.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/addon/addon.js b/addon/addon.js index f7fb696..2ac3a9b 100644 --- a/addon/addon.js +++ b/addon/addon.js @@ -22,13 +22,14 @@ const limiter = new Bottleneck({ highWater: process.env.LIMIT_QUEUE_SIZE || 220, strategy: Bottleneck.strategy.OVERFLOW }); +const limiterOptions = { expiration: 2 * 60 * 1000 } builder.defineStreamHandler((args) => { if (!args.id.match(/tt\d+/i) && !args.id.match(/kitsu:\d+/i)) { return Promise.resolve({ streams: [] }); } - return cacheWrapStream(args.id, () => limiter.schedule(() => streamHandler(args) + return cacheWrapStream(args.id, () => limiter.schedule(limiterOptions, () => streamHandler(args) .then(records => records .sort((a, b) => b.torrent.seeders - a.torrent.seeders || b.torrent.uploadDate - a.torrent.uploadDate) .map(record => toStreamInfo(record))))) From 4ee220a1ad3b2a80fd16d6ad820c3b58f8138e75 Mon Sep 17 00:00:00 2001 From: TheBeastLT Date: Sun, 21 Jan 2024 23:14:44 +0200 Subject: [PATCH 2/7] move limiter before mongo call --- addon/addon.js | 6 +++--- addon/lib/cache.js | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/addon/addon.js b/addon/addon.js index 2ac3a9b..ccbaa76 100644 --- a/addon/addon.js +++ b/addon/addon.js @@ -18,8 +18,8 @@ const STALE_ERROR_AGE = 7 * 24 * 60 * 60; // 7 days const builder = new addonBuilder(dummyManifest()); const limiter = new Bottleneck({ - maxConcurrent: process.env.LIMIT_MAX_CONCURRENT || 200, - highWater: process.env.LIMIT_QUEUE_SIZE || 220, + maxConcurrent: process.env.LIMIT_MAX_CONCURRENT || 100, + highWater: process.env.LIMIT_QUEUE_SIZE || 120, strategy: Bottleneck.strategy.OVERFLOW }); const limiterOptions = { expiration: 2 * 60 * 1000 } @@ -29,7 +29,7 @@ builder.defineStreamHandler((args) => { return Promise.resolve({ streams: [] }); } - return cacheWrapStream(args.id, () => limiter.schedule(limiterOptions, () => streamHandler(args) + return limiter.schedule(limiterOptions, () => cacheWrapStream(args.id, () => streamHandler(args) .then(records => records .sort((a, b) => b.torrent.seeders - a.torrent.seeders || b.torrent.uploadDate - a.torrent.uploadDate) .map(record => toStreamInfo(record))))) diff --git a/addon/lib/cache.js b/addon/lib/cache.js index 710f50e..4c2ec71 100644 --- a/addon/lib/cache.js +++ b/addon/lib/cache.js @@ -29,7 +29,7 @@ function initiateRemoteCache() { uri: MONGO_URI, options: { collection: 'torrentio_addon_collection', - socketTimeoutMS: 120000, + socketTimeoutMS: 30000, useNewUrlParser: true, useUnifiedTopology: false, ttl: STREAM_EMPTY_TTL From 777241fdfa51e1ac04d309e11623cddab579482b Mon Sep 17 00:00:00 2001 From: TheBeastLT Date: Sun, 21 Jan 2024 23:25:07 +0200 Subject: [PATCH 3/7] add mongo pool size --- addon/lib/cache.js | 1 + 1 file changed, 1 insertion(+) diff --git a/addon/lib/cache.js b/addon/lib/cache.js index 4c2ec71..48a3d35 100644 --- a/addon/lib/cache.js +++ b/addon/lib/cache.js @@ -30,6 +30,7 @@ function initiateRemoteCache() { options: { collection: 'torrentio_addon_collection', socketTimeoutMS: 30000, + poolSize: 150, useNewUrlParser: true, useUnifiedTopology: false, ttl: STREAM_EMPTY_TTL From 1572ae4af514e96d22addae6cdd6782bee9fa4b7 Mon Sep 17 00:00:00 2001 From: TheBeastLT Date: Sun, 21 Jan 2024 23:31:58 +0200 Subject: [PATCH 4/7] add psql pool config --- addon/lib/repository.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addon/lib/repository.js b/addon/lib/repository.js index e6c3029..003b2b5 100644 --- a/addon/lib/repository.js +++ b/addon/lib/repository.js @@ -3,7 +3,7 @@ const Op = Sequelize.Op; const DATABASE_URI = process.env.DATABASE_URI; -const database = new Sequelize(DATABASE_URI, { logging: false }); +const database = new Sequelize(DATABASE_URI, { logging: false, pool: { max: 100, evict: 120000} }); const Torrent = database.define('torrent', { From f22df6c5cbea5becf9602b6d8901b27235f846b8 Mon Sep 17 00:00:00 2001 From: TheBeastLT Date: Sun, 21 Jan 2024 23:36:07 +0200 Subject: [PATCH 5/7] try to test bypassing mongo --- addon/addon.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/addon/addon.js b/addon/addon.js index ccbaa76..9e05053 100644 --- a/addon/addon.js +++ b/addon/addon.js @@ -29,10 +29,10 @@ builder.defineStreamHandler((args) => { return Promise.resolve({ streams: [] }); } - return limiter.schedule(limiterOptions, () => cacheWrapStream(args.id, () => streamHandler(args) + return limiter.schedule(limiterOptions, () => streamHandler(args) .then(records => records .sort((a, b) => b.torrent.seeders - a.torrent.seeders || b.torrent.uploadDate - a.torrent.uploadDate) - .map(record => toStreamInfo(record))))) + .map(record => toStreamInfo(record)))) .then(streams => applyFilters(streams, args.extra)) .then(streams => applySorting(streams, args.extra, args.type)) .then(streams => applyStaticInfo(streams)) From a51ddc53c5f80a6695f8d04d552c09ac01e9ab12 Mon Sep 17 00:00:00 2001 From: TheBeastLT Date: Sun, 21 Jan 2024 23:46:15 +0200 Subject: [PATCH 6/7] revert bypassing mongo --- addon/addon.js | 4 ++-- addon/lib/repository.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/addon/addon.js b/addon/addon.js index 9e05053..f7566e0 100644 --- a/addon/addon.js +++ b/addon/addon.js @@ -29,10 +29,10 @@ builder.defineStreamHandler((args) => { return Promise.resolve({ streams: [] }); } - return limiter.schedule(limiterOptions, () => streamHandler(args) + return cacheWrapStream(args.id, () => limiter.schedule(limiterOptions, () => streamHandler(args) .then(records => records .sort((a, b) => b.torrent.seeders - a.torrent.seeders || b.torrent.uploadDate - a.torrent.uploadDate) - .map(record => toStreamInfo(record)))) + .map(record => toStreamInfo(record))))) .then(streams => applyFilters(streams, args.extra)) .then(streams => applySorting(streams, args.extra, args.type)) .then(streams => applyStaticInfo(streams)) diff --git a/addon/lib/repository.js b/addon/lib/repository.js index 003b2b5..594fef9 100644 --- a/addon/lib/repository.js +++ b/addon/lib/repository.js @@ -3,7 +3,7 @@ const Op = Sequelize.Op; const DATABASE_URI = process.env.DATABASE_URI; -const database = new Sequelize(DATABASE_URI, { logging: false, pool: { max: 100, evict: 120000} }); +const database = new Sequelize(DATABASE_URI, { logging: false, pool: { max: 50, evict: 120000 } }); const Torrent = database.define('torrent', { From 0fdc968a86993e3264ee6157d76ac3fef6ab9a0e Mon Sep 17 00:00:00 2001 From: TheBeastLT Date: Mon, 22 Jan 2024 00:13:27 +0200 Subject: [PATCH 7/7] remove psql pool evict --- addon/lib/repository.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addon/lib/repository.js b/addon/lib/repository.js index 594fef9..cdee1ac 100644 --- a/addon/lib/repository.js +++ b/addon/lib/repository.js @@ -3,7 +3,7 @@ const Op = Sequelize.Op; const DATABASE_URI = process.env.DATABASE_URI; -const database = new Sequelize(DATABASE_URI, { logging: false, pool: { max: 50, evict: 120000 } }); +const database = new Sequelize(DATABASE_URI, { logging: false, pool: { max: 50 } }); const Torrent = database.define('torrent', {