tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] reduce seeders update delay and use mostly udp trackers

Browse Source
This commit is contained in:
TheBeastLT
2021-02-09 12:59:47 +01:00
parent 4c8f2b6a25
commit 849e308e75
2 changed files with 6 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
scraper/lib/torrent.js
View File

@@ -8,9 +8,9 @@ const { Type } = require('./types');
const { isVideo, isSubtitle } = require('./extension'); const { isVideo, isSubtitle } = require('./extension');
const { cacheTrackers } = require('./cache'); const { cacheTrackers } = require('./cache');
const TRACKERS_URL = 'https://ngosang.github.io/trackerslist/trackers_all.txt'; const TRACKERS_URL = 'https://ngosang.github.io/trackerslist/trackers_all_udp.txt';
const MAX_PEER_CONNECTIONS = process.env.MAX_PEER_CONNECTIONS || 20; const MAX_PEER_CONNECTIONS = process.env.MAX_PEER_CONNECTIONS || 20;
const SEEDS_CHECK_TIMEOUT = 30 * 1000; // 30 secs const SEEDS_CHECK_TIMEOUT = 20 * 1000; // 30 secs
const ADDITIONAL_TRACKERS = [ const ADDITIONAL_TRACKERS = [
'http://tracker.trackerfix.com:80/announce', 'http://tracker.trackerfix.com:80/announce',
'udp://9.rarbg.me:2780', 'udp://9.rarbg.me:2780',
@@ -18,8 +18,8 @@ const ADDITIONAL_TRACKERS = [
]; ];
const ANIME_TRACKERS = [ const ANIME_TRACKERS = [
"http://nyaa.tracker.wf:7777/announce", "http://nyaa.tracker.wf:7777/announce",
"http://anidex.moe:6969/announce", "udp://anidex.moe:6969/announce",
"http://tracker.anirena.com:80/announce", "udp://tracker-udp.anirena.com:80/announce",
"udp://tracker.uw0.xyz:6969/announce" "udp://tracker.uw0.xyz:6969/announce"
]; ];

4
scraper/scheduler/seeders.js
View File

@@ -4,13 +4,13 @@ const { delay } = require('../lib/promises')
const { updateCurrentSeeders } = require('../lib/torrent') const { updateCurrentSeeders } = require('../lib/torrent')
const { updateTorrentSeeders } = require('../lib/torrentEntries') const { updateTorrentSeeders } = require('../lib/torrentEntries')
const DELAY_MS = 15 * 1000; // 15 seconds const DELAY_MS = 0; // 0 seconds
const updateLimiter = new Bottleneck({ maxConcurrent: 5 }); const updateLimiter = new Bottleneck({ maxConcurrent: 5 });
const statistics = {}; const statistics = {};
function scheduleUpdateSeeders() { function scheduleUpdateSeeders() {
console.log('Starting seeders update...') console.log('Starting seeders update...')
return repository.getUpdateSeedersTorrents(50) return repository.getUpdateSeedersTorrents(100)
.then(torrents => updateCurrentSeeders(torrents)) .then(torrents => updateCurrentSeeders(torrents))
.then(updatedTorrents => Promise.all( .then(updatedTorrents => Promise.all(
updatedTorrents.map(updated => updateLimiter.schedule(() => updateTorrentSeeders(updated))))) updatedTorrents.map(updated => updateLimiter.schedule(() => updateTorrentSeeders(updated)))))