tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] adds anime specific trackers

Browse Source
This commit is contained in:
TheBeastLT
2020-12-02 19:30:23 +01:00
parent 5b82ac33be
commit 519799b634
2 changed files with 12 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
scraper/lib/repository.js
View File

@@ -186,7 +186,7 @@ function getTorrentsWithoutSize() {
function getUpdateSeedersTorrents() { function getUpdateSeedersTorrents() {
const until = moment().subtract(7, 'days').format('YYYY-MM-DD'); const until = moment().subtract(7, 'days').format('YYYY-MM-DD');
return Torrent.findAll({ return Torrent.findAll({
where: literal(`torrent."updatedAt" < \'${until}\' and torrent."provider" not in (\'NyaaSi\')`), where: literal(`torrent."updatedAt" < \'${until}\'`),
limit: 100, limit: 100,
order: [ order: [
['seeders', 'DESC'], ['seeders', 'DESC'],

14
scraper/lib/torrent.js
View File

@@ -4,12 +4,19 @@ const parseTorrent = require('parse-torrent');
const BTClient = require('bittorrent-tracker') const BTClient = require('bittorrent-tracker')
const async = require('async'); const async = require('async');
const decode = require('magnet-uri'); const decode = require('magnet-uri');
const { Type } = require('./types');
const { isVideo, isSubtitle } = require('./extension'); const { isVideo, isSubtitle } = require('./extension');
const { cacheTrackers } = require('./cache'); const { cacheTrackers } = require('./cache');
const TRACKERS_URL = 'https://ngosang.github.io/trackerslist/trackers_best.txt'; const TRACKERS_URL = 'https://ngosang.github.io/trackerslist/trackers_best.txt';
const MAX_PEER_CONNECTIONS = process.env.MAX_PEER_CONNECTIONS || 20; const MAX_PEER_CONNECTIONS = process.env.MAX_PEER_CONNECTIONS || 20;
const SEEDS_CHECK_TIMEOUT = process.env.SEEDS_CHECK_TIMEOUT || 10 * 1000; // 10 secs const SEEDS_CHECK_TIMEOUT = process.env.SEEDS_CHECK_TIMEOUT || 10 * 1000; // 10 secs
const ANIME_TRACKERS = [
"http://nyaa.tracker.wf:7777/announce",
"http://anidex.moe:6969/announce",
"http://tracker.anirena.com:80/announce",
"udp://tracker.uw0.xyz:6969/announce"
];
async function updateCurrentSeeders(torrent) { async function updateCurrentSeeders(torrent) {
return new Promise(async (resolve) => { return new Promise(async (resolve) => {
@@ -20,7 +27,7 @@ async function updateCurrentSeeders(torrent) {
const seeders = {}; const seeders = {};
const magnetTrackers = torrent.magnetLink && decode(torrent.magnetLink).tr; const magnetTrackers = torrent.magnetLink && decode(torrent.magnetLink).tr;
const torrentTrackers = torrent.trackers && torrent.trackers.split(','); const torrentTrackers = torrent.trackers && torrent.trackers.split(',');
const trackers = magnetTrackers || torrentTrackers || await getDefaultTrackers(); const trackers = magnetTrackers || torrentTrackers || await getDefaultTrackers(torrent);
const callback = () => resolve(Math.max(...Object.values(seeders).map(values => values[0]).concat(0))); const callback = () => resolve(Math.max(...Object.values(seeders).map(values => values[0]).concat(0)));
setTimeout(callback, SEEDS_CHECK_TIMEOUT); setTimeout(callback, SEEDS_CHECK_TIMEOUT);
@@ -144,10 +151,11 @@ function filterSubtitles(files) {
return files.filter(file => isSubtitle(file.path)); return files.filter(file => isSubtitle(file.path));
} }
async function getDefaultTrackers() { async function getDefaultTrackers(torrent) {
return cacheTrackers(() => needle('get', TRACKERS_URL, { open_timeout: SEEDS_CHECK_TIMEOUT }) return cacheTrackers(() => needle('get', TRACKERS_URL, { open_timeout: SEEDS_CHECK_TIMEOUT })
.then(response => response.body && response.body.trim()) .then(response => response.body && response.body.trim())
.then(body => body && body.split('\n\n') || [])); .then(body => body && body.split('\n\n') || []))
.then(trackers => torrent.type === Type.ANIME ? trackers.concat(ANIME_TRACKERS) : trackers);
} }
module.exports = { updateCurrentSeeders, updateTorrentSize, sizeAndFiles, torrentFiles } module.exports = { updateCurrentSeeders, updateTorrentSize, sizeAndFiles, torrentFiles }