tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[scraper] disallow rutor russian series without imdb id

Browse Source
This commit is contained in:
TheBeastLT
2021-02-15 17:43:03 +01:00
parent fae2e8f360
commit 817d64f2eb
1 changed files with 11 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
scraper/scrapers/rutor/rutor_scraper.js
View File

@@ -1,5 +1,6 @@
const moment = require('moment'); const moment = require('moment');
const Bottleneck = require('bottleneck'); const Bottleneck = require('bottleneck');
const { parse } = require('parse-torrent-title');
const rutor = require('./rutor_api'); const rutor = require('./rutor_api');
const { Type } = require('../../lib/types'); const { Type } = require('../../lib/types');
const repository = require('../../lib/repository'); const repository = require('../../lib/repository');
@@ -17,8 +18,7 @@ const TYPE_MAPPING = {
'tv': Type.SERIES, 'tv': Type.SERIES,
'multiki': Type.MOVIE, 'multiki': Type.MOVIE,
'anime': Type.ANIME 'anime': Type.ANIME
} };
const ALLOWED_WITHOUT_IMDB = ['kino', 'seriali', 'anime'];
const api_limiter = new Bottleneck({ maxConcurrent: 1, minTime: 5000 }); const api_limiter = new Bottleneck({ maxConcurrent: 1, minTime: 5000 });
const api_entry_limiter = new Bottleneck({ maxConcurrent: 1, minTime: 2500 }); const api_entry_limiter = new Bottleneck({ maxConcurrent: 1, minTime: 2500 });
@@ -90,7 +90,7 @@ async function processTorrentRecord(record) {
if (!foundTorrent || !TYPE_MAPPING[foundTorrent.category]) { if (!foundTorrent || !TYPE_MAPPING[foundTorrent.category]) {
return Promise.resolve(`${NAME}: Invalid torrent record: ${record.torrentId}`); return Promise.resolve(`${NAME}: Invalid torrent record: ${record.torrentId}`);
} }
if (!foundTorrent.imdbId && !ALLOWED_WITHOUT_IMDB.includes(foundTorrent.category)) { if (!foundTorrent.imdbId && disallowWithoutImdbId(foundTorrent)) {
return Promise.resolve(`${NAME}: No imdbId defined: ${record.torrentId}`); return Promise.resolve(`${NAME}: No imdbId defined: ${record.torrentId}`);
} }
@@ -122,4 +122,12 @@ function getMaxPage(category) {
} }
} }
function disallowWithoutImdbId(torrent) {
if (['kino', 'anime'].includes(torrent.category)) {
return false; // allow to search foreign movie and anime ids via search
}
// allow to search id for non russian series titles via search
return !(torrent.category === 'seriali' && !parse(torrent.title).title.match(/[\u0400-\u04ff]/i));
}
module.exports = { scrape, updateSeeders, NAME }; module.exports = { scrape, updateSeeders, NAME };