From b39e54e2e03be6e8bf732adebc84773d367374f0 Mon Sep 17 00:00:00 2001 From: TheBeastLT Date: Sat, 19 Dec 2020 14:52:12 +0100 Subject: [PATCH] [scraper] encode uri component for kitsu id query --- addon/moch/alldebrid.js | 2 +- addon/moch/realdebrid.js | 2 +- scraper/lib/metadata.js | 5 +++-- scraper/scrapers/nyaasi/nyaa_si_scraper.js | 2 +- scraper/scrapers/thepiratebay/thepiratebay_scraper.js | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/addon/moch/alldebrid.js b/addon/moch/alldebrid.js index a32d9b1..5cd8860 100644 --- a/addon/moch/alldebrid.js +++ b/addon/moch/alldebrid.js @@ -40,7 +40,7 @@ async function getCatalog(apiKey, offset = 0) { return AD.magnet.status() .then(response => response.data.magnets) .then(torrents => (torrents || []) - .filter(torrent => statusReady(torrent.statusCode)) + .filter(torrent => torrent && statusReady(torrent.statusCode)) .map(torrent => ({ id: `${KEY}:${torrent.id}`, type: Type.OTHER, diff --git a/addon/moch/realdebrid.js b/addon/moch/realdebrid.js index 2a27116..9170943 100644 --- a/addon/moch/realdebrid.js +++ b/addon/moch/realdebrid.js @@ -78,7 +78,7 @@ async function getCatalog(apiKey, offset = 0) { return _getAllTorrents(RD) .then(torrents => Array.isArray(torrents) ? torrents : []) .then(torrents => torrents - .filter(torrent => statusReady(torrent.status)) + .filter(torrent => torrent && statusReady(torrent.status)) .map(torrent => ({ id: `${KEY}:${torrent.id}`, type: Type.OTHER, diff --git a/scraper/lib/metadata.js b/scraper/lib/metadata.js index ff063d6..83e78e5 100644 --- a/scraper/lib/metadata.js +++ b/scraper/lib/metadata.js @@ -124,9 +124,10 @@ async function getKitsuId(info) { const title = escapeTitle(info.title.replace(/\s\|\s.*/, '')); const year = info.year ? ` ${info.year}` : ''; const season = info.season > 1 ? ` S${info.season}` : ''; - const query = `${title}${year}${season}`; + const key = `${title}${year}${season}`; + const query = encodeURIComponent(key); - return cacheWrapKitsuId(query, + return cacheWrapKitsuId(key, () => needle('get', `${KITSU_URL}/catalog/series/kitsu-anime-list/search=${query}.json`, { open_timeout: 60000 }) .then((response) => { const body = response.body; diff --git a/scraper/scrapers/nyaasi/nyaa_si_scraper.js b/scraper/scrapers/nyaasi/nyaa_si_scraper.js index d16c90f..f3aa350 100644 --- a/scraper/scrapers/nyaasi/nyaa_si_scraper.js +++ b/scraper/scrapers/nyaasi/nyaa_si_scraper.js @@ -45,7 +45,7 @@ async function scrapeLatestTorrents() { async function scrapeLatestTorrentsForCategory(category, page = 1) { console.log(`Scrapping ${NAME} ${category} category page ${page}`); - return nyaasi.browse(({ page })) + return nyaasi.browse({ page, sort: 'id' }) .catch(error => { console.warn(`Failed ${NAME} scrapping for [${page}] ${category} due: `, error); return Promise.resolve([]); diff --git a/scraper/scrapers/thepiratebay/thepiratebay_scraper.js b/scraper/scrapers/thepiratebay/thepiratebay_scraper.js index 7a10c8d..6b7d15d 100644 --- a/scraper/scrapers/thepiratebay/thepiratebay_scraper.js +++ b/scraper/scrapers/thepiratebay/thepiratebay_scraper.js @@ -48,7 +48,7 @@ async function scrapeLatestTorrents() { async function scrapeLatestTorrentsForCategory(category, page = 1) { console.log(`Scrapping ${NAME} ${category} category page ${page}`); - return thepiratebay.browse(({ category, page })) + return thepiratebay.browse({ category, page }) .catch(error => { console.warn(`Failed ${NAME} scrapping for [${page}] ${category} due: `, error); return Promise.resolve([]);