tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

adds kitsu mapping for horrible subs

Browse Source
This commit is contained in:
TheBeastLT
2019-12-30 18:35:02 +01:00
parent a73cdb6b54
commit 7aa0572fb8
3 changed files with 3612 additions and 8832 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12422
horrible_subs_mapping.json
View File

File diff suppressed because it is too large Load Diff

5
lib/metadata.js
View File

@@ -88,8 +88,9 @@ async function getImdbId(info) {
}
async function getKitsuId(title) {
return cacheWrapImdbId(title,
() => needle('get', `${KITSU_URL}/catalog/series/kitsu-anime-list/search=${title}.json`, { open_timeout: 60000 })
const query = title.replace(/[;]+/g, ' ').replace(/[,%']+/g, '');
return cacheWrapImdbId(query,
() => needle('get', `${KITSU_URL}/catalog/series/kitsu-anime-list/search=${query}.json`, { open_timeout: 60000 })
.then((response) => {
const body = response.body;
if (body && body.metas && body.metas.length) {

17
scrapers/horiblesubs_scraper.js
View File

@@ -39,11 +39,12 @@ async function _scrapeAllShows() {
async function initMapping() {
console.log(`${NAME}: initiating kitsu mapping...`);
const currentMapping = require('../horrible_subs_mapping');
const mappings = Object.values(currentMapping);
const shows = await horriblesubs.allShows()
.then((shows) => Promise.all(shows.map((show) => limiter.schedule(() => enrichShow(show)))))
.then((shows) => shows.reduce((map, show) => (map[show.showId] = show, map), {}));
const kitsuIds = Object.values(shows).map((show) => show.kitsu_id);
console.log(JSON.stringify(kitsuIds));
.then((shows) => shows.filter((show) => !mappings.find((mapping) => mapping.title === show.title)))
.then((shows) => Promise.all(shows.map((show) => limiter.schedule(() => enrichShow(show)))))
.then((shows) => shows.reduce((map, show) => (map[show.title] = show, map), currentMapping));
fs.writeFile("./horrible_subs_mapping.json", JSON.stringify(shows), 'utf8', function (err) {
if (err) {
@@ -57,8 +58,7 @@ async function enrichShow(show) {
console.log(`${NAME}: getting show info for ${show.title}...`);
const showId = await horriblesubs._getShowId(show.url)
.catch((error) => show.title);
const slug = show.url.replace(/^.*\//, '');
const metadata = await getKitsuId(slug)
const metadata = await getKitsuId(show.title)
.then((kitsuId) => getKitsuMetadata(kitsuId))
.catch((error) => {
console.log(`Failed getting kitsu meta: ${error.message}`);
@@ -75,11 +75,6 @@ async function enrichShow(show) {
}
}
const hardcodedShows = {
'199': 'tt2098220',
'347': 'tt0388629'
};
async function _parseShowData(showData) {
console.log(`${NAME}: scrapping ${showData.title} data...`);
const imdbId = hardcodedShows[showData.showId] || await getImdbId({