diff --git a/src/node/consumer/src/lib/interfaces/cinemeta_metadata.ts b/src/node/consumer/src/lib/interfaces/cinemeta_metadata.ts new file mode 100644 index 0000000..48cdc8a --- /dev/null +++ b/src/node/consumer/src/lib/interfaces/cinemeta_metadata.ts @@ -0,0 +1,80 @@ +import {CommonVideoMetadata} from "./common_video_metadata"; + +export interface CinemetaJsonResponse { + meta?: CinemetaMetaData; + trailerStreams?: CinemetaTrailerStream[]; + links?: CinemetaLink[]; + behaviorHints?: CinemetaBehaviorHints; +} +export interface CinemetaMetaData { + awards?: string; + cast?: string[]; + country?: string; + description?: string; + director?: null; + dvdRelease?: null; + genre?: string[]; + imdbRating?: string; + imdb_id?: string; + name?: string; + popularity?: number; + poster?: string; + released?: string; + runtime?: string; + status?: string; + tvdb_id?: number; + type?: string; + writer?: string[]; + year?: string; + background?: string; + logo?: string; + popularities?: CinemetaPopularities; + moviedb_id?: number; + slug?: string; + trailers?: CinemetaTrailer[]; + id?: string; + genres?: string[]; + releaseInfo?: string; + videos?: CinemetaVideo[]; +} +export interface CinemetaPopularities { + PXS_TEST?: number; + PXS?: number; + SCM?: number; + EXMD?: number; + ALLIANCE?: number; + EJD?: number; + moviedb?: number; + trakt?: number; + stremio?: number; + stremio_lib?: number; +} +export interface CinemetaTrailer { + source?: string; + type?: string; +} +export interface CinemetaVideo extends CommonVideoMetadata { + name?: string; + number?: number; + firstAired?: string; + tvdb_id?: number; + rating?: string; + overview?: string; + thumbnail?: string; + id?: string; + released?: string; + description?: string; +} +export interface CinemetaTrailerStream { + title?: string; + ytId?: string; +} +export interface CinemetaLink { + name?: string; + category?: string; + url?: string; +} +export interface CinemetaBehaviorHints { + defaultVideoId?: null; + hasScheduledVideos?: boolean; +} \ No newline at end of file diff --git a/src/node/consumer/src/lib/interfaces/common_video_metadata.ts b/src/node/consumer/src/lib/interfaces/common_video_metadata.ts new file mode 100644 index 0000000..4383417 --- /dev/null +++ b/src/node/consumer/src/lib/interfaces/common_video_metadata.ts @@ -0,0 +1,4 @@ +export interface CommonVideoMetadata { + season?: number; + episode?: number; +} \ No newline at end of file diff --git a/src/node/consumer/src/lib/interfaces/kitsu_catalog_metadata.ts b/src/node/consumer/src/lib/interfaces/kitsu_catalog_metadata.ts new file mode 100644 index 0000000..42ac1bd --- /dev/null +++ b/src/node/consumer/src/lib/interfaces/kitsu_catalog_metadata.ts @@ -0,0 +1,23 @@ +import {KitsuLink, KitsuTrailer} from "./kitsu_metadata"; + +export interface KitsuCatalogJsonResponse { + metas: KitsuCatalogMetaData[]; +} + +export interface KitsuCatalogMetaData { + id: string; + type: string; + animeType: string; + name: string; + aliases: string[]; + description: string; + releaseInfo: string; + runtime: string; + imdbRating: string; + genres: string[]; + logo?: string; + poster: string; + background: string; + trailers: KitsuTrailer[]; + links: KitsuLink[]; +} \ No newline at end of file diff --git a/src/node/consumer/src/lib/interfaces/kitsu_metadata.ts b/src/node/consumer/src/lib/interfaces/kitsu_metadata.ts new file mode 100644 index 0000000..fc63293 --- /dev/null +++ b/src/node/consumer/src/lib/interfaces/kitsu_metadata.ts @@ -0,0 +1,49 @@ +import {CommonVideoMetadata} from "./common_video_metadata"; + +export interface KitsuJsonResponse { + cacheMaxAge?: number; + meta?: KitsuMeta; +} +export interface KitsuMeta { + aliases?: string[]; + animeType?: string; + background?: string; + description?: string; + country?: string; + genres?: string[]; + id?: string; + imdbRating?: string; + imdb_id?: string; + kitsu_id?: string; + links?: KitsuLink[]; + logo?: string; + name?: string; + poster?: string; + releaseInfo?: string; + runtime?: string; + slug?: string; + status?: string; + trailers?: KitsuTrailer[]; + type?: string; + userCount?: number; + videos?: KitsuVideo[]; + year?: string; +} +export interface KitsuVideo extends CommonVideoMetadata { + id?: string; + imdbEpisode?: number; + imdbSeason?: number; + imdb_id?: string; + released?: string; + thumbnail?: string; + title?: string; +} +export interface KitsuTrailer { + source?: string; + type?: string; +} +export interface KitsuLink { + name?: string; + category?: string; + url?: string; +} \ No newline at end of file diff --git a/src/node/consumer/src/lib/interfaces/metadata_query.ts b/src/node/consumer/src/lib/interfaces/metadata_query.ts new file mode 100644 index 0000000..ecb85ea --- /dev/null +++ b/src/node/consumer/src/lib/interfaces/metadata_query.ts @@ -0,0 +1,8 @@ +export interface MetaDataQuery { + title?: string + type?: string + year?: number | string + date?: string + season?: number + episode?: number +} \ No newline at end of file diff --git a/src/node/consumer/src/lib/interfaces/metadata_response.ts b/src/node/consumer/src/lib/interfaces/metadata_response.ts new file mode 100644 index 0000000..e1509c5 --- /dev/null +++ b/src/node/consumer/src/lib/interfaces/metadata_response.ts @@ -0,0 +1,13 @@ +export interface MetadataResponse { + kitsuId?: number; + imdbId?: number; + type?: string; + title?: string; + year?: number; + country?: string; + genres?: string[]; + status?: string; + videos?: any[]; + episodeCount?: number[]; + totalCount?: number; +} \ No newline at end of file diff --git a/src/node/consumer/src/lib/metadata.js b/src/node/consumer/src/lib/metadata.js deleted file mode 100644 index 09da123..0000000 --- a/src/node/consumer/src/lib/metadata.js +++ /dev/null @@ -1,165 +0,0 @@ -import axios from 'axios'; -import { search } from 'google-sr'; -import nameToImdb from 'name-to-imdb'; -import { cacheWrapImdbId, cacheWrapKitsuId, cacheWrapMetadata } from './cache.js'; -import { TorrentType } from './enums/torrent_types'; - -const CINEMETA_URL = 'https://v3-cinemeta.strem.io'; -const KITSU_URL = 'https://anime-kitsu.strem.fun'; -const TIMEOUT = 20000; - -export function getMetadata(id, type = TorrentType.SERIES) { - if (!id) { - return Promise.reject("no valid id provided"); - } - - const key = Number.isInteger(id) || id.match(/^\d+$/) ? `kitsu:${id}` : id; - const metaType = type === TorrentType.MOVIE ? TorrentType.MOVIE : TorrentType.SERIES; - return cacheWrapMetadata(key, () => _requestMetadata(`${KITSU_URL}/meta/${metaType}/${key}.json`) - .catch(() => _requestMetadata(`${CINEMETA_URL}/meta/${metaType}/${key}.json`)) - .catch(() => { - // try different type in case there was a mismatch - const otherType = metaType === TorrentType.MOVIE ? TorrentType.SERIES : TorrentType.MOVIE; - return _requestMetadata(`${CINEMETA_URL}/meta/${otherType}/${key}.json`) - }) - .catch((error) => { - throw new Error(`failed metadata query ${key} due: ${error.message}`); - })); -} - -function _requestMetadata(url) { - return axios.get(url, { timeout: TIMEOUT }) - .then((response) => { - const body = response.data; - if (body && body.meta && (body.meta.imdb_id || body.meta.kitsu_id)) { - return { - kitsuId: body.meta.kitsu_id, - imdbId: body.meta.imdb_id, - type: body.meta.type, - title: body.meta.name, - year: body.meta.year, - country: body.meta.country, - genres: body.meta.genres, - status: body.meta.status, - videos: (body.meta.videos || []) - .map((video) => Number.isInteger(video.imdbSeason) - ? { - name: video.name || video.title, - season: video.season, - episode: video.episode, - imdbSeason: video.imdbSeason, - imdbEpisode: video.imdbEpisode - } - : { - name: video.name || video.title, - season: video.season, - episode: video.episode, - kitsuId: video.kitsu_id, - kitsuEpisode: video.kitsuEpisode, - released: video.released - } - ), - episodeCount: Object.values((body.meta.videos || []) - .filter((entry) => entry.season !== 0 && entry.episode !== 0) - .sort((a, b) => a.season - b.season) - .reduce((map, next) => { - map[next.season] = map[next.season] + 1 || 1; - return map; - }, {})), - totalCount: body.meta.videos && body.meta.videos - .filter((entry) => entry.season !== 0 && entry.episode !== 0).length - }; - } else { - throw new Error('No search results'); - } - }); -} - -export function escapeTitle(title) { - return title.toLowerCase() - .normalize('NFKD') // normalize non-ASCII characters - .replace(/[\u0300-\u036F]/g, '') - .replace(/&/g, 'and') - .replace(/[;, ~./]+/g, ' ') // replace dots, commas or underscores with spaces - .replace(/[^\w \-()×+#@!'\u0400-\u04ff]+/g, '') // remove all non-alphanumeric chars - .replace(/^\d{1,2}[.#\s]+(?=(?:\d+[.\s]*)?[\u0400-\u04ff])/i, '') // remove russian movie numbering - .replace(/\s{2,}/, ' ') // replace multiple spaces - .trim(); -} - -export async function getImdbId(info, type) { - const name = escapeTitle(info.title); - const year = info.year || (info.date && info.date.slice(0, 4)); - const key = `${name}_${year || 'NA'}_${type}`; - const query = `${name} ${year || ''} ${type} imdb`; - const fallbackQuery = `${name} ${type} imdb`; - const googleQuery = year ? query : fallbackQuery; - - try { - const imdbId = await cacheWrapImdbId(key, - () => getIMDbIdFromNameToImdb(name, info.year, type) - ); - return imdbId && 'tt' + imdbId.replace(/tt0*([1-9][0-9]*)$/, '$1').padStart(7, '0'); - } catch (error) { - const imdbIdFallback = await getIMDbIdFromGoogle(googleQuery); - return imdbIdFallback && 'tt' + imdbIdFallback.replace(/tt0*([1-9][0-9]*)$/, '$1').padStart(7, '0'); - } -} - -function getIMDbIdFromNameToImdb(name, year, type) { - return new Promise((resolve, reject) => { - nameToImdb({ name, year, type }, function(err, res) { - if (res) { - resolve(res); - } else { - reject(err || new Error('Failed IMDbId search')); - } - }); - }); -} - -async function getIMDbIdFromGoogle(query) { - try { - const searchResults = await search({ query: query }); - for (const result of searchResults) { - if (result.link.includes('imdb.com/title/')) { - const match = result.link.match(/imdb\.com\/title\/(tt\d+)/); - if (match) { - return match[1]; - } - } - } - return undefined; - } - catch (error) { - throw new Error('Failed to find IMDb ID from Google search'); - } -} - -export async function getKitsuId(info) { - const title = escapeTitle(info.title.replace(/\s\|\s.*/, '')); - const year = info.year ? ` ${info.year}` : ''; - const season = info.season > 1 ? ` S${info.season}` : ''; - const key = `${title}${year}${season}`; - const query = encodeURIComponent(key); - - return cacheWrapKitsuId(key, - () => axios.get(`${KITSU_URL}/catalog/series/kitsu-anime-list/search=${query}.json`, { timeout: 60000 }) - .then((response) => { - const body = response.data; - if (body && body.metas && body.metas.length) { - return body.metas[0].id.replace('kitsu:', ''); - } else { - throw new Error('No search results'); - } - })); -} - -export async function isEpisodeImdbId(imdbId) { - if (!imdbId) { - return false; - } - return axios.get(`https://www.imdb.com/title/${imdbId}/`, { timeout: 10000 }) - .then(response => !!(response.data && response.data.includes('video.episode'))) - .catch(() => false); -} \ No newline at end of file diff --git a/src/node/consumer/src/lib/metadata.ts b/src/node/consumer/src/lib/metadata.ts new file mode 100644 index 0000000..e29e80b --- /dev/null +++ b/src/node/consumer/src/lib/metadata.ts @@ -0,0 +1,216 @@ +import axios, {AxiosResponse} from 'axios'; +import {search, ResultTypes} from 'google-sr'; +import nameToImdb from 'name-to-imdb'; +import { cacheWrapImdbId, cacheWrapKitsuId, cacheWrapMetadata } from './cache.js'; +import { TorrentType } from './enums/torrent_types'; +import {MetadataResponse} from "./interfaces/metadata_response"; +import {CinemetaJsonResponse} from "./interfaces/cinemeta_metadata"; +import {CommonVideoMetadata} from "./interfaces/common_video_metadata"; +import {KitsuJsonResponse} from "./interfaces/kitsu_metadata"; +import {MetaDataQuery} from "./interfaces/metadata_query"; +import {KitsuCatalogJsonResponse} from "./interfaces/kitsu_catalog_metadata"; + +const CINEMETA_URL = 'https://v3-cinemeta.strem.io'; +const KITSU_URL = 'https://anime-kitsu.strem.fun'; +const TIMEOUT = 20000; + +async function _requestMetadata(url: string): Promise { + let response: AxiosResponse = await axios.get(url, {timeout: TIMEOUT}); + let result : MetadataResponse; + const body = response.data; + if ('kitsu_id' in body.meta) { + result = handleKitsuResponse(body as KitsuJsonResponse); + } + else if ('imdb_id' in body.meta) { + result = handleCinemetaResponse(body as CinemetaJsonResponse); + } + else { + throw new Error('No valid metadata'); + } + + return result; +} + +function handleCinemetaResponse(body: CinemetaJsonResponse) : MetadataResponse { + return { + imdbId: parseInt(body.meta.imdb_id), + type: body.meta.type, + title: body.meta.name, + year: parseInt(body.meta.year), + country: body.meta.country, + genres: body.meta.genres, + status: body.meta.status, + videos: body.meta.videos + ? body.meta.videos.map(video => ({ + name: video.name, + season: video.season, + episode: video.episode, + imdbSeason: video.season, + imdbEpisode: video.episode, + })) + : [], + episodeCount: body.meta.videos + ? getEpisodeCount(body.meta.videos) + : [], + totalCount: body.meta.videos + ? body.meta.videos.filter( + entry => entry.season !== 0 && entry.episode !== 0 + ).length + : 0, + }; +} + +function handleKitsuResponse(body: KitsuJsonResponse) : MetadataResponse { + return { + kitsuId: parseInt(body.meta.kitsu_id), + type: body.meta.type, + title: body.meta.name, + year: parseInt(body.meta.year), + country: body.meta.country, + genres: body.meta.genres, + status: body.meta.status, + videos: body.meta.videos + ? body.meta.videos.map(video => ({ + name: video.title, + season: video.season, + episode: video.episode, + kitsuId: video.id, + kitsuEpisode: video.episode, + released: video.released, + })) + : [], + episodeCount: body.meta.videos + ? getEpisodeCount(body.meta.videos) + : [], + totalCount: body.meta.videos + ? body.meta.videos.filter( + entry => entry.season !== 0 && entry.episode !== 0 + ).length + : 0, + }; +} + +function getEpisodeCount(videos: CommonVideoMetadata[]) { + return Object.values( + videos + .filter(entry => entry.season !== 0 && entry.episode !== 0) + .sort((a, b) => a.season - b.season) + .reduce((map, next) => { + map[next.season] = map[next.season] + 1 || 1; + return map; + }, {}) + ); +} + + +export function escapeTitle(title: string): string { + return title.toLowerCase() + .normalize('NFKD') // normalize non-ASCII characters + .replace(/[\u0300-\u036F]/g, '') + .replace(/&/g, 'and') + .replace(/[;, ~./]+/g, ' ') // replace dots, commas or underscores with spaces + .replace(/[^\w \-()×+#@!'\u0400-\u04ff]+/g, '') // remove all non-alphanumeric chars + .replace(/^\d{1,2}[.#\s]+(?=(?:\d+[.\s]*)?[\u0400-\u04ff])/i, '') // remove russian movie numbering + .replace(/\s{2,}/, ' ') // replace multiple spaces + .trim(); +} + +function getIMDbIdFromNameToImdb(name: string, info: MetaDataQuery) : Promise { + const year = info.year; + const type = info.type; + return new Promise((resolve, reject) => { + nameToImdb({ name, year, type }, function(err: Error, res: string) { + if (res) { + resolve(res); + } else { + reject(err || new Error('Failed IMDbId search')); + } + }); + }); +} + +async function getIMDbIdFromGoogle(query: string): Promise{ + try { + const searchResults = await search({ query: query }); + for(const result of searchResults) { + if(result.type === ResultTypes.SearchResult) { + if(result.link.includes('imdb.com/title/')){ + const match = result.link.match(/imdb\.com\/title\/(tt\d+)/); + if(match){ + return match[1]; + } + } + } + } + return undefined; + } + catch (error) { + throw new Error('Failed to find IMDb ID from Google search'); + } +} + +export async function getKitsuId(info: MetaDataQuery): Promise { + const title = escapeTitle(info.title.replace(/\s\|\s.*/, '')); + const year = info.year ? ` ${info.year}` : ''; + const season = info.season > 1 ? ` S${info.season}` : ''; + const key = `${title}${year}${season}`; + const query = encodeURIComponent(key); + + return cacheWrapKitsuId(key, + () => axios.get(`${KITSU_URL}/catalog/series/kitsu-anime-list/search=${query}.json`, { timeout: 60000 }) + .then((response) => { + const body = response.data as KitsuCatalogJsonResponse; + if (body && body.metas && body.metas.length) { + return body.metas[0].id.replace('kitsu:', ''); + } else { + throw new Error('No search results'); + } + })); +} + +export async function getImdbId(info: MetaDataQuery): Promise { + const name = escapeTitle(info.title); + const year = info.year || (info.date && info.date.slice(0, 4)); + const key = `${name}_${year || 'NA'}_${info.type}`; + const query = `${name} ${year || ''} ${info.type} imdb`; + const fallbackQuery = `${name} ${info.type} imdb`; + const googleQuery = year ? query : fallbackQuery; + + try { + const imdbId = await cacheWrapImdbId(key, + () => getIMDbIdFromNameToImdb(name, info) + ); + return imdbId && 'tt' + imdbId.replace(/tt0*([1-9][0-9]*)$/, '$1').padStart(7, '0'); + } catch (error) { + const imdbIdFallback = await getIMDbIdFromGoogle(googleQuery); + return imdbIdFallback && 'tt' + imdbIdFallback.toString().replace(/tt0*([1-9][0-9]*)$/, '$1').padStart(7, '0'); + } +} + +export function getMetadata(id: string | number, type: TorrentType = TorrentType.SERIES): Promise { + if (!id) { + return Promise.reject("no valid id provided"); + } + + const key = Number.isInteger(id) || id.toString().match(/^\d+$/) ? `kitsu:${id}` : id; + const metaType = type === TorrentType.MOVIE ? TorrentType.MOVIE : TorrentType.SERIES; + return cacheWrapMetadata(key, () => _requestMetadata(`${KITSU_URL}/meta/${metaType}/${key}.json`) + .catch(() => _requestMetadata(`${CINEMETA_URL}/meta/${metaType}/${key}.json`)) + .catch(() => { + // try different type in case there was a mismatch + const otherType = metaType === TorrentType.MOVIE ? TorrentType.SERIES : TorrentType.MOVIE; + return _requestMetadata(`${CINEMETA_URL}/meta/${otherType}/${key}.json`) + }) + .catch((error) => { + throw new Error(`failed metadata query ${key} due: ${error.message}`); + })); +} + +export async function isEpisodeImdbId(imdbId: string | undefined): Promise { + if (!imdbId) { + return false; + } + return axios.get(`https://www.imdb.com/title/${imdbId}/`, { timeout: 10000 }) + .then(response => !!(response.data && response.data.includes('video.episode'))) + .catch(() => false); +} \ No newline at end of file diff --git a/src/node/consumer/src/lib/torrentEntries.js b/src/node/consumer/src/lib/torrentEntries.js index d5a629d..18d4940 100644 --- a/src/node/consumer/src/lib/torrentEntries.js +++ b/src/node/consumer/src/lib/torrentEntries.js @@ -1,5 +1,5 @@ import { parse } from 'parse-torrent-title'; -import { getImdbId, getKitsuId } from './metadata.js'; +import { getImdbId, getKitsuId } from './metadata'; import { isPackTorrent } from './parseHelper.js'; import * as Promises from './promises.js'; import { repository } from '../repository/database_repository'; @@ -12,7 +12,12 @@ export async function createTorrentEntry(torrent, overwrite = false) { const titleInfo = parse(torrent.title); if (!torrent.imdbId && torrent.type !== TorrentType.ANIME) { - torrent.imdbId = await getImdbId(titleInfo, torrent.type) + const imdbQuery = { + title: titleInfo.title, + year: titleInfo.year, + type: torrent.type + }; + torrent.imdbId = await getImdbId(imdbQuery) .catch(() => undefined); } if (torrent.imdbId && torrent.imdbId.length < 9) { @@ -24,7 +29,12 @@ export async function createTorrentEntry(torrent, overwrite = false) { torrent.imdbId = torrent.imdbId.replace(/tt0+([0-9]{7,})$/, 'tt$1'); } if (!torrent.kitsuId && torrent.type === TorrentType.ANIME) { - torrent.kitsuId = await getKitsuId(titleInfo) + const kitsuQuery = { + title: titleInfo.title, + year: titleInfo.year, + season: titleInfo.season, + }; + torrent.kitsuId = await getKitsuId(kitsuQuery) .catch(() => undefined); } diff --git a/src/node/consumer/src/lib/torrentFiles.js b/src/node/consumer/src/lib/torrentFiles.js index 0a59c56..31e8d98 100644 --- a/src/node/consumer/src/lib/torrentFiles.js +++ b/src/node/consumer/src/lib/torrentFiles.js @@ -4,7 +4,7 @@ import moment from 'moment'; import { parse } from 'parse-torrent-title'; import { metadataConfig } from './config.js'; import { isDisk } from './extension.js'; -import { getMetadata, getImdbId, getKitsuId } from './metadata.js'; +import { getMetadata, getImdbId, getKitsuId } from './metadata'; import { parseSeriesVideos, isPackTorrent } from './parseHelper.js'; import * as Promises from './promises.js'; import {torrentFiles} from "./torrent.js"; @@ -472,12 +472,25 @@ async function updateToCinemetaMetadata(metadata) { function findMovieImdbId(title) { const parsedTitle = typeof title === 'string' ? parse(title) : title; logger.debug(`Finding movie imdbId for ${title}`); - return imdb_limiter.schedule(() => getImdbId(parsedTitle, TorrentType.MOVIE).catch(() => undefined)); + return imdb_limiter.schedule(() => { + const imdbQuery = { + title: parsedTitle.title, + year: parsedTitle.year, + type: TorrentType.MOVIE + }; + return getImdbId(imdbQuery).catch(() => undefined); + }); } function findMovieKitsuId(title) { const parsedTitle = typeof title === 'string' ? parse(title) : title; - return getKitsuId(parsedTitle, TorrentType.MOVIE).catch(() => undefined); + const kitsuQuery = { + title: parsedTitle.title, + year: parsedTitle.year, + season: parsedTitle.season, + type: TorrentType.MOVIE + }; + return getKitsuId(kitsuQuery).catch(() => undefined); } function isDiskTorrent(contents) {