diff --git a/src/node/consumer/.dockerignore b/src/node/consumer/.dockerignore index 886cfa9..9b49524 100644 --- a/src/node/consumer/.dockerignore +++ b/src/node/consumer/.dockerignore @@ -1,2 +1,15 @@ -build.sh -node_modules/ \ No newline at end of file +node_modules +Dockerfile* +docker-compose* +.dockerignore +.git +.gitignore +README.md +LICENSE +.vscode +Makefile +helm-charts +.env +.editorconfig +.idea +coverage* \ No newline at end of file diff --git a/src/node/consumer/src/index.js b/src/node/consumer/src/index.js index 9425c05..ac1dd83 100644 --- a/src/node/consumer/src/index.js +++ b/src/node/consumer/src/index.js @@ -1,13 +1,9 @@ import { listenToQueue } from './jobs/processTorrents.js'; -import { jobConfig } from "./lib/config.js"; import { connect } from './lib/repository.js'; import { getTrackers } from "./lib/trackerService.js"; (async () => { await getTrackers(); await connect(); - - if (jobConfig.JOBS_ENABLED) { - await listenToQueue(); - } + await listenToQueue(); })(); \ No newline at end of file diff --git a/src/node/consumer/src/lib/ingestedTorrent.js b/src/node/consumer/src/lib/ingestedTorrent.js index 44473c1..4948173 100644 --- a/src/node/consumer/src/lib/ingestedTorrent.js +++ b/src/node/consumer/src/lib/ingestedTorrent.js @@ -1,12 +1,13 @@ import { createTorrentEntry, checkAndUpdateTorrent } from './torrentEntries.js'; import {getTrackers} from "./trackerService.js"; -import { Type } from './types.js'; +import { TorrentType } from './types.js'; +import {logger} from "./logger.js"; export async function processTorrentRecord(torrent) { const {category} = torrent; - const type = category === 'tv' ? Type.SERIES : Type.MOVIE; + const type = category === 'tv' ? TorrentType.SERIES : TorrentType.MOVIE; const torrentInfo = await parseTorrent(torrent, type); - console.log(`Processing torrent ${torrentInfo.title} with infoHash ${torrentInfo.infoHash}`) + logger.info(`Processing torrent ${torrentInfo.title} with infoHash ${torrentInfo.infoHash}`) if (await checkAndUpdateTorrent(torrentInfo)) { return torrentInfo; diff --git a/src/node/consumer/src/lib/metadata.js b/src/node/consumer/src/lib/metadata.js index 6d76282..874decb 100644 --- a/src/node/consumer/src/lib/metadata.js +++ b/src/node/consumer/src/lib/metadata.js @@ -2,24 +2,24 @@ import axios from 'axios'; import { search } from 'google-sr'; import nameToImdb from 'name-to-imdb'; import { cacheWrapImdbId, cacheWrapKitsuId, cacheWrapMetadata } from './cache.js'; -import { Type } from './types.js'; +import { TorrentType } from './types.js'; const CINEMETA_URL = 'https://v3-cinemeta.strem.io'; const KITSU_URL = 'https://anime-kitsu.strem.fun'; const TIMEOUT = 20000; -export function getMetadata(id, type = Type.SERIES) { +export function getMetadata(id, type = TorrentType.SERIES) { if (!id) { return Promise.reject("no valid id provided"); } const key = Number.isInteger(id) || id.match(/^\d+$/) ? `kitsu:${id}` : id; - const metaType = type === Type.MOVIE ? Type.MOVIE : Type.SERIES; + const metaType = type === TorrentType.MOVIE ? TorrentType.MOVIE : TorrentType.SERIES; return cacheWrapMetadata(key, () => _requestMetadata(`${KITSU_URL}/meta/${metaType}/${key}.json`) .catch(() => _requestMetadata(`${CINEMETA_URL}/meta/${metaType}/${key}.json`)) .catch(() => { // try different type in case there was a mismatch - const otherType = metaType === Type.MOVIE ? Type.SERIES : Type.MOVIE; + const otherType = metaType === TorrentType.MOVIE ? TorrentType.SERIES : TorrentType.MOVIE; return _requestMetadata(`${CINEMETA_URL}/meta/${otherType}/${key}.json`) }) .catch((error) => { diff --git a/src/node/consumer/src/lib/parseHelper.js b/src/node/consumer/src/lib/parseHelper.js index bfb9c5d..b088687 100644 --- a/src/node/consumer/src/lib/parseHelper.js +++ b/src/node/consumer/src/lib/parseHelper.js @@ -1,5 +1,5 @@ import { parse } from 'parse-torrent-title'; -import { Type } from './types.js'; +import { TorrentType } from './types.js'; const MULTIPLE_FILES_SIZE = 4 * 1024 * 1024 * 1024; // 4 GB @@ -65,7 +65,7 @@ function isMovieVideo(video, otherVideos, type, hasMovies) { // movie if video explicitly has numbered movie keyword in the name, ie. 1 Movie or Movie 1 return true; } - if (!hasMovies && type !== Type.ANIME) { + if (!hasMovies && type !== TorrentType.ANIME) { // not movie if torrent name does not contain movies keyword or is not a pack torrent and is not anime return false; } @@ -85,7 +85,7 @@ export function isPackTorrent(torrent) { return true; } const parsedInfo = parse(torrent.title); - if (torrent.type === Type.MOVIE) { + if (torrent.type === TorrentType.MOVIE) { return parsedInfo.complete || typeof parsedInfo.year === 'string' || /movies/i.test(torrent.title); } const hasMultipleEpisodes = parsedInfo.complete || diff --git a/src/node/consumer/src/lib/repository.js b/src/node/consumer/src/lib/repository.js index 731e381..eb26ec0 100644 --- a/src/node/consumer/src/lib/repository.js +++ b/src/node/consumer/src/lib/repository.js @@ -1,6 +1,7 @@ import moment from 'moment'; import { Sequelize, Op, DataTypes, fn, col, literal } from 'sequelize'; import { databaseConfig } from './config.js'; +import {logger} from "./logger.js"; import * as Promises from './promises.js'; const database = new Sequelize( @@ -185,7 +186,7 @@ export function connect() { if (databaseConfig.ENABLE_SYNC) { return database.sync({ alter: true }) .catch(error => { - console.error('Failed syncing database: ', error); + logger.error('Failed syncing database: ', error); throw error; }); } diff --git a/src/node/consumer/src/lib/torrentEntries.js b/src/node/consumer/src/lib/torrentEntries.js index 3eff76b..998b92e 100644 --- a/src/node/consumer/src/lib/torrentEntries.js +++ b/src/node/consumer/src/lib/torrentEntries.js @@ -5,12 +5,13 @@ import * as Promises from './promises.js'; import * as repository from './repository.js'; import { parseTorrentFiles } from './torrentFiles.js'; import { assignSubtitles } from './torrentSubtitles.js'; -import { Type } from './types.js'; +import { TorrentType } from './types.js'; +import {logger} from "./logger.js"; export async function createTorrentEntry(torrent, overwrite = false) { const titleInfo = parse(torrent.title); - if (!torrent.imdbId && torrent.type !== Type.ANIME) { + if (!torrent.imdbId && torrent.type !== TorrentType.ANIME) { torrent.imdbId = await getImdbId(titleInfo, torrent.type) .catch(() => undefined); } @@ -22,13 +23,13 @@ export async function createTorrentEntry(torrent, overwrite = false) { // sanitize imdbId from redundant zeros torrent.imdbId = torrent.imdbId.replace(/tt0+([0-9]{7,})$/, 'tt$1'); } - if (!torrent.kitsuId && torrent.type === Type.ANIME) { + if (!torrent.kitsuId && torrent.type === TorrentType.ANIME) { torrent.kitsuId = await getKitsuId(titleInfo) .catch(() => undefined); } if (!torrent.imdbId && !torrent.kitsuId && !isPackTorrent(torrent)) { - console.log(`imdbId or kitsuId not found: ${torrent.provider} ${torrent.title}`); + logger.warn(`imdbId or kitsuId not found: ${torrent.provider} ${torrent.title}`); return; } @@ -36,17 +37,17 @@ export async function createTorrentEntry(torrent, overwrite = false) { .then(torrentContents => overwrite ? overwriteExistingFiles(torrent, torrentContents) : torrentContents) .then(torrentContents => assignSubtitles(torrentContents)) .catch(error => { - console.log(`Failed getting files for ${torrent.title}`, error.message); + logger.warn(`Failed getting files for ${torrent.title}`, error.message); return {}; }); if (!videos || !videos.length) { - console.log(`no video files found for ${torrent.provider} [${torrent.infoHash}] ${torrent.title}`); + logger.warn(`no video files found for ${torrent.provider} [${torrent.infoHash}] ${torrent.title}`); return; } return repository.createTorrent({ ...torrent, contents, subtitles }) .then(() => Promises.sequence(videos.map(video => () => repository.createFile(video)))) - .then(() => console.log(`Created ${torrent.provider} entry for [${torrent.infoHash}] ${torrent.title}`)); + .then(() => logger.info(`Created ${torrent.provider} entry for [${torrent.infoHash}] ${torrent.title}`)); } async function overwriteExistingFiles(torrent, torrentContents) { @@ -106,7 +107,7 @@ export async function checkAndUpdateTorrent(torrent) { if (!storedTorrent.languages && torrent.languages && storedTorrent.provider !== 'RARBG') { storedTorrent.languages = torrent.languages; await storedTorrent.save(); - console.log(`Updated [${storedTorrent.infoHash}] ${storedTorrent.title} language to ${torrent.languages}`); + logger.debug(`Updated [${storedTorrent.infoHash}] ${storedTorrent.title} language to ${torrent.languages}`); } return createTorrentContents({ ...storedTorrent.get(), torrentLink: torrent.torrentLink }) .then(() => updateTorrentSeeders(torrent)); @@ -128,7 +129,7 @@ export async function createTorrentContents(torrent) { .then(torrentContents => notOpenedVideo ? torrentContents : { ...torrentContents, videos: storedVideos }) .then(torrentContents => assignSubtitles(torrentContents)) .catch(error => { - console.log(`Failed getting contents for [${torrent.infoHash}] ${torrent.title}`, error.message); + logger.warn(`Failed getting contents for [${torrent.infoHash}] ${torrent.title}`, error.message); return {}; }); @@ -149,14 +150,14 @@ export async function createTorrentContents(torrent) { return repository.createTorrent({ ...torrent, contents, subtitles }) .then(() => { if (shouldDeleteOld) { - console.error(`Deleting old video for [${torrent.infoHash}] ${torrent.title}`) + logger.debug(`Deleting old video for [${torrent.infoHash}] ${torrent.title}`) return storedVideos[0].destroy(); } return Promise.resolve(); }) .then(() => Promises.sequence(videos.map(video => () => repository.createFile(video)))) - .then(() => console.log(`Created contents for ${torrent.provider} [${torrent.infoHash}] ${torrent.title}`)) - .catch(error => console.error(`Failed saving contents for [${torrent.infoHash}] ${torrent.title}`, error)); + .then(() => logger.info(`Created contents for ${torrent.provider} [${torrent.infoHash}] ${torrent.title}`)) + .catch(error => logger.error(`Failed saving contents for [${torrent.infoHash}] ${torrent.title}`, error)); } export async function updateTorrentSeeders(torrent) { @@ -166,7 +167,7 @@ export async function updateTorrentSeeders(torrent) { return repository.setTorrentSeeders(torrent, torrent.seeders) .catch(error => { - console.warn('Failed updating seeders:', error); + logger.warn('Failed updating seeders:', error); return undefined; }); } diff --git a/src/node/consumer/src/lib/torrentFiles.js b/src/node/consumer/src/lib/torrentFiles.js index c1f6757..0ac9ea9 100644 --- a/src/node/consumer/src/lib/torrentFiles.js +++ b/src/node/consumer/src/lib/torrentFiles.js @@ -8,26 +8,27 @@ import { getMetadata, getImdbId, getKitsuId } from './metadata.js'; import { parseSeriesVideos, isPackTorrent } from './parseHelper.js'; import * as Promises from './promises.js'; import {torrentFiles} from "./torrent.js"; -import { Type } from './types.js'; +import { TorrentType } from './types.js'; +import {logger} from "./logger.js"; const MIN_SIZE = 5 * 1024 * 1024; // 5 MB const imdb_limiter = new Bottleneck({ maxConcurrent: metadataConfig.IMDB_CONCURRENT, minTime: metadataConfig.IMDB_INTERVAL_MS }); export async function parseTorrentFiles(torrent) { const parsedTorrentName = parse(torrent.title); - const metadata = await getMetadata(torrent.kitsuId || torrent.imdbId, torrent.type || Type.MOVIE) + const metadata = await getMetadata(torrent.kitsuId || torrent.imdbId, torrent.type || TorrentType.MOVIE) .then(meta => Object.assign({}, meta)) .catch(() => undefined); // if (metadata && metadata.type !== torrent.type && torrent.type !== Type.ANIME) { // throw new Error(`Mismatching entry type for ${torrent.name}: ${torrent.type}!=${metadata.type}`); // } - if (torrent.type !== Type.ANIME && metadata && metadata.type && metadata.type !== torrent.type) { + if (torrent.type !== TorrentType.ANIME && metadata && metadata.type && metadata.type !== torrent.type) { // it's actually a movie/series torrent.type = metadata.type; } - if (torrent.type === Type.MOVIE && (!parsedTorrentName.seasons || + if (torrent.type === TorrentType.MOVIE && (!parsedTorrentName.seasons || parsedTorrentName.season === 5 && [1, 5].includes(parsedTorrentName.episode))) { return parseMovieFiles(torrent, parsedTorrentName, metadata); } @@ -133,9 +134,9 @@ async function mapSeriesEpisode(file, torrent, files) { } async function mapSeriesMovie(file, torrent) { - const kitsuId = torrent.type === Type.ANIME ? await findMovieKitsuId(file) : undefined; + const kitsuId = torrent.type === TorrentType.ANIME ? await findMovieKitsuId(file) : undefined; const imdbId = !kitsuId ? await findMovieImdbId(file) : undefined; - const metadata = await getMetadata(kitsuId || imdbId, Type.MOVIE).catch(() => ({})); + const metadata = await getMetadata(kitsuId || imdbId, TorrentType.MOVIE).catch(() => ({})); const hasEpisode = metadata.videos && metadata.videos.length && (file.episode || metadata.videos.length === 1); const episodeVideo = hasEpisode && metadata.videos[(file.episode || 1) - 1]; return [{ @@ -158,7 +159,7 @@ async function decomposeEpisodes(torrent, files, metadata = { episodeCount: [] } preprocessEpisodes(files); - if (torrent.type === Type.ANIME && torrent.kitsuId) { + if (torrent.type === TorrentType.ANIME && torrent.kitsuId) { if (needsCinemetaMetadataForAnime(files, metadata)) { // In some cases anime could be resolved to wrong kitsuId // because of imdb season naming/absolute per series naming/multiple seasons @@ -240,7 +241,7 @@ function isDateEpisodeFiles(files, metadata) { function isAbsoluteEpisodeFiles(torrent, files, metadata) { const threshold = Math.ceil(files.length / 5); - const isAnime = torrent.type === Type.ANIME && torrent.kitsuId; + const isAnime = torrent.type === TorrentType.ANIME && torrent.kitsuId; const nonMovieEpisodes = files .filter(file => !file.isMovie && file.episodes); const absoluteEpisodes = files @@ -255,7 +256,7 @@ function isNewEpisodeNotInMetadata(torrent, file, metadata) { // new episode might not yet been indexed by cinemeta. // detect this if episode number is larger than the last episode or season is larger than the last one // only for non anime metas - const isAnime = torrent.type === Type.ANIME && torrent.kitsuId; + const isAnime = torrent.type === TorrentType.ANIME && torrent.kitsuId; return !isAnime && !file.isMovie && file.episodes && file.season !== 1 && /continuing|current/i.test(metadata.status) && file.season >= metadata.episodeCount.length @@ -355,7 +356,7 @@ function getTimeZoneOffset(country) { function assignKitsuOrImdbEpisodes(torrent, files, metadata) { if (!metadata || !metadata.videos || !metadata.videos.length) { - if (torrent.type === Type.ANIME) { + if (torrent.type === TorrentType.ANIME) { // assign episodes as kitsu episodes for anime when no metadata available for imdb mapping files .filter(file => file.season && file.episodes) @@ -364,7 +365,7 @@ function assignKitsuOrImdbEpisodes(torrent, files, metadata) { file.season = undefined; file.episodes = undefined; }) - if (metadata.type === Type.MOVIE && files.every(file => !file.imdbId)) { + if (metadata.type === TorrentType.MOVIE && files.every(file => !file.imdbId)) { // sometimes a movie has episode naming, thus not recognized as a movie and imdbId not assigned files.forEach(file => file.imdbId = metadata.imdbId); } @@ -465,18 +466,18 @@ async function updateToCinemetaMetadata(metadata) { metadata.totalCount = newMetadata.totalCount; return metadata; }) - .catch(error => console.warn(`Failed ${metadata.imdbId} metadata cinemeta update due: ${error.message}`)); + .catch(error => logger.warn(`Failed ${metadata.imdbId} metadata cinemeta update due: ${error.message}`)); } function findMovieImdbId(title) { const parsedTitle = typeof title === 'string' ? parse(title) : title; - console.log(`Finding movie imdbId for ${title}`); - return imdb_limiter.schedule(() => getImdbId(parsedTitle, Type.MOVIE).catch(() => undefined)); + logger.debug(`Finding movie imdbId for ${title}`); + return imdb_limiter.schedule(() => getImdbId(parsedTitle, TorrentType.MOVIE).catch(() => undefined)); } function findMovieKitsuId(title) { const parsedTitle = typeof title === 'string' ? parse(title) : title; - return getKitsuId(parsedTitle, Type.MOVIE).catch(() => undefined); + return getKitsuId(parsedTitle, TorrentType.MOVIE).catch(() => undefined); } function isDiskTorrent(contents) { diff --git a/src/node/consumer/src/lib/trackerService.js b/src/node/consumer/src/lib/trackerService.js index 2f9b496..1af9b13 100644 --- a/src/node/consumer/src/lib/trackerService.js +++ b/src/node/consumer/src/lib/trackerService.js @@ -1,6 +1,7 @@ import axios from 'axios'; import {cacheTrackers} from "./cache.js"; import { trackerConfig } from './config.js'; +import {logger} from "./logger.js"; const downloadTrackers = async () => { const response = await axios.get(trackerConfig.TRACKERS_URL); @@ -15,7 +16,7 @@ const downloadTrackers = async () => { urlTrackers = urlTrackers.filter(line => !line.startsWith('udp://')); } - console.log(`Trackers updated at ${Date.now()}: ${urlTrackers.length} trackers`); + logger.info(`Trackers updated at ${Date.now()}: ${urlTrackers.length} trackers`); return urlTrackers; };