roll logging levels out throughout the consumer

This commit is contained in:
iPromKnight
2024-02-03 22:06:38 +00:00
parent 4bd76d682f
commit 8c1b6fc91b
9 changed files with 61 additions and 47 deletions

View File

@@ -1,2 +1,15 @@
build.sh node_modules
node_modules/ Dockerfile*
docker-compose*
.dockerignore
.git
.gitignore
README.md
LICENSE
.vscode
Makefile
helm-charts
.env
.editorconfig
.idea
coverage*

View File

@@ -1,13 +1,9 @@
import { listenToQueue } from './jobs/processTorrents.js'; import { listenToQueue } from './jobs/processTorrents.js';
import { jobConfig } from "./lib/config.js";
import { connect } from './lib/repository.js'; import { connect } from './lib/repository.js';
import { getTrackers } from "./lib/trackerService.js"; import { getTrackers } from "./lib/trackerService.js";
(async () => { (async () => {
await getTrackers(); await getTrackers();
await connect(); await connect();
await listenToQueue();
if (jobConfig.JOBS_ENABLED) {
await listenToQueue();
}
})(); })();

View File

@@ -1,12 +1,13 @@
import { createTorrentEntry, checkAndUpdateTorrent } from './torrentEntries.js'; import { createTorrentEntry, checkAndUpdateTorrent } from './torrentEntries.js';
import {getTrackers} from "./trackerService.js"; import {getTrackers} from "./trackerService.js";
import { Type } from './types.js'; import { TorrentType } from './types.js';
import {logger} from "./logger.js";
export async function processTorrentRecord(torrent) { export async function processTorrentRecord(torrent) {
const {category} = torrent; const {category} = torrent;
const type = category === 'tv' ? Type.SERIES : Type.MOVIE; const type = category === 'tv' ? TorrentType.SERIES : TorrentType.MOVIE;
const torrentInfo = await parseTorrent(torrent, type); const torrentInfo = await parseTorrent(torrent, type);
console.log(`Processing torrent ${torrentInfo.title} with infoHash ${torrentInfo.infoHash}`) logger.info(`Processing torrent ${torrentInfo.title} with infoHash ${torrentInfo.infoHash}`)
if (await checkAndUpdateTorrent(torrentInfo)) { if (await checkAndUpdateTorrent(torrentInfo)) {
return torrentInfo; return torrentInfo;

View File

@@ -2,24 +2,24 @@ import axios from 'axios';
import { search } from 'google-sr'; import { search } from 'google-sr';
import nameToImdb from 'name-to-imdb'; import nameToImdb from 'name-to-imdb';
import { cacheWrapImdbId, cacheWrapKitsuId, cacheWrapMetadata } from './cache.js'; import { cacheWrapImdbId, cacheWrapKitsuId, cacheWrapMetadata } from './cache.js';
import { Type } from './types.js'; import { TorrentType } from './types.js';
const CINEMETA_URL = 'https://v3-cinemeta.strem.io'; const CINEMETA_URL = 'https://v3-cinemeta.strem.io';
const KITSU_URL = 'https://anime-kitsu.strem.fun'; const KITSU_URL = 'https://anime-kitsu.strem.fun';
const TIMEOUT = 20000; const TIMEOUT = 20000;
export function getMetadata(id, type = Type.SERIES) { export function getMetadata(id, type = TorrentType.SERIES) {
if (!id) { if (!id) {
return Promise.reject("no valid id provided"); return Promise.reject("no valid id provided");
} }
const key = Number.isInteger(id) || id.match(/^\d+$/) ? `kitsu:${id}` : id; const key = Number.isInteger(id) || id.match(/^\d+$/) ? `kitsu:${id}` : id;
const metaType = type === Type.MOVIE ? Type.MOVIE : Type.SERIES; const metaType = type === TorrentType.MOVIE ? TorrentType.MOVIE : TorrentType.SERIES;
return cacheWrapMetadata(key, () => _requestMetadata(`${KITSU_URL}/meta/${metaType}/${key}.json`) return cacheWrapMetadata(key, () => _requestMetadata(`${KITSU_URL}/meta/${metaType}/${key}.json`)
.catch(() => _requestMetadata(`${CINEMETA_URL}/meta/${metaType}/${key}.json`)) .catch(() => _requestMetadata(`${CINEMETA_URL}/meta/${metaType}/${key}.json`))
.catch(() => { .catch(() => {
// try different type in case there was a mismatch // try different type in case there was a mismatch
const otherType = metaType === Type.MOVIE ? Type.SERIES : Type.MOVIE; const otherType = metaType === TorrentType.MOVIE ? TorrentType.SERIES : TorrentType.MOVIE;
return _requestMetadata(`${CINEMETA_URL}/meta/${otherType}/${key}.json`) return _requestMetadata(`${CINEMETA_URL}/meta/${otherType}/${key}.json`)
}) })
.catch((error) => { .catch((error) => {

View File

@@ -1,5 +1,5 @@
import { parse } from 'parse-torrent-title'; import { parse } from 'parse-torrent-title';
import { Type } from './types.js'; import { TorrentType } from './types.js';
const MULTIPLE_FILES_SIZE = 4 * 1024 * 1024 * 1024; // 4 GB const MULTIPLE_FILES_SIZE = 4 * 1024 * 1024 * 1024; // 4 GB
@@ -65,7 +65,7 @@ function isMovieVideo(video, otherVideos, type, hasMovies) {
// movie if video explicitly has numbered movie keyword in the name, ie. 1 Movie or Movie 1 // movie if video explicitly has numbered movie keyword in the name, ie. 1 Movie or Movie 1
return true; return true;
} }
if (!hasMovies && type !== Type.ANIME) { if (!hasMovies && type !== TorrentType.ANIME) {
// not movie if torrent name does not contain movies keyword or is not a pack torrent and is not anime // not movie if torrent name does not contain movies keyword or is not a pack torrent and is not anime
return false; return false;
} }
@@ -85,7 +85,7 @@ export function isPackTorrent(torrent) {
return true; return true;
} }
const parsedInfo = parse(torrent.title); const parsedInfo = parse(torrent.title);
if (torrent.type === Type.MOVIE) { if (torrent.type === TorrentType.MOVIE) {
return parsedInfo.complete || typeof parsedInfo.year === 'string' || /movies/i.test(torrent.title); return parsedInfo.complete || typeof parsedInfo.year === 'string' || /movies/i.test(torrent.title);
} }
const hasMultipleEpisodes = parsedInfo.complete || const hasMultipleEpisodes = parsedInfo.complete ||

View File

@@ -1,6 +1,7 @@
import moment from 'moment'; import moment from 'moment';
import { Sequelize, Op, DataTypes, fn, col, literal } from 'sequelize'; import { Sequelize, Op, DataTypes, fn, col, literal } from 'sequelize';
import { databaseConfig } from './config.js'; import { databaseConfig } from './config.js';
import {logger} from "./logger.js";
import * as Promises from './promises.js'; import * as Promises from './promises.js';
const database = new Sequelize( const database = new Sequelize(
@@ -185,7 +186,7 @@ export function connect() {
if (databaseConfig.ENABLE_SYNC) { if (databaseConfig.ENABLE_SYNC) {
return database.sync({ alter: true }) return database.sync({ alter: true })
.catch(error => { .catch(error => {
console.error('Failed syncing database: ', error); logger.error('Failed syncing database: ', error);
throw error; throw error;
}); });
} }

View File

@@ -5,12 +5,13 @@ import * as Promises from './promises.js';
import * as repository from './repository.js'; import * as repository from './repository.js';
import { parseTorrentFiles } from './torrentFiles.js'; import { parseTorrentFiles } from './torrentFiles.js';
import { assignSubtitles } from './torrentSubtitles.js'; import { assignSubtitles } from './torrentSubtitles.js';
import { Type } from './types.js'; import { TorrentType } from './types.js';
import {logger} from "./logger.js";
export async function createTorrentEntry(torrent, overwrite = false) { export async function createTorrentEntry(torrent, overwrite = false) {
const titleInfo = parse(torrent.title); const titleInfo = parse(torrent.title);
if (!torrent.imdbId && torrent.type !== Type.ANIME) { if (!torrent.imdbId && torrent.type !== TorrentType.ANIME) {
torrent.imdbId = await getImdbId(titleInfo, torrent.type) torrent.imdbId = await getImdbId(titleInfo, torrent.type)
.catch(() => undefined); .catch(() => undefined);
} }
@@ -22,13 +23,13 @@ export async function createTorrentEntry(torrent, overwrite = false) {
// sanitize imdbId from redundant zeros // sanitize imdbId from redundant zeros
torrent.imdbId = torrent.imdbId.replace(/tt0+([0-9]{7,})$/, 'tt$1'); torrent.imdbId = torrent.imdbId.replace(/tt0+([0-9]{7,})$/, 'tt$1');
} }
if (!torrent.kitsuId && torrent.type === Type.ANIME) { if (!torrent.kitsuId && torrent.type === TorrentType.ANIME) {
torrent.kitsuId = await getKitsuId(titleInfo) torrent.kitsuId = await getKitsuId(titleInfo)
.catch(() => undefined); .catch(() => undefined);
} }
if (!torrent.imdbId && !torrent.kitsuId && !isPackTorrent(torrent)) { if (!torrent.imdbId && !torrent.kitsuId && !isPackTorrent(torrent)) {
console.log(`imdbId or kitsuId not found: ${torrent.provider} ${torrent.title}`); logger.warn(`imdbId or kitsuId not found: ${torrent.provider} ${torrent.title}`);
return; return;
} }
@@ -36,17 +37,17 @@ export async function createTorrentEntry(torrent, overwrite = false) {
.then(torrentContents => overwrite ? overwriteExistingFiles(torrent, torrentContents) : torrentContents) .then(torrentContents => overwrite ? overwriteExistingFiles(torrent, torrentContents) : torrentContents)
.then(torrentContents => assignSubtitles(torrentContents)) .then(torrentContents => assignSubtitles(torrentContents))
.catch(error => { .catch(error => {
console.log(`Failed getting files for ${torrent.title}`, error.message); logger.warn(`Failed getting files for ${torrent.title}`, error.message);
return {}; return {};
}); });
if (!videos || !videos.length) { if (!videos || !videos.length) {
console.log(`no video files found for ${torrent.provider} [${torrent.infoHash}] ${torrent.title}`); logger.warn(`no video files found for ${torrent.provider} [${torrent.infoHash}] ${torrent.title}`);
return; return;
} }
return repository.createTorrent({ ...torrent, contents, subtitles }) return repository.createTorrent({ ...torrent, contents, subtitles })
.then(() => Promises.sequence(videos.map(video => () => repository.createFile(video)))) .then(() => Promises.sequence(videos.map(video => () => repository.createFile(video))))
.then(() => console.log(`Created ${torrent.provider} entry for [${torrent.infoHash}] ${torrent.title}`)); .then(() => logger.info(`Created ${torrent.provider} entry for [${torrent.infoHash}] ${torrent.title}`));
} }
async function overwriteExistingFiles(torrent, torrentContents) { async function overwriteExistingFiles(torrent, torrentContents) {
@@ -106,7 +107,7 @@ export async function checkAndUpdateTorrent(torrent) {
if (!storedTorrent.languages && torrent.languages && storedTorrent.provider !== 'RARBG') { if (!storedTorrent.languages && torrent.languages && storedTorrent.provider !== 'RARBG') {
storedTorrent.languages = torrent.languages; storedTorrent.languages = torrent.languages;
await storedTorrent.save(); await storedTorrent.save();
console.log(`Updated [${storedTorrent.infoHash}] ${storedTorrent.title} language to ${torrent.languages}`); logger.debug(`Updated [${storedTorrent.infoHash}] ${storedTorrent.title} language to ${torrent.languages}`);
} }
return createTorrentContents({ ...storedTorrent.get(), torrentLink: torrent.torrentLink }) return createTorrentContents({ ...storedTorrent.get(), torrentLink: torrent.torrentLink })
.then(() => updateTorrentSeeders(torrent)); .then(() => updateTorrentSeeders(torrent));
@@ -128,7 +129,7 @@ export async function createTorrentContents(torrent) {
.then(torrentContents => notOpenedVideo ? torrentContents : { ...torrentContents, videos: storedVideos }) .then(torrentContents => notOpenedVideo ? torrentContents : { ...torrentContents, videos: storedVideos })
.then(torrentContents => assignSubtitles(torrentContents)) .then(torrentContents => assignSubtitles(torrentContents))
.catch(error => { .catch(error => {
console.log(`Failed getting contents for [${torrent.infoHash}] ${torrent.title}`, error.message); logger.warn(`Failed getting contents for [${torrent.infoHash}] ${torrent.title}`, error.message);
return {}; return {};
}); });
@@ -149,14 +150,14 @@ export async function createTorrentContents(torrent) {
return repository.createTorrent({ ...torrent, contents, subtitles }) return repository.createTorrent({ ...torrent, contents, subtitles })
.then(() => { .then(() => {
if (shouldDeleteOld) { if (shouldDeleteOld) {
console.error(`Deleting old video for [${torrent.infoHash}] ${torrent.title}`) logger.debug(`Deleting old video for [${torrent.infoHash}] ${torrent.title}`)
return storedVideos[0].destroy(); return storedVideos[0].destroy();
} }
return Promise.resolve(); return Promise.resolve();
}) })
.then(() => Promises.sequence(videos.map(video => () => repository.createFile(video)))) .then(() => Promises.sequence(videos.map(video => () => repository.createFile(video))))
.then(() => console.log(`Created contents for ${torrent.provider} [${torrent.infoHash}] ${torrent.title}`)) .then(() => logger.info(`Created contents for ${torrent.provider} [${torrent.infoHash}] ${torrent.title}`))
.catch(error => console.error(`Failed saving contents for [${torrent.infoHash}] ${torrent.title}`, error)); .catch(error => logger.error(`Failed saving contents for [${torrent.infoHash}] ${torrent.title}`, error));
} }
export async function updateTorrentSeeders(torrent) { export async function updateTorrentSeeders(torrent) {
@@ -166,7 +167,7 @@ export async function updateTorrentSeeders(torrent) {
return repository.setTorrentSeeders(torrent, torrent.seeders) return repository.setTorrentSeeders(torrent, torrent.seeders)
.catch(error => { .catch(error => {
console.warn('Failed updating seeders:', error); logger.warn('Failed updating seeders:', error);
return undefined; return undefined;
}); });
} }

View File

@@ -8,26 +8,27 @@ import { getMetadata, getImdbId, getKitsuId } from './metadata.js';
import { parseSeriesVideos, isPackTorrent } from './parseHelper.js'; import { parseSeriesVideos, isPackTorrent } from './parseHelper.js';
import * as Promises from './promises.js'; import * as Promises from './promises.js';
import {torrentFiles} from "./torrent.js"; import {torrentFiles} from "./torrent.js";
import { Type } from './types.js'; import { TorrentType } from './types.js';
import {logger} from "./logger.js";
const MIN_SIZE = 5 * 1024 * 1024; // 5 MB const MIN_SIZE = 5 * 1024 * 1024; // 5 MB
const imdb_limiter = new Bottleneck({ maxConcurrent: metadataConfig.IMDB_CONCURRENT, minTime: metadataConfig.IMDB_INTERVAL_MS }); const imdb_limiter = new Bottleneck({ maxConcurrent: metadataConfig.IMDB_CONCURRENT, minTime: metadataConfig.IMDB_INTERVAL_MS });
export async function parseTorrentFiles(torrent) { export async function parseTorrentFiles(torrent) {
const parsedTorrentName = parse(torrent.title); const parsedTorrentName = parse(torrent.title);
const metadata = await getMetadata(torrent.kitsuId || torrent.imdbId, torrent.type || Type.MOVIE) const metadata = await getMetadata(torrent.kitsuId || torrent.imdbId, torrent.type || TorrentType.MOVIE)
.then(meta => Object.assign({}, meta)) .then(meta => Object.assign({}, meta))
.catch(() => undefined); .catch(() => undefined);
// if (metadata && metadata.type !== torrent.type && torrent.type !== Type.ANIME) { // if (metadata && metadata.type !== torrent.type && torrent.type !== Type.ANIME) {
// throw new Error(`Mismatching entry type for ${torrent.name}: ${torrent.type}!=${metadata.type}`); // throw new Error(`Mismatching entry type for ${torrent.name}: ${torrent.type}!=${metadata.type}`);
// } // }
if (torrent.type !== Type.ANIME && metadata && metadata.type && metadata.type !== torrent.type) { if (torrent.type !== TorrentType.ANIME && metadata && metadata.type && metadata.type !== torrent.type) {
// it's actually a movie/series // it's actually a movie/series
torrent.type = metadata.type; torrent.type = metadata.type;
} }
if (torrent.type === Type.MOVIE && (!parsedTorrentName.seasons || if (torrent.type === TorrentType.MOVIE && (!parsedTorrentName.seasons ||
parsedTorrentName.season === 5 && [1, 5].includes(parsedTorrentName.episode))) { parsedTorrentName.season === 5 && [1, 5].includes(parsedTorrentName.episode))) {
return parseMovieFiles(torrent, parsedTorrentName, metadata); return parseMovieFiles(torrent, parsedTorrentName, metadata);
} }
@@ -133,9 +134,9 @@ async function mapSeriesEpisode(file, torrent, files) {
} }
async function mapSeriesMovie(file, torrent) { async function mapSeriesMovie(file, torrent) {
const kitsuId = torrent.type === Type.ANIME ? await findMovieKitsuId(file) : undefined; const kitsuId = torrent.type === TorrentType.ANIME ? await findMovieKitsuId(file) : undefined;
const imdbId = !kitsuId ? await findMovieImdbId(file) : undefined; const imdbId = !kitsuId ? await findMovieImdbId(file) : undefined;
const metadata = await getMetadata(kitsuId || imdbId, Type.MOVIE).catch(() => ({})); const metadata = await getMetadata(kitsuId || imdbId, TorrentType.MOVIE).catch(() => ({}));
const hasEpisode = metadata.videos && metadata.videos.length && (file.episode || metadata.videos.length === 1); const hasEpisode = metadata.videos && metadata.videos.length && (file.episode || metadata.videos.length === 1);
const episodeVideo = hasEpisode && metadata.videos[(file.episode || 1) - 1]; const episodeVideo = hasEpisode && metadata.videos[(file.episode || 1) - 1];
return [{ return [{
@@ -158,7 +159,7 @@ async function decomposeEpisodes(torrent, files, metadata = { episodeCount: [] }
preprocessEpisodes(files); preprocessEpisodes(files);
if (torrent.type === Type.ANIME && torrent.kitsuId) { if (torrent.type === TorrentType.ANIME && torrent.kitsuId) {
if (needsCinemetaMetadataForAnime(files, metadata)) { if (needsCinemetaMetadataForAnime(files, metadata)) {
// In some cases anime could be resolved to wrong kitsuId // In some cases anime could be resolved to wrong kitsuId
// because of imdb season naming/absolute per series naming/multiple seasons // because of imdb season naming/absolute per series naming/multiple seasons
@@ -240,7 +241,7 @@ function isDateEpisodeFiles(files, metadata) {
function isAbsoluteEpisodeFiles(torrent, files, metadata) { function isAbsoluteEpisodeFiles(torrent, files, metadata) {
const threshold = Math.ceil(files.length / 5); const threshold = Math.ceil(files.length / 5);
const isAnime = torrent.type === Type.ANIME && torrent.kitsuId; const isAnime = torrent.type === TorrentType.ANIME && torrent.kitsuId;
const nonMovieEpisodes = files const nonMovieEpisodes = files
.filter(file => !file.isMovie && file.episodes); .filter(file => !file.isMovie && file.episodes);
const absoluteEpisodes = files const absoluteEpisodes = files
@@ -255,7 +256,7 @@ function isNewEpisodeNotInMetadata(torrent, file, metadata) {
// new episode might not yet been indexed by cinemeta. // new episode might not yet been indexed by cinemeta.
// detect this if episode number is larger than the last episode or season is larger than the last one // detect this if episode number is larger than the last episode or season is larger than the last one
// only for non anime metas // only for non anime metas
const isAnime = torrent.type === Type.ANIME && torrent.kitsuId; const isAnime = torrent.type === TorrentType.ANIME && torrent.kitsuId;
return !isAnime && !file.isMovie && file.episodes && file.season !== 1 return !isAnime && !file.isMovie && file.episodes && file.season !== 1
&& /continuing|current/i.test(metadata.status) && /continuing|current/i.test(metadata.status)
&& file.season >= metadata.episodeCount.length && file.season >= metadata.episodeCount.length
@@ -355,7 +356,7 @@ function getTimeZoneOffset(country) {
function assignKitsuOrImdbEpisodes(torrent, files, metadata) { function assignKitsuOrImdbEpisodes(torrent, files, metadata) {
if (!metadata || !metadata.videos || !metadata.videos.length) { if (!metadata || !metadata.videos || !metadata.videos.length) {
if (torrent.type === Type.ANIME) { if (torrent.type === TorrentType.ANIME) {
// assign episodes as kitsu episodes for anime when no metadata available for imdb mapping // assign episodes as kitsu episodes for anime when no metadata available for imdb mapping
files files
.filter(file => file.season && file.episodes) .filter(file => file.season && file.episodes)
@@ -364,7 +365,7 @@ function assignKitsuOrImdbEpisodes(torrent, files, metadata) {
file.season = undefined; file.season = undefined;
file.episodes = undefined; file.episodes = undefined;
}) })
if (metadata.type === Type.MOVIE && files.every(file => !file.imdbId)) { if (metadata.type === TorrentType.MOVIE && files.every(file => !file.imdbId)) {
// sometimes a movie has episode naming, thus not recognized as a movie and imdbId not assigned // sometimes a movie has episode naming, thus not recognized as a movie and imdbId not assigned
files.forEach(file => file.imdbId = metadata.imdbId); files.forEach(file => file.imdbId = metadata.imdbId);
} }
@@ -465,18 +466,18 @@ async function updateToCinemetaMetadata(metadata) {
metadata.totalCount = newMetadata.totalCount; metadata.totalCount = newMetadata.totalCount;
return metadata; return metadata;
}) })
.catch(error => console.warn(`Failed ${metadata.imdbId} metadata cinemeta update due: ${error.message}`)); .catch(error => logger.warn(`Failed ${metadata.imdbId} metadata cinemeta update due: ${error.message}`));
} }
function findMovieImdbId(title) { function findMovieImdbId(title) {
const parsedTitle = typeof title === 'string' ? parse(title) : title; const parsedTitle = typeof title === 'string' ? parse(title) : title;
console.log(`Finding movie imdbId for ${title}`); logger.debug(`Finding movie imdbId for ${title}`);
return imdb_limiter.schedule(() => getImdbId(parsedTitle, Type.MOVIE).catch(() => undefined)); return imdb_limiter.schedule(() => getImdbId(parsedTitle, TorrentType.MOVIE).catch(() => undefined));
} }
function findMovieKitsuId(title) { function findMovieKitsuId(title) {
const parsedTitle = typeof title === 'string' ? parse(title) : title; const parsedTitle = typeof title === 'string' ? parse(title) : title;
return getKitsuId(parsedTitle, Type.MOVIE).catch(() => undefined); return getKitsuId(parsedTitle, TorrentType.MOVIE).catch(() => undefined);
} }
function isDiskTorrent(contents) { function isDiskTorrent(contents) {

View File

@@ -1,6 +1,7 @@
import axios from 'axios'; import axios from 'axios';
import {cacheTrackers} from "./cache.js"; import {cacheTrackers} from "./cache.js";
import { trackerConfig } from './config.js'; import { trackerConfig } from './config.js';
import {logger} from "./logger.js";
const downloadTrackers = async () => { const downloadTrackers = async () => {
const response = await axios.get(trackerConfig.TRACKERS_URL); const response = await axios.get(trackerConfig.TRACKERS_URL);
@@ -15,7 +16,7 @@ const downloadTrackers = async () => {
urlTrackers = urlTrackers.filter(line => !line.startsWith('udp://')); urlTrackers = urlTrackers.filter(line => !line.startsWith('udp://'));
} }
console.log(`Trackers updated at ${Date.now()}: ${urlTrackers.length} trackers`); logger.info(`Trackers updated at ${Date.now()}: ${urlTrackers.length} trackers`);
return urlTrackers; return urlTrackers;
}; };