brock/torrentio
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

roll logging levels out throughout the consumer

Browse Source
This commit is contained in:
iPromKnight
2024-02-03 22:06:38 +00:00
parent 4bd76d682f
commit 8c1b6fc91b
9 changed files with 61 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
src/node/consumer/.dockerignore
View File

@@ -1,2 +1,15 @@
build.sh
node_modules/
node_modules
Dockerfile*
docker-compose*
.dockerignore
.git
.gitignore
README.md
LICENSE
.vscode
Makefile
helm-charts
.env
.editorconfig
.idea
coverage*

6
src/node/consumer/src/index.js
View File

@@ -1,13 +1,9 @@
import { listenToQueue } from './jobs/processTorrents.js';
import { jobConfig } from "./lib/config.js";
import { connect } from './lib/repository.js';
import { getTrackers } from "./lib/trackerService.js";
(async () => {
await getTrackers();
await connect();
if (jobConfig.JOBS_ENABLED) {
await listenToQueue();
}
await listenToQueue();
})();

7
src/node/consumer/src/lib/ingestedTorrent.js
View File

@@ -1,12 +1,13 @@
import { createTorrentEntry, checkAndUpdateTorrent } from './torrentEntries.js';
import {getTrackers} from "./trackerService.js";
import { Type } from './types.js';
import { TorrentType } from './types.js';
import {logger} from "./logger.js";
export async function processTorrentRecord(torrent) {
const {category} = torrent;
const type = category === 'tv' ? Type.SERIES : Type.MOVIE;
const type = category === 'tv' ? TorrentType.SERIES : TorrentType.MOVIE;
const torrentInfo = await parseTorrent(torrent, type);
console.log(`Processing torrent ${torrentInfo.title} with infoHash ${torrentInfo.infoHash}`)
logger.info(`Processing torrent ${torrentInfo.title} with infoHash ${torrentInfo.infoHash}`)
if (await checkAndUpdateTorrent(torrentInfo)) {
return torrentInfo;

8
src/node/consumer/src/lib/metadata.js
View File

@@ -2,24 +2,24 @@ import axios from 'axios';
import { search } from 'google-sr';
import nameToImdb from 'name-to-imdb';
import { cacheWrapImdbId, cacheWrapKitsuId, cacheWrapMetadata } from './cache.js';
import { Type } from './types.js';
import { TorrentType } from './types.js';
const CINEMETA_URL = 'https://v3-cinemeta.strem.io';
const KITSU_URL = 'https://anime-kitsu.strem.fun';
const TIMEOUT = 20000;
export function getMetadata(id, type = Type.SERIES) {
export function getMetadata(id, type = TorrentType.SERIES) {
if (!id) {
return Promise.reject("no valid id provided");
}
const key = Number.isInteger(id) || id.match(/^\d+$/) ? `kitsu:${id}` : id;
const metaType = type === Type.MOVIE ? Type.MOVIE : Type.SERIES;
const metaType = type === TorrentType.MOVIE ? TorrentType.MOVIE : TorrentType.SERIES;
return cacheWrapMetadata(key, () => _requestMetadata(`${KITSU_URL}/meta/${metaType}/${key}.json`)
.catch(() => _requestMetadata(`${CINEMETA_URL}/meta/${metaType}/${key}.json`))
.catch(() => {
// try different type in case there was a mismatch
const otherType = metaType === Type.MOVIE ? Type.SERIES : Type.MOVIE;
const otherType = metaType === TorrentType.MOVIE ? TorrentType.SERIES : TorrentType.MOVIE;
return _requestMetadata(`${CINEMETA_URL}/meta/${otherType}/${key}.json`)
})
.catch((error) => {

6
src/node/consumer/src/lib/parseHelper.js
View File

@@ -1,5 +1,5 @@
import { parse } from 'parse-torrent-title';
import { Type } from './types.js';
import { TorrentType } from './types.js';
const MULTIPLE_FILES_SIZE = 4 * 1024 * 1024 * 1024; // 4 GB
@@ -65,7 +65,7 @@ function isMovieVideo(video, otherVideos, type, hasMovies) {
// movie if video explicitly has numbered movie keyword in the name, ie. 1 Movie or Movie 1
return true;
}
if (!hasMovies && type !== Type.ANIME) {
if (!hasMovies && type !== TorrentType.ANIME) {
// not movie if torrent name does not contain movies keyword or is not a pack torrent and is not anime
return false;
}
@@ -85,7 +85,7 @@ export function isPackTorrent(torrent) {
return true;
}
const parsedInfo = parse(torrent.title);
if (torrent.type === Type.MOVIE) {
if (torrent.type === TorrentType.MOVIE) {
return parsedInfo.complete || typeof parsedInfo.year === 'string' || /movies/i.test(torrent.title);
}
const hasMultipleEpisodes = parsedInfo.complete ||

3
src/node/consumer/src/lib/repository.js
View File

@@ -1,6 +1,7 @@
import moment from 'moment';
import { Sequelize, Op, DataTypes, fn, col, literal } from 'sequelize';
import { databaseConfig } from './config.js';
import {logger} from "./logger.js";
import * as Promises from './promises.js';
const database = new Sequelize(
@@ -185,7 +186,7 @@ export function connect() {
if (databaseConfig.ENABLE_SYNC) {
return database.sync({ alter: true })
.catch(error => {
console.error('Failed syncing database: ', error);
logger.error('Failed syncing database: ', error);
throw error;
});
}

27
src/node/consumer/src/lib/torrentEntries.js
View File

@@ -5,12 +5,13 @@ import * as Promises from './promises.js';
import * as repository from './repository.js';
import { parseTorrentFiles } from './torrentFiles.js';
import { assignSubtitles } from './torrentSubtitles.js';
import { Type } from './types.js';
import { TorrentType } from './types.js';
import {logger} from "./logger.js";
export async function createTorrentEntry(torrent, overwrite = false) {
const titleInfo = parse(torrent.title);
if (!torrent.imdbId && torrent.type !== Type.ANIME) {
if (!torrent.imdbId && torrent.type !== TorrentType.ANIME) {
torrent.imdbId = await getImdbId(titleInfo, torrent.type)
.catch(() => undefined);
}
@@ -22,13 +23,13 @@ export async function createTorrentEntry(torrent, overwrite = false) {
// sanitize imdbId from redundant zeros
torrent.imdbId = torrent.imdbId.replace(/tt0+([0-9]{7,})$/, 'tt$1');
}
if (!torrent.kitsuId && torrent.type === Type.ANIME) {
if (!torrent.kitsuId && torrent.type === TorrentType.ANIME) {
torrent.kitsuId = await getKitsuId(titleInfo)
.catch(() => undefined);
}
if (!torrent.imdbId && !torrent.kitsuId && !isPackTorrent(torrent)) {
console.log(`imdbId or kitsuId not found: ${torrent.provider} ${torrent.title}`);
logger.warn(`imdbId or kitsuId not found: ${torrent.provider} ${torrent.title}`);
return;
}
@@ -36,17 +37,17 @@ export async function createTorrentEntry(torrent, overwrite = false) {
.then(torrentContents => overwrite ? overwriteExistingFiles(torrent, torrentContents) : torrentContents)
.then(torrentContents => assignSubtitles(torrentContents))
.catch(error => {
console.log(`Failed getting files for ${torrent.title}`, error.message);
logger.warn(`Failed getting files for ${torrent.title}`, error.message);
return {};
});
if (!videos || !videos.length) {
console.log(`no video files found for ${torrent.provider} [${torrent.infoHash}] ${torrent.title}`);
logger.warn(`no video files found for ${torrent.provider} [${torrent.infoHash}] ${torrent.title}`);
return;
}
return repository.createTorrent({ ...torrent, contents, subtitles })
.then(() => Promises.sequence(videos.map(video => () => repository.createFile(video))))
.then(() => console.log(`Created ${torrent.provider} entry for [${torrent.infoHash}] ${torrent.title}`));
.then(() => logger.info(`Created ${torrent.provider} entry for [${torrent.infoHash}] ${torrent.title}`));
}
async function overwriteExistingFiles(torrent, torrentContents) {
@@ -106,7 +107,7 @@ export async function checkAndUpdateTorrent(torrent) {
if (!storedTorrent.languages && torrent.languages && storedTorrent.provider !== 'RARBG') {
storedTorrent.languages = torrent.languages;
await storedTorrent.save();
console.log(`Updated [${storedTorrent.infoHash}] ${storedTorrent.title} language to ${torrent.languages}`);
logger.debug(`Updated [${storedTorrent.infoHash}] ${storedTorrent.title} language to ${torrent.languages}`);
}
return createTorrentContents({ ...storedTorrent.get(), torrentLink: torrent.torrentLink })
.then(() => updateTorrentSeeders(torrent));
@@ -128,7 +129,7 @@ export async function createTorrentContents(torrent) {
.then(torrentContents => notOpenedVideo ? torrentContents : { ...torrentContents, videos: storedVideos })
.then(torrentContents => assignSubtitles(torrentContents))
.catch(error => {
console.log(`Failed getting contents for [${torrent.infoHash}] ${torrent.title}`, error.message);
logger.warn(`Failed getting contents for [${torrent.infoHash}] ${torrent.title}`, error.message);
return {};
});
@@ -149,14 +150,14 @@ export async function createTorrentContents(torrent) {
return repository.createTorrent({ ...torrent, contents, subtitles })
.then(() => {
if (shouldDeleteOld) {
console.error(`Deleting old video for [${torrent.infoHash}] ${torrent.title}`)
logger.debug(`Deleting old video for [${torrent.infoHash}] ${torrent.title}`)
return storedVideos[0].destroy();
}
return Promise.resolve();
})
.then(() => Promises.sequence(videos.map(video => () => repository.createFile(video))))
.then(() => console.log(`Created contents for ${torrent.provider} [${torrent.infoHash}] ${torrent.title}`))
.catch(error => console.error(`Failed saving contents for [${torrent.infoHash}] ${torrent.title}`, error));
.then(() => logger.info(`Created contents for ${torrent.provider} [${torrent.infoHash}] ${torrent.title}`))
.catch(error => logger.error(`Failed saving contents for [${torrent.infoHash}] ${torrent.title}`, error));
}
export async function updateTorrentSeeders(torrent) {
@@ -166,7 +167,7 @@ export async function updateTorrentSeeders(torrent) {
return repository.setTorrentSeeders(torrent, torrent.seeders)
.catch(error => {
console.warn('Failed updating seeders:', error);
logger.warn('Failed updating seeders:', error);
return undefined;
});
}

31
src/node/consumer/src/lib/torrentFiles.js
View File

@@ -8,26 +8,27 @@ import { getMetadata, getImdbId, getKitsuId } from './metadata.js';
import { parseSeriesVideos, isPackTorrent } from './parseHelper.js';
import * as Promises from './promises.js';
import {torrentFiles} from "./torrent.js";
import { Type } from './types.js';
import { TorrentType } from './types.js';
import {logger} from "./logger.js";
const MIN_SIZE = 5 * 1024 * 1024; // 5 MB
const imdb_limiter = new Bottleneck({ maxConcurrent: metadataConfig.IMDB_CONCURRENT, minTime: metadataConfig.IMDB_INTERVAL_MS });
export async function parseTorrentFiles(torrent) {
const parsedTorrentName = parse(torrent.title);
const metadata = await getMetadata(torrent.kitsuId || torrent.imdbId, torrent.type || Type.MOVIE)
const metadata = await getMetadata(torrent.kitsuId || torrent.imdbId, torrent.type || TorrentType.MOVIE)
.then(meta => Object.assign({}, meta))
.catch(() => undefined);
// if (metadata && metadata.type !== torrent.type && torrent.type !== Type.ANIME) {
// throw new Error(`Mismatching entry type for ${torrent.name}: ${torrent.type}!=${metadata.type}`);
// }
if (torrent.type !== Type.ANIME && metadata && metadata.type && metadata.type !== torrent.type) {
if (torrent.type !== TorrentType.ANIME && metadata && metadata.type && metadata.type !== torrent.type) {
// it's actually a movie/series
torrent.type = metadata.type;
}
if (torrent.type === Type.MOVIE && (!parsedTorrentName.seasons ||
if (torrent.type === TorrentType.MOVIE && (!parsedTorrentName.seasons ||
parsedTorrentName.season === 5 && [1, 5].includes(parsedTorrentName.episode))) {
return parseMovieFiles(torrent, parsedTorrentName, metadata);
}
@@ -133,9 +134,9 @@ async function mapSeriesEpisode(file, torrent, files) {
}
async function mapSeriesMovie(file, torrent) {
const kitsuId = torrent.type === Type.ANIME ? await findMovieKitsuId(file) : undefined;
const kitsuId = torrent.type === TorrentType.ANIME ? await findMovieKitsuId(file) : undefined;
const imdbId = !kitsuId ? await findMovieImdbId(file) : undefined;
const metadata = await getMetadata(kitsuId || imdbId, Type.MOVIE).catch(() => ({}));
const metadata = await getMetadata(kitsuId || imdbId, TorrentType.MOVIE).catch(() => ({}));
const hasEpisode = metadata.videos && metadata.videos.length && (file.episode || metadata.videos.length === 1);
const episodeVideo = hasEpisode && metadata.videos[(file.episode || 1) - 1];
return [{
@@ -158,7 +159,7 @@ async function decomposeEpisodes(torrent, files, metadata = { episodeCount: [] }
preprocessEpisodes(files);
if (torrent.type === Type.ANIME && torrent.kitsuId) {
if (torrent.type === TorrentType.ANIME && torrent.kitsuId) {
if (needsCinemetaMetadataForAnime(files, metadata)) {
// In some cases anime could be resolved to wrong kitsuId
// because of imdb season naming/absolute per series naming/multiple seasons
@@ -240,7 +241,7 @@ function isDateEpisodeFiles(files, metadata) {
function isAbsoluteEpisodeFiles(torrent, files, metadata) {
const threshold = Math.ceil(files.length / 5);
const isAnime = torrent.type === Type.ANIME && torrent.kitsuId;
const isAnime = torrent.type === TorrentType.ANIME && torrent.kitsuId;
const nonMovieEpisodes = files
.filter(file => !file.isMovie && file.episodes);
const absoluteEpisodes = files
@@ -255,7 +256,7 @@ function isNewEpisodeNotInMetadata(torrent, file, metadata) {
// new episode might not yet been indexed by cinemeta.
// detect this if episode number is larger than the last episode or season is larger than the last one
// only for non anime metas
const isAnime = torrent.type === Type.ANIME && torrent.kitsuId;
const isAnime = torrent.type === TorrentType.ANIME && torrent.kitsuId;
return !isAnime && !file.isMovie && file.episodes && file.season !== 1
&& /continuing|current/i.test(metadata.status)
&& file.season >= metadata.episodeCount.length
@@ -355,7 +356,7 @@ function getTimeZoneOffset(country) {
function assignKitsuOrImdbEpisodes(torrent, files, metadata) {
if (!metadata || !metadata.videos || !metadata.videos.length) {
if (torrent.type === Type.ANIME) {
if (torrent.type === TorrentType.ANIME) {
// assign episodes as kitsu episodes for anime when no metadata available for imdb mapping
files
.filter(file => file.season && file.episodes)
@@ -364,7 +365,7 @@ function assignKitsuOrImdbEpisodes(torrent, files, metadata) {
file.season = undefined;
file.episodes = undefined;
})
if (metadata.type === Type.MOVIE && files.every(file => !file.imdbId)) {
if (metadata.type === TorrentType.MOVIE && files.every(file => !file.imdbId)) {
// sometimes a movie has episode naming, thus not recognized as a movie and imdbId not assigned
files.forEach(file => file.imdbId = metadata.imdbId);
}
@@ -465,18 +466,18 @@ async function updateToCinemetaMetadata(metadata) {
metadata.totalCount = newMetadata.totalCount;
return metadata;
})
.catch(error => console.warn(`Failed ${metadata.imdbId} metadata cinemeta update due: ${error.message}`));
.catch(error => logger.warn(`Failed ${metadata.imdbId} metadata cinemeta update due: ${error.message}`));
}
function findMovieImdbId(title) {
const parsedTitle = typeof title === 'string' ? parse(title) : title;
console.log(`Finding movie imdbId for ${title}`);
return imdb_limiter.schedule(() => getImdbId(parsedTitle, Type.MOVIE).catch(() => undefined));
logger.debug(`Finding movie imdbId for ${title}`);
return imdb_limiter.schedule(() => getImdbId(parsedTitle, TorrentType.MOVIE).catch(() => undefined));
}
function findMovieKitsuId(title) {
const parsedTitle = typeof title === 'string' ? parse(title) : title;
return getKitsuId(parsedTitle, Type.MOVIE).catch(() => undefined);
return getKitsuId(parsedTitle, TorrentType.MOVIE).catch(() => undefined);
}
function isDiskTorrent(contents) {

3
src/node/consumer/src/lib/trackerService.js
View File

@@ -1,6 +1,7 @@
import axios from 'axios';
import {cacheTrackers} from "./cache.js";
import { trackerConfig } from './config.js';
import {logger} from "./logger.js";
const downloadTrackers = async () => {
const response = await axios.get(trackerConfig.TRACKERS_URL);
@@ -15,7 +16,7 @@ const downloadTrackers = async () => {
urlTrackers = urlTrackers.filter(line => !line.startsWith('udp://'));
}
console.log(`Trackers updated at ${Date.now()}: ${urlTrackers.length} trackers`);
logger.info(`Trackers updated at ${Date.now()}: ${urlTrackers.length} trackers`);
return urlTrackers;
};