Merge pull request #69 from iPromKnight/ts-repo
This commit is contained in:
iPromKnight
2
.gitignore
vendored
2
.gitignore
vendored
@@ -403,4 +403,4 @@ FodyWeavers.xsd
|
||||
# JetBrains Rider
|
||||
*.sln.iml
|
||||
|
||||
dist/
|
||||
dist/
|
||||
@@ -23,7 +23,9 @@ RABBIT_URI=amqp://guest:guest@rabbitmq:5672/?heartbeat=30
|
||||
QUEUE_NAME=ingested
|
||||
JOB_CONCURRENCY=5
|
||||
JOBS_ENABLED=true
|
||||
MAX_SINGLE_TORRENT_CONNECTIONS=10
|
||||
LOG_LEVEL=info # can be debug for extra verbosity (a lot more verbosity - useful for development)
|
||||
MAX_CONNECTIONS_PER_TORRENT=10
|
||||
MAX_CONNECTIONS_OVERALL=100
|
||||
TORRENT_TIMEOUT=30000
|
||||
UDP_TRACKERS_ENABLED=true
|
||||
CONSUMER_REPLICAS=3
|
||||
|
||||
@@ -1 +1,3 @@
|
||||
*.ts
|
||||
dist/
|
||||
esbuild.ts
|
||||
jest.config.ts
|
||||
84
src/node/consumer/.eslintrc
Normal file
84
src/node/consumer/.eslintrc
Normal file
@@ -0,0 +1,84 @@
|
||||
{
|
||||
"root": true,
|
||||
"extends": [
|
||||
"eslint:recommended",
|
||||
"plugin:@typescript-eslint/recommended"
|
||||
],
|
||||
"parser": "@typescript-eslint/parser",
|
||||
"parserOptions": {
|
||||
"project": [
|
||||
"./tsconfig.json"
|
||||
]
|
||||
},
|
||||
"plugins": [
|
||||
"@typescript-eslint",
|
||||
"import",
|
||||
"import-helpers"
|
||||
],
|
||||
"rules": {
|
||||
"default-case": "off",
|
||||
"import/no-duplicates": "error",
|
||||
"import/no-extraneous-dependencies": "error",
|
||||
"import/order": "off",
|
||||
"import-helpers/order-imports": [
|
||||
"warn",
|
||||
{
|
||||
"alphabetize": {
|
||||
"order": "asc"
|
||||
}
|
||||
}
|
||||
],
|
||||
"lines-between-class-members": [
|
||||
"error",
|
||||
"always",
|
||||
{
|
||||
"exceptAfterSingleLine": true
|
||||
}
|
||||
],
|
||||
"no-continue": "off",
|
||||
"no-param-reassign": "off",
|
||||
"no-plusplus": [
|
||||
"error",
|
||||
{
|
||||
"allowForLoopAfterthoughts": true
|
||||
}
|
||||
],
|
||||
"no-restricted-syntax": "off",
|
||||
"no-unused-expressions": [
|
||||
"off",
|
||||
{
|
||||
"allowShortCircuit": true
|
||||
}
|
||||
],
|
||||
"no-unused-vars": "off",
|
||||
"no-use-before-define": "off",
|
||||
"one-var": [
|
||||
"error",
|
||||
{
|
||||
"uninitialized": "consecutive"
|
||||
}
|
||||
],
|
||||
"prefer-destructuring": "error",
|
||||
"@typescript-eslint/explicit-function-return-type": "error",
|
||||
"@typescript-eslint/consistent-type-assertions": [
|
||||
"error",
|
||||
{
|
||||
"assertionStyle": "as",
|
||||
"objectLiteralTypeAssertions": "never"
|
||||
}
|
||||
]
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
"files": [
|
||||
"*.test.ts"
|
||||
],
|
||||
"rules": {
|
||||
"@typescript-eslint/consistent-type-assertions": "off",
|
||||
"@typescript-eslint/explicit-function-return-type": "off",
|
||||
"@typescript-eslint/no-explicit-any": "off",
|
||||
"@typescript-eslint/no-unused-vars": "off"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
/** @type {import("eslint").ESLint.Options} */
|
||||
module.exports = {
|
||||
env: {
|
||||
es2024: true,
|
||||
node: true,
|
||||
},
|
||||
globals: {
|
||||
Atomics: "readonly",
|
||||
SharedArrayBuffer: "readonly",
|
||||
},
|
||||
parserOptions: {
|
||||
sourceType: "module",
|
||||
},
|
||||
plugins: ["import", "import-helpers"],
|
||||
rules: {
|
||||
"default-case": "off",
|
||||
"import/no-duplicates": "off",
|
||||
"import/no-extraneous-dependencies": ["off", { devDependencies: ["backend", "frontend", "mobile"] }],
|
||||
"import/order": "off",
|
||||
"import-helpers/order-imports": [
|
||||
"warn",
|
||||
{
|
||||
alphabetize: {
|
||||
order: "asc",
|
||||
},
|
||||
},
|
||||
],
|
||||
"lines-between-class-members": ["error", "always", { exceptAfterSingleLine: true }],
|
||||
"no-continue": "off",
|
||||
"no-param-reassign": "off",
|
||||
"no-plusplus": ["error", { allowForLoopAfterthoughts: true }],
|
||||
"no-restricted-syntax": "off",
|
||||
"no-unused-expressions": ["off", { allowShortCircuit: true }],
|
||||
"no-unused-vars": "off",
|
||||
"no-use-before-define": "off",
|
||||
"one-var": ["error", { uninitialized: "consecutive" }],
|
||||
"prefer-destructuring": "off",
|
||||
},
|
||||
};
|
||||
1
src/node/consumer/.nvmrc
Normal file
1
src/node/consumer/.nvmrc
Normal file
@@ -0,0 +1 @@
|
||||
v20.10.0
|
||||
@@ -1,8 +1,6 @@
|
||||
FROM node:lts-buster-slim as builder
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y git && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
RUN apt update && apt install -y git && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
@@ -10,16 +8,16 @@ COPY package*.json ./
|
||||
RUN npm install
|
||||
COPY . .
|
||||
RUN npm run build
|
||||
RUN npm prune --omit=dev
|
||||
|
||||
# --- Runtime Stage ---
|
||||
FROM node:lts-buster-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
ENV NODE_ENV production
|
||||
|
||||
COPY --from=builder /app ./
|
||||
RUN npm prune --omit=dev
|
||||
|
||||
ENV NODE_ENV production
|
||||
ENV NODE_OPTIONS "--no-deprecation"
|
||||
|
||||
# CIS-DI-0001
|
||||
RUN useradd -d /home/consumer -m -s /bin/bash consumer
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
import { build } from "esbuild";
|
||||
import { readFileSync, rmSync } from "fs";
|
||||
|
||||
const { devDependencies } = JSON.parse(readFileSync("./package.json", "utf8"));
|
||||
|
||||
const start = Date.now();
|
||||
|
||||
try {
|
||||
@@ -13,9 +11,8 @@ try {
|
||||
build({
|
||||
bundle: true,
|
||||
entryPoints: [
|
||||
"./src/index.js",
|
||||
"./src/main.ts",
|
||||
],
|
||||
external: [...(devDependencies && Object.keys(devDependencies))],
|
||||
keepNames: true,
|
||||
minify: true,
|
||||
outbase: "./src",
|
||||
@@ -42,7 +39,6 @@ try {
|
||||
}
|
||||
],
|
||||
}).then(() => {
|
||||
// biome-ignore lint/style/useTemplate: <explanation>
|
||||
console.log("⚡ " + "\x1b[32m" + `Done in ${Date.now() - start}ms`);
|
||||
});
|
||||
} catch (e) {
|
||||
14
src/node/consumer/jest.config.ts
Normal file
14
src/node/consumer/jest.config.ts
Normal file
@@ -0,0 +1,14 @@
|
||||
import { pathsToModuleNameMapper } from 'ts-jest';
|
||||
import { compilerOptions } from './tsconfig.json';
|
||||
|
||||
export default {
|
||||
preset: 'ts-jest',
|
||||
testEnvironment: 'node',
|
||||
moduleNameMapper: pathsToModuleNameMapper(compilerOptions.paths, { prefix: '<rootDir>/src/' }),
|
||||
modulePaths: [
|
||||
'<rootDir>'
|
||||
],
|
||||
transform: {
|
||||
'^.+\\.tsx?$': 'ts-jest',
|
||||
},
|
||||
};
|
||||
@@ -1,21 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"baseUrl": "./src",
|
||||
"checkJs": true,
|
||||
"isolatedModules": true,
|
||||
"lib": ["es6"],
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "node",
|
||||
"outDir": "./dist",
|
||||
"pretty": true,
|
||||
"removeComments": true,
|
||||
"resolveJsonModule": true,
|
||||
"rootDir": "./src",
|
||||
"skipLibCheck": true,
|
||||
"sourceMap": true,
|
||||
"target": "ES6",
|
||||
"types": ["node"],
|
||||
"typeRoots": ["node_modules/@types", "src/@types"]
|
||||
},
|
||||
"exclude": ["node_modules"]
|
||||
}
|
||||
6389
src/node/consumer/package-lock.json
generated
6389
src/node/consumer/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@@ -3,10 +3,14 @@
|
||||
"version": "0.0.1",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"build": "node esbuild.js",
|
||||
"dev": "tsx watch --ignore node_modules src/index.js | pino-pretty",
|
||||
"start": "node dist/index.cjs",
|
||||
"lint": "eslint . --ext .ts,.js"
|
||||
"clean": "rm -rf dist",
|
||||
"build": "tsx esbuild.ts",
|
||||
"dev": "tsx watch --ignore node_modules src/main.ts | pino-pretty",
|
||||
"start": "node dist/main.cjs",
|
||||
"lint": "eslint ./src --ext .ts,.js",
|
||||
"lint-fix": "npm run lint -- --fix",
|
||||
"test": "jest",
|
||||
"test:watch": "jest --watch"
|
||||
},
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
@@ -16,7 +20,7 @@
|
||||
"bottleneck": "^2.19.5",
|
||||
"cache-manager": "^5.4.0",
|
||||
"google-sr": "^3.2.1",
|
||||
"jaro-winkler": "^0.2.8",
|
||||
"inversify": "^6.0.2",
|
||||
"magnet-uri": "^6.2.0",
|
||||
"moment": "^2.30.1",
|
||||
"name-to-imdb": "^3.0.4",
|
||||
@@ -24,18 +28,32 @@
|
||||
"pg": "^8.11.3",
|
||||
"pg-hstore": "^2.3.4",
|
||||
"pino": "^8.18.0",
|
||||
"sequelize": "^6.31.1",
|
||||
"torrent-stream": "^1.2.1",
|
||||
"user-agents": "^1.0.1444"
|
||||
"reflect-metadata": "^0.2.1",
|
||||
"sequelize": "^6.36.0",
|
||||
"sequelize-typescript": "^2.1.6",
|
||||
"torrent-stream": "^1.2.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.11.6",
|
||||
"@types/stremio-addon-sdk": "^1.6.10",
|
||||
"@types/amqplib": "^0.10.4",
|
||||
"@types/jest": "^29.5.12",
|
||||
"@types/magnet-uri": "^5.1.5",
|
||||
"@types/node": "^20.11.16",
|
||||
"@types/pg": "^8.11.0",
|
||||
"@types/torrent-stream": "^0.0.9",
|
||||
"@types/validator": "^13.11.8",
|
||||
"@typescript-eslint/eslint-plugin": "^6.21.0",
|
||||
"@typescript-eslint/parser": "^6.21.0",
|
||||
"esbuild": "^0.20.0",
|
||||
"eslint": "^8.56.0",
|
||||
"eslint-plugin-import": "^2.29.1",
|
||||
"eslint-plugin-import-helpers": "^1.3.1",
|
||||
"jest": "^29.7.0",
|
||||
"msw": "^2.1.7",
|
||||
"pino-pretty": "^10.3.1",
|
||||
"ts-jest": "^29.1.2",
|
||||
"ts-node": "^10.9.2",
|
||||
"tsconfig-paths": "^4.2.0",
|
||||
"tsx": "^4.7.0",
|
||||
"pino-pretty": "^10.3.1"
|
||||
"typescript": "^5.3.3"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,9 +0,0 @@
|
||||
import { listenToQueue } from './jobs/processTorrents.js';
|
||||
import { connect } from './lib/repository.js';
|
||||
import { getTrackers } from "./lib/trackerService.js";
|
||||
|
||||
(async () => {
|
||||
await getTrackers();
|
||||
await connect();
|
||||
await listenToQueue();
|
||||
})();
|
||||
@@ -1,37 +0,0 @@
|
||||
import amqp from 'amqplib'
|
||||
import { rabbitConfig, jobConfig } from '../lib/config.js'
|
||||
import { processTorrentRecord } from "../lib/ingestedTorrent.js";
|
||||
import {logger} from "../lib/logger.js";
|
||||
|
||||
const assertQueueOptions = { durable: true }
|
||||
const consumeQueueOptions = { noAck: false }
|
||||
|
||||
const processMessage = msg => processTorrentRecord(getMessageAsJson(msg));
|
||||
|
||||
const getMessageAsJson = msg =>
|
||||
JSON.parse(msg.content.toString()).message;
|
||||
|
||||
const assertAndConsumeQueue = async channel => {
|
||||
logger.info('Worker is running! Waiting for new torrents...')
|
||||
|
||||
const ackMsg = msg =>
|
||||
processMessage(msg)
|
||||
.then(() => channel.ack(msg))
|
||||
.catch(error => logger.error('Failed processing torrent', error));
|
||||
|
||||
channel.assertQueue(rabbitConfig.QUEUE_NAME, assertQueueOptions)
|
||||
.then(() => channel.prefetch(jobConfig.JOB_CONCURRENCY))
|
||||
.then(() => channel.consume(rabbitConfig.QUEUE_NAME, ackMsg, consumeQueueOptions))
|
||||
.catch(error => logger.error('Failed to setup channel', error));
|
||||
}
|
||||
|
||||
export const listenToQueue = async () => {
|
||||
if (!jobConfig.JOBS_ENABLED) {
|
||||
return;
|
||||
}
|
||||
|
||||
return amqp.connect(rabbitConfig.URI)
|
||||
.then(connection => connection.createChannel())
|
||||
.then(channel => assertAndConsumeQueue(channel))
|
||||
.catch(error => logger.error('Failed to connect and setup channel', error));
|
||||
};
|
||||
@@ -1,84 +0,0 @@
|
||||
import { createCache, memoryStore} from 'cache-manager';
|
||||
import { mongoDbStore } from '@tirke/node-cache-manager-mongodb'
|
||||
import { cacheConfig } from './config.js';
|
||||
import { logger } from './logger.js';
|
||||
import { CacheType } from "./types.js";
|
||||
|
||||
const GLOBAL_KEY_PREFIX = 'knightcrawler-consumer';
|
||||
const IMDB_ID_PREFIX = `${GLOBAL_KEY_PREFIX}|imdb_id`;
|
||||
const KITSU_ID_PREFIX = `${GLOBAL_KEY_PREFIX}|kitsu_id`;
|
||||
const METADATA_PREFIX = `${GLOBAL_KEY_PREFIX}|metadata`;
|
||||
const TRACKERS_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|trackers`;
|
||||
|
||||
const GLOBAL_TTL = process.env.METADATA_TTL || 7 * 24 * 60 * 60; // 7 days
|
||||
const MEMORY_TTL = process.env.METADATA_TTL || 2 * 60 * 60; // 2 hours
|
||||
const TRACKERS_TTL = 2 * 24 * 60 * 60; // 2 days
|
||||
|
||||
const initiateMemoryCache = () =>
|
||||
createCache(memoryStore(), {
|
||||
ttl: parseInt(MEMORY_TTL)
|
||||
});
|
||||
|
||||
const initiateMongoCache = () => {
|
||||
const store = mongoDbStore({
|
||||
collectionName: cacheConfig.COLLECTION_NAME,
|
||||
ttl: parseInt(GLOBAL_TTL),
|
||||
url: cacheConfig.MONGO_URI,
|
||||
mongoConfig:{
|
||||
socketTimeoutMS: 120000,
|
||||
appName: 'knightcrawler-consumer',
|
||||
}
|
||||
});
|
||||
|
||||
return createCache(store, {
|
||||
ttl: parseInt(GLOBAL_TTL),
|
||||
});
|
||||
}
|
||||
|
||||
const initiateRemoteCache = ()=> {
|
||||
if (cacheConfig.NO_CACHE) {
|
||||
logger.debug('Cache is disabled');
|
||||
return null;
|
||||
}
|
||||
return cacheConfig.MONGO_URI ? initiateMongoCache() : initiateMemoryCache();
|
||||
}
|
||||
|
||||
const getCacheType = (cacheType) => {
|
||||
switch (cacheType) {
|
||||
case CacheType.MEMORY:
|
||||
return memoryCache;
|
||||
case CacheType.MONGODB:
|
||||
return remoteCache;
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
const memoryCache = initiateMemoryCache()
|
||||
const remoteCache = initiateRemoteCache()
|
||||
|
||||
const cacheWrap = async (cacheType, key, method, options) => {
|
||||
const cache = getCacheType(cacheType);
|
||||
|
||||
if (cacheConfig.NO_CACHE || !cache) {
|
||||
return method();
|
||||
}
|
||||
|
||||
logger.debug(`Cache type: ${cacheType}`);
|
||||
logger.debug(`Cache key: ${key}`);
|
||||
logger.debug(`Cache options: ${JSON.stringify(options)}`);
|
||||
|
||||
return cache.wrap(key, method, options.ttl);
|
||||
}
|
||||
|
||||
export const cacheWrapImdbId = (key, method) =>
|
||||
cacheWrap(CacheType.MONGODB, `${IMDB_ID_PREFIX}:${key}`, method, { ttl: parseInt(GLOBAL_TTL) });
|
||||
|
||||
export const cacheWrapKitsuId = (key, method) =>
|
||||
cacheWrap(CacheType.MONGODB, `${KITSU_ID_PREFIX}:${key}`, method, { ttl: parseInt(GLOBAL_TTL) });
|
||||
|
||||
export const cacheWrapMetadata = (id, method) =>
|
||||
cacheWrap(CacheType.MEMORY, `${METADATA_PREFIX}:${id}`, method, { ttl: parseInt(MEMORY_TTL) });
|
||||
|
||||
export const cacheTrackers = (method) =>
|
||||
cacheWrap(CacheType.MEMORY, `${TRACKERS_KEY_PREFIX}`, method, { ttl: parseInt(TRACKERS_TTL) });
|
||||
@@ -1,63 +0,0 @@
|
||||
export const rabbitConfig = {
|
||||
URI: process.env.RABBIT_URI || 'amqp://localhost',
|
||||
QUEUE_NAME: process.env.QUEUE_NAME || 'test-queue'
|
||||
}
|
||||
|
||||
export const cacheConfig = {
|
||||
MONGODB_HOST: process.env.MONGODB_HOST || 'mongodb',
|
||||
MONGODB_PORT: process.env.MONGODB_PORT || '27017',
|
||||
MONGODB_DB: process.env.MONGODB_DB || 'knightcrawler',
|
||||
MONGO_INITDB_ROOT_USERNAME: process.env.MONGO_INITDB_ROOT_USERNAME || 'mongo',
|
||||
MONGO_INITDB_ROOT_PASSWORD: process.env.MONGO_INITDB_ROOT_PASSWORD || 'mongo',
|
||||
NO_CACHE: parseBool(process.env.NO_CACHE, false),
|
||||
COLLECTION_NAME: process.env.MONGODB_COLLECTION || 'knightcrawler_consumer_collection'
|
||||
}
|
||||
|
||||
// Combine the environment variables into a connection string
|
||||
// The combined string will look something like:
|
||||
// 'mongodb://mongo:mongo@localhost:27017/knightcrawler?authSource=admin'
|
||||
cacheConfig.MONGO_URI = 'mongodb://' + cacheConfig.MONGO_INITDB_ROOT_USERNAME + ':' + cacheConfig.MONGO_INITDB_ROOT_PASSWORD + '@' + cacheConfig.MONGODB_HOST + ':' + cacheConfig.MONGODB_PORT + '/' + cacheConfig.MONGODB_DB + '?authSource=admin';
|
||||
|
||||
export const databaseConfig = {
|
||||
POSTGRES_HOST: process.env.POSTGRES_HOST || 'postgres',
|
||||
POSTGRES_PORT: process.env.POSTGRES_PORT || '5432',
|
||||
POSTGRES_DB: process.env.POSTGRES_DB || 'knightcrawler',
|
||||
POSTGRES_USER: process.env.POSTGRES_USER || 'postgres',
|
||||
POSTGRES_PASSWORD: process.env.POSTGRES_PASSWORD || 'postgres',
|
||||
AUTO_CREATE_AND_APPLY_MIGRATIONS: parseBool(process.env.AUTO_CREATE_AND_APPLY_MIGRATIONS, false)
|
||||
}
|
||||
|
||||
// Combine the environment variables into a connection string
|
||||
// The combined string will look something like:
|
||||
// 'postgres://postgres:postgres@localhost:5432/knightcrawler'
|
||||
databaseConfig.POSTGRES_URI = 'postgres://' + databaseConfig.POSTGRES_USER + ':' + databaseConfig.POSTGRES_PASSWORD + '@' + databaseConfig.POSTGRES_HOST + ':' + databaseConfig.POSTGRES_PORT + '/' + databaseConfig.POSTGRES_DB;
|
||||
|
||||
export const jobConfig = {
|
||||
JOB_CONCURRENCY: parseInt(process.env.JOB_CONCURRENCY || 1),
|
||||
JOBS_ENABLED: parseBool(process.env.JOBS_ENABLED || true)
|
||||
}
|
||||
|
||||
export const metadataConfig = {
|
||||
IMDB_CONCURRENT: parseInt(process.env.IMDB_CONCURRENT || 1),
|
||||
IMDB_INTERVAL_MS: parseInt(process.env.IMDB_INTERVAL_MS || 1000),
|
||||
}
|
||||
|
||||
export const trackerConfig = {
|
||||
TRACKERS_URL: process.env.TRACKERS_URL || 'https://ngosang.github.io/trackerslist/trackers_all.txt',
|
||||
UDP_ENABLED: parseBool(process.env.UDP_TRACKERS_ENABLED || false),
|
||||
}
|
||||
|
||||
export const torrentConfig = {
|
||||
MAX_CONNECTIONS_PER_TORRENT: parseInt(process.env.MAX_SINGLE_TORRENT_CONNECTIONS || 20),
|
||||
TIMEOUT: parseInt(process.env.TORRENT_TIMEOUT || 30000),
|
||||
}
|
||||
|
||||
function parseBool(boolString, defaultValue) {
|
||||
const isString = typeof boolString === 'string' || boolString instanceof String;
|
||||
|
||||
if (!isString) {
|
||||
return defaultValue;
|
||||
}
|
||||
|
||||
return boolString.toLowerCase() === 'true' ? true : defaultValue;
|
||||
}
|
||||
4
src/node/consumer/src/lib/enums/cache_types.ts
Normal file
4
src/node/consumer/src/lib/enums/cache_types.ts
Normal file
@@ -0,0 +1,4 @@
|
||||
export enum CacheType {
|
||||
Memory = 'memory',
|
||||
MongoDb = 'mongodb'
|
||||
}
|
||||
5
src/node/consumer/src/lib/enums/torrent_types.ts
Normal file
5
src/node/consumer/src/lib/enums/torrent_types.ts
Normal file
@@ -0,0 +1,5 @@
|
||||
export enum TorrentType {
|
||||
Series = 'Series',
|
||||
Movie = 'Movie',
|
||||
Anime = 'anime',
|
||||
}
|
||||
@@ -1,62 +0,0 @@
|
||||
const VIDEO_EXTENSIONS = [
|
||||
"3g2",
|
||||
"3gp",
|
||||
"avi",
|
||||
"flv",
|
||||
"mkv",
|
||||
"mk3d",
|
||||
"mov",
|
||||
"mp2",
|
||||
"mp4",
|
||||
"m4v",
|
||||
"mpe",
|
||||
"mpeg",
|
||||
"mpg",
|
||||
"mpv",
|
||||
"webm",
|
||||
"wmv",
|
||||
"ogm",
|
||||
"divx"
|
||||
];
|
||||
const SUBTITLE_EXTENSIONS = [
|
||||
"aqt",
|
||||
"gsub",
|
||||
"jss",
|
||||
"sub",
|
||||
"ttxt",
|
||||
"pjs",
|
||||
"psb",
|
||||
"rt",
|
||||
"smi",
|
||||
"slt",
|
||||
"ssf",
|
||||
"srt",
|
||||
"ssa",
|
||||
"ass",
|
||||
"usf",
|
||||
"idx",
|
||||
"vtt"
|
||||
];
|
||||
const DISK_EXTENSIONS = [
|
||||
"iso",
|
||||
"m2ts",
|
||||
"ts",
|
||||
"vob"
|
||||
]
|
||||
|
||||
export function isVideo(filename) {
|
||||
return isExtension(filename, VIDEO_EXTENSIONS);
|
||||
}
|
||||
|
||||
export function isSubtitle(filename) {
|
||||
return isExtension(filename, SUBTITLE_EXTENSIONS);
|
||||
}
|
||||
|
||||
export function isDisk(filename) {
|
||||
return isExtension(filename, DISK_EXTENSIONS);
|
||||
}
|
||||
|
||||
export function isExtension(filename, extensions) {
|
||||
const extensionMatch = filename.match(/\.(\w{2,4})$/);
|
||||
return extensionMatch && extensions.includes(extensionMatch[1].toLowerCase());
|
||||
}
|
||||
18
src/node/consumer/src/lib/helpers/boolean_helpers.ts
Normal file
18
src/node/consumer/src/lib/helpers/boolean_helpers.ts
Normal file
@@ -0,0 +1,18 @@
|
||||
export const BooleanHelpers = {
|
||||
parseBool: (value: string | undefined, defaultValue: boolean): boolean => {
|
||||
switch (value?.trim().toLowerCase()) {
|
||||
case undefined:
|
||||
return defaultValue;
|
||||
case 'true':
|
||||
case 'yes':
|
||||
case '1':
|
||||
return true;
|
||||
case 'false':
|
||||
case 'no':
|
||||
case '0':
|
||||
return false;
|
||||
default:
|
||||
throw new Error(`Invalid boolean value: '${value}'. Allowed values are 'true', 'false', 'yes', 'no', '1', or '0'.`);
|
||||
}
|
||||
}
|
||||
}
|
||||
66
src/node/consumer/src/lib/helpers/extension_helpers.ts
Normal file
66
src/node/consumer/src/lib/helpers/extension_helpers.ts
Normal file
@@ -0,0 +1,66 @@
|
||||
const VIDEO_EXTENSIONS = [
|
||||
"3g2",
|
||||
"3gp",
|
||||
"avi",
|
||||
"flv",
|
||||
"mkv",
|
||||
"mk3d",
|
||||
"mov",
|
||||
"mp2",
|
||||
"mp4",
|
||||
"m4v",
|
||||
"mpe",
|
||||
"mpeg",
|
||||
"mpg",
|
||||
"mpv",
|
||||
"webm",
|
||||
"wmv",
|
||||
"ogm",
|
||||
"divx"
|
||||
];
|
||||
|
||||
const SUBTITLE_EXTENSIONS = [
|
||||
"aqt",
|
||||
"gsub",
|
||||
"jss",
|
||||
"sub",
|
||||
"ttxt",
|
||||
"pjs",
|
||||
"psb",
|
||||
"rt",
|
||||
"smi",
|
||||
"slt",
|
||||
"ssf",
|
||||
"srt",
|
||||
"ssa",
|
||||
"ass",
|
||||
"usf",
|
||||
"idx",
|
||||
"vtt"
|
||||
];
|
||||
|
||||
const DISK_EXTENSIONS = [
|
||||
"iso",
|
||||
"m2ts",
|
||||
"ts",
|
||||
"vob"
|
||||
];
|
||||
|
||||
export const ExtensionHelpers = {
|
||||
isVideo(filename: string): boolean {
|
||||
return this.isExtension(filename, VIDEO_EXTENSIONS);
|
||||
},
|
||||
|
||||
isSubtitle(filename: string): boolean {
|
||||
return this.isExtension(filename, SUBTITLE_EXTENSIONS);
|
||||
},
|
||||
|
||||
isDisk(filename: string): boolean {
|
||||
return this.isExtension(filename, DISK_EXTENSIONS);
|
||||
},
|
||||
|
||||
isExtension(filename: string, extensions: string[]): boolean {
|
||||
const extensionMatch = filename.match(/\.(\w{2,4})$/);
|
||||
return extensionMatch !== null && extensions.includes(extensionMatch[1].toLowerCase());
|
||||
}
|
||||
}
|
||||
37
src/node/consumer/src/lib/helpers/promises_helpers.ts
Normal file
37
src/node/consumer/src/lib/helpers/promises_helpers.ts
Normal file
@@ -0,0 +1,37 @@
|
||||
/* eslint-disable @typescript-eslint/no-explicit-any */
|
||||
export const PromiseHelpers = {
|
||||
|
||||
sequence: async function (promises: (() => Promise<any>)[]): Promise<any> {
|
||||
return promises.reduce((promise, func) =>
|
||||
promise.then(result => func().then(res => result.concat(res))), Promise.resolve([]));
|
||||
},
|
||||
|
||||
first: async function (promises: any): Promise<any> {
|
||||
return Promise.all(promises.map((p: any) => {
|
||||
return p.then((val: any) => Promise.reject(val), (err: any) => Promise.resolve(err));
|
||||
})).then(
|
||||
(errors) => Promise.reject(errors),
|
||||
(val) => Promise.resolve(val)
|
||||
);
|
||||
},
|
||||
|
||||
delay: async function (duration: number): Promise<void> {
|
||||
return new Promise<void>(resolve => setTimeout(() => resolve(), duration));
|
||||
},
|
||||
|
||||
timeout: async function (timeoutMs: number, promise: any, message = 'Timed out'): Promise<any> {
|
||||
return Promise.race([
|
||||
promise,
|
||||
new Promise(function (resolve, reject) {
|
||||
setTimeout(function () {
|
||||
reject(message);
|
||||
}, timeoutMs);
|
||||
})
|
||||
]);
|
||||
},
|
||||
|
||||
mostCommonValue: function (array: any[]): any {
|
||||
return array.sort((a, b) => array.filter(v => v === a).length - array.filter(v => v === b).length).pop();
|
||||
}
|
||||
};
|
||||
/* eslint-enable @typescript-eslint/no-explicit-any */
|
||||
@@ -1,46 +0,0 @@
|
||||
import { createTorrentEntry, checkAndUpdateTorrent } from './torrentEntries.js';
|
||||
import {getTrackers} from "./trackerService.js";
|
||||
import { TorrentType } from './types.js';
|
||||
import {logger} from "./logger.js";
|
||||
|
||||
export async function processTorrentRecord(torrent) {
|
||||
const {category} = torrent;
|
||||
const type = category === 'tv' ? TorrentType.SERIES : TorrentType.MOVIE;
|
||||
const torrentInfo = await parseTorrent(torrent, type);
|
||||
logger.info(`Processing torrent ${torrentInfo.title} with infoHash ${torrentInfo.infoHash}`)
|
||||
|
||||
if (await checkAndUpdateTorrent(torrentInfo)) {
|
||||
return torrentInfo;
|
||||
}
|
||||
|
||||
return createTorrentEntry(torrentInfo);
|
||||
}
|
||||
|
||||
async function assignTorrentTrackers() {
|
||||
const trackers = await getTrackers();
|
||||
return trackers.join(',');
|
||||
}
|
||||
|
||||
async function parseTorrent(torrent, category) {
|
||||
const infoHash = torrent.infoHash?.trim().toLowerCase()
|
||||
return {
|
||||
title: torrent.name,
|
||||
torrentId: `${torrent.name}_${infoHash}`,
|
||||
infoHash: infoHash,
|
||||
seeders: 100,
|
||||
size: torrent.size,
|
||||
uploadDate: torrent.createdAt,
|
||||
imdbId: parseImdbId(torrent),
|
||||
type: category,
|
||||
provider: torrent.source,
|
||||
trackers: await assignTorrentTrackers(),
|
||||
}
|
||||
}
|
||||
|
||||
function parseImdbId(torrent) {
|
||||
if (torrent.imdb === undefined || torrent.imdb === null) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return torrent.imdb;
|
||||
}
|
||||
3
src/node/consumer/src/lib/interfaces/cache_options.ts
Normal file
3
src/node/consumer/src/lib/interfaces/cache_options.ts
Normal file
@@ -0,0 +1,3 @@
|
||||
export interface ICacheOptions {
|
||||
ttl: number;
|
||||
}
|
||||
11
src/node/consumer/src/lib/interfaces/cache_service.ts
Normal file
11
src/node/consumer/src/lib/interfaces/cache_service.ts
Normal file
@@ -0,0 +1,11 @@
|
||||
import {CacheMethod} from "@services/cache_service";
|
||||
|
||||
/* eslint-disable @typescript-eslint/no-explicit-any */
|
||||
export interface ICacheService {
|
||||
cacheWrapImdbId: (key: string, method: CacheMethod) => Promise<any>;
|
||||
cacheWrapKitsuId: (key: string, method: CacheMethod) => Promise<any>;
|
||||
cacheWrapMetadata: (id: string, method: CacheMethod) => Promise<any>;
|
||||
cacheTrackers: (method: CacheMethod) => Promise<any>;
|
||||
}
|
||||
|
||||
/* eslint-enable @typescript-eslint/no-explicit-any */
|
||||
84
src/node/consumer/src/lib/interfaces/cinemeta_metadata.ts
Normal file
84
src/node/consumer/src/lib/interfaces/cinemeta_metadata.ts
Normal file
@@ -0,0 +1,84 @@
|
||||
import {ICommonVideoMetadata} from "@interfaces/common_video_metadata";
|
||||
|
||||
export interface ICinemetaJsonResponse {
|
||||
meta?: ICinemetaMetaData;
|
||||
trailerStreams?: ICinemetaTrailerStream[];
|
||||
links?: ICinemetaLink[];
|
||||
behaviorHints?: ICinemetaBehaviorHints;
|
||||
}
|
||||
|
||||
export interface ICinemetaMetaData {
|
||||
awards?: string;
|
||||
cast?: string[];
|
||||
country?: string;
|
||||
description?: string;
|
||||
director?: null;
|
||||
dvdRelease?: null;
|
||||
genre?: string[];
|
||||
imdbRating?: string;
|
||||
name?: string;
|
||||
popularity?: number;
|
||||
poster?: string;
|
||||
released?: string;
|
||||
runtime?: string;
|
||||
status?: string;
|
||||
tvdb_id?: number;
|
||||
type?: string;
|
||||
writer?: string[];
|
||||
year?: string;
|
||||
background?: string;
|
||||
logo?: string;
|
||||
popularities?: ICinemetaPopularities;
|
||||
moviedb_id?: number;
|
||||
slug?: string;
|
||||
trailers?: ICinemetaTrailer[];
|
||||
id?: string;
|
||||
genres?: string[];
|
||||
releaseInfo?: string;
|
||||
videos?: ICinemetaVideo[];
|
||||
}
|
||||
|
||||
export interface ICinemetaPopularities {
|
||||
PXS_TEST?: number;
|
||||
PXS?: number;
|
||||
SCM?: number;
|
||||
EXMD?: number;
|
||||
ALLIANCE?: number;
|
||||
EJD?: number;
|
||||
moviedb?: number;
|
||||
trakt?: number;
|
||||
stremio?: number;
|
||||
stremio_lib?: number;
|
||||
}
|
||||
|
||||
export interface ICinemetaTrailer {
|
||||
source?: string;
|
||||
type?: string;
|
||||
}
|
||||
|
||||
export interface ICinemetaVideo extends ICommonVideoMetadata {
|
||||
name?: string;
|
||||
number?: number;
|
||||
firstAired?: string;
|
||||
tvdb_id?: number;
|
||||
rating?: string;
|
||||
overview?: string;
|
||||
thumbnail?: string;
|
||||
description?: string;
|
||||
}
|
||||
|
||||
export interface ICinemetaTrailerStream {
|
||||
title?: string;
|
||||
ytId?: string;
|
||||
}
|
||||
|
||||
export interface ICinemetaLink {
|
||||
name?: string;
|
||||
category?: string;
|
||||
url?: string;
|
||||
}
|
||||
|
||||
export interface ICinemetaBehaviorHints {
|
||||
defaultVideoId?: null;
|
||||
hasScheduledVideos?: boolean;
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
export interface ICommonVideoMetadata {
|
||||
season?: number;
|
||||
episode?: number;
|
||||
released?: string;
|
||||
title?: string;
|
||||
name?: string;
|
||||
id?: string;
|
||||
imdb_id?: string;
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
export interface IIngestedRabbitTorrent {
|
||||
name: string;
|
||||
source: string;
|
||||
category: string;
|
||||
infoHash: string;
|
||||
size: string;
|
||||
seeders: number;
|
||||
leechers: number;
|
||||
imdb: string;
|
||||
processed: boolean;
|
||||
}
|
||||
|
||||
export interface IIngestedRabbitMessage {
|
||||
message: IIngestedRabbitTorrent;
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
import {IKitsuLink, IKitsuTrailer} from "@interfaces/kitsu_metadata";
|
||||
|
||||
export interface IKitsuCatalogJsonResponse {
|
||||
metas: IKitsuCatalogMetaData[];
|
||||
}
|
||||
|
||||
export interface IKitsuCatalogMetaData {
|
||||
id: string;
|
||||
type: string;
|
||||
animeType: string;
|
||||
name: string;
|
||||
aliases: string[];
|
||||
description: string;
|
||||
releaseInfo: string;
|
||||
runtime: string;
|
||||
imdbRating: string;
|
||||
genres: string[];
|
||||
logo?: string;
|
||||
poster: string;
|
||||
background: string;
|
||||
trailers: IKitsuTrailer[];
|
||||
links: IKitsuLink[];
|
||||
}
|
||||
49
src/node/consumer/src/lib/interfaces/kitsu_metadata.ts
Normal file
49
src/node/consumer/src/lib/interfaces/kitsu_metadata.ts
Normal file
@@ -0,0 +1,49 @@
|
||||
import {ICommonVideoMetadata} from "@interfaces/common_video_metadata";
|
||||
|
||||
export interface IKitsuJsonResponse {
|
||||
cacheMaxAge?: number;
|
||||
meta?: IKitsuMeta;
|
||||
}
|
||||
|
||||
export interface IKitsuMeta {
|
||||
aliases?: string[];
|
||||
animeType?: string;
|
||||
background?: string;
|
||||
description?: string;
|
||||
country?: string;
|
||||
genres?: string[];
|
||||
id?: string;
|
||||
imdbRating?: string;
|
||||
imdb_id?: string;
|
||||
kitsu_id?: string;
|
||||
links?: IKitsuLink[];
|
||||
logo?: string;
|
||||
name?: string;
|
||||
poster?: string;
|
||||
releaseInfo?: string;
|
||||
runtime?: string;
|
||||
slug?: string;
|
||||
status?: string;
|
||||
trailers?: IKitsuTrailer[];
|
||||
type?: string;
|
||||
userCount?: number;
|
||||
videos?: IKitsuVideo[];
|
||||
year?: string;
|
||||
}
|
||||
|
||||
export interface IKitsuVideo extends ICommonVideoMetadata {
|
||||
imdbEpisode?: number;
|
||||
imdbSeason?: number;
|
||||
thumbnail?: string;
|
||||
}
|
||||
|
||||
export interface IKitsuTrailer {
|
||||
source?: string;
|
||||
type?: string;
|
||||
}
|
||||
|
||||
export interface IKitsuLink {
|
||||
name?: string;
|
||||
category?: string;
|
||||
url?: string;
|
||||
}
|
||||
12
src/node/consumer/src/lib/interfaces/logging_service.ts
Normal file
12
src/node/consumer/src/lib/interfaces/logging_service.ts
Normal file
@@ -0,0 +1,12 @@
|
||||
/* eslint-disable @typescript-eslint/no-explicit-any */
|
||||
export interface ILoggingService {
|
||||
info(message: string, ...args: any[]): void;
|
||||
|
||||
error(message: string, ...args: any[]): void;
|
||||
|
||||
debug(message: string, ...args: any[]): void;
|
||||
|
||||
warn(message: string, ...args: any[]): void;
|
||||
}
|
||||
|
||||
/* eslint-enable @typescript-eslint/no-explicit-any */
|
||||
9
src/node/consumer/src/lib/interfaces/metadata_query.ts
Normal file
9
src/node/consumer/src/lib/interfaces/metadata_query.ts
Normal file
@@ -0,0 +1,9 @@
|
||||
export interface IMetaDataQuery {
|
||||
title?: string
|
||||
type?: string
|
||||
year?: number | string
|
||||
date?: string
|
||||
season?: number
|
||||
episode?: number
|
||||
id?: string | number
|
||||
}
|
||||
15
src/node/consumer/src/lib/interfaces/metadata_response.ts
Normal file
15
src/node/consumer/src/lib/interfaces/metadata_response.ts
Normal file
@@ -0,0 +1,15 @@
|
||||
import {ICommonVideoMetadata} from "@interfaces/common_video_metadata";
|
||||
|
||||
export interface IMetadataResponse {
|
||||
kitsuId?: number;
|
||||
imdbId?: number;
|
||||
type?: string;
|
||||
title?: string;
|
||||
year?: number;
|
||||
country?: string;
|
||||
genres?: string[];
|
||||
status?: string;
|
||||
videos?: ICommonVideoMetadata[];
|
||||
episodeCount?: number[];
|
||||
totalCount?: number;
|
||||
}
|
||||
14
src/node/consumer/src/lib/interfaces/metadata_service.ts
Normal file
14
src/node/consumer/src/lib/interfaces/metadata_service.ts
Normal file
@@ -0,0 +1,14 @@
|
||||
import {IMetaDataQuery} from "@interfaces/metadata_query";
|
||||
import {IMetadataResponse} from "@interfaces/metadata_response";
|
||||
|
||||
export interface IMetadataService {
|
||||
getKitsuId(info: IMetaDataQuery): Promise<number | Error>;
|
||||
|
||||
getImdbId(info: IMetaDataQuery): Promise<string | undefined>;
|
||||
|
||||
getMetadata(query: IMetaDataQuery): Promise<IMetadataResponse | Error>;
|
||||
|
||||
isEpisodeImdbId(imdbId: string | undefined): Promise<boolean>;
|
||||
|
||||
escapeTitle(title: string): string;
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
import {IFileAttributes} from "@repository/interfaces/file_attributes";
|
||||
|
||||
export interface IParseTorrentTitleResult {
|
||||
title?: string;
|
||||
date?: string;
|
||||
year?: number | string;
|
||||
resolution?: string;
|
||||
extended?: boolean;
|
||||
unrated?: boolean;
|
||||
proper?: boolean;
|
||||
repack?: boolean;
|
||||
convert?: boolean;
|
||||
hardcoded?: boolean;
|
||||
retail?: boolean;
|
||||
remastered?: boolean;
|
||||
complete?: boolean;
|
||||
region?: string;
|
||||
container?: string;
|
||||
extension?: string;
|
||||
source?: string;
|
||||
codec?: string;
|
||||
bitDepth?: string;
|
||||
hdr?: Array<string>;
|
||||
audio?: string;
|
||||
group?: string;
|
||||
volumes?: Array<number>;
|
||||
seasons?: Array<number>;
|
||||
season?: number;
|
||||
episodes?: Array<number>;
|
||||
episode?: number;
|
||||
languages?: string;
|
||||
dubbed?: boolean;
|
||||
videoFile?: IFileAttributes;
|
||||
folderName?: string;
|
||||
fileName?: string;
|
||||
}
|
||||
18
src/node/consumer/src/lib/interfaces/parsed_torrent.ts
Normal file
18
src/node/consumer/src/lib/interfaces/parsed_torrent.ts
Normal file
@@ -0,0 +1,18 @@
|
||||
import {TorrentType} from "@enums/torrent_types";
|
||||
import {IParseTorrentTitleResult} from "@interfaces/parse_torrent_title_result";
|
||||
import {ITorrentFileCollection} from "@interfaces/torrent_file_collection";
|
||||
|
||||
export interface IParsedTorrent extends IParseTorrentTitleResult {
|
||||
size?: number;
|
||||
isPack?: boolean;
|
||||
imdbId?: string | number;
|
||||
kitsuId?: number;
|
||||
trackers?: string;
|
||||
provider?: string | null;
|
||||
infoHash: string;
|
||||
type: string | TorrentType;
|
||||
uploadDate?: Date;
|
||||
seeders?: number;
|
||||
torrentId?: string;
|
||||
fileCollection?: ITorrentFileCollection;
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
export interface IProcessTorrentsJob {
|
||||
listenToQueue: () => Promise<void>;
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
import {IParsedTorrent} from "@interfaces/parsed_torrent";
|
||||
import {ITorrentFileCollection} from "@interfaces/torrent_file_collection";
|
||||
|
||||
export interface ITorrentDownloadService {
|
||||
getTorrentFiles(torrent: IParsedTorrent, timeout: number): Promise<ITorrentFileCollection>;
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import {IParsedTorrent} from "@interfaces/parsed_torrent";
|
||||
import {ITorrentAttributes, ITorrentCreationAttributes} from "@repository/interfaces/torrent_attributes";
|
||||
import {SkipTorrent} from "@repository/models/skipTorrent";
|
||||
import {Torrent} from "@repository/models/torrent";
|
||||
|
||||
export interface ITorrentEntriesService {
|
||||
createTorrentEntry(torrent: IParsedTorrent, overwrite: boolean): Promise<void>;
|
||||
|
||||
createSkipTorrentEntry(torrent: ITorrentCreationAttributes): Promise<[SkipTorrent, boolean | null]>;
|
||||
|
||||
getStoredTorrentEntry(torrent: Torrent): Promise<Torrent | SkipTorrent | null | undefined>;
|
||||
|
||||
checkAndUpdateTorrent(torrent: IParsedTorrent): Promise<boolean>;
|
||||
|
||||
createTorrentContents(torrent: Torrent): Promise<void>;
|
||||
|
||||
updateTorrentSeeders(torrent: ITorrentAttributes): Promise<[number] | undefined>;
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
import {IContentAttributes} from "@repository/interfaces/content_attributes";
|
||||
import {IFileAttributes} from "@repository/interfaces/file_attributes";
|
||||
import {ISubtitleAttributes} from "@repository/interfaces/subtitle_attributes";
|
||||
|
||||
export interface ITorrentFileCollection {
|
||||
contents?: IContentAttributes[];
|
||||
videos?: IFileAttributes[];
|
||||
subtitles?: ISubtitleAttributes[];
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
import {IParsedTorrent} from "@interfaces/parsed_torrent";
|
||||
import {ITorrentFileCollection} from "@interfaces/torrent_file_collection";
|
||||
|
||||
export interface ITorrentFileService {
|
||||
parseTorrentFiles(torrent: IParsedTorrent): Promise<ITorrentFileCollection>;
|
||||
|
||||
isPackTorrent(torrent: IParsedTorrent): boolean;
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
import {IIngestedTorrentAttributes} from "@repository/interfaces/ingested_torrent_attributes";
|
||||
|
||||
export interface ITorrentProcessingService {
|
||||
processTorrentRecord(torrent: IIngestedTorrentAttributes): Promise<void>;
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
import {ITorrentFileCollection} from "@interfaces/torrent_file_collection";
|
||||
|
||||
export interface ITorrentSubtitleService {
|
||||
assignSubtitles(fileCollection: ITorrentFileCollection): ITorrentFileCollection;
|
||||
}
|
||||
3
src/node/consumer/src/lib/interfaces/tracker_service.ts
Normal file
3
src/node/consumer/src/lib/interfaces/tracker_service.ts
Normal file
@@ -0,0 +1,3 @@
|
||||
export interface ITrackerService {
|
||||
getTrackers(): Promise<string[]>;
|
||||
}
|
||||
63
src/node/consumer/src/lib/jobs/process_torrents_job.ts
Normal file
63
src/node/consumer/src/lib/jobs/process_torrents_job.ts
Normal file
@@ -0,0 +1,63 @@
|
||||
import {IIngestedRabbitMessage, IIngestedRabbitTorrent} from "@interfaces/ingested_rabbit_message";
|
||||
import {ILoggingService} from "@interfaces/logging_service";
|
||||
import {IProcessTorrentsJob} from "@interfaces/process_torrents_job";
|
||||
import {ITorrentProcessingService} from "@interfaces/torrent_processing_service";
|
||||
import {IIngestedTorrentAttributes} from "@repository/interfaces/ingested_torrent_attributes";
|
||||
import {configurationService} from '@services/configuration_service';
|
||||
import {IocTypes} from "@setup/ioc_types";
|
||||
import client, {Channel, Connection, ConsumeMessage, Options} from 'amqplib'
|
||||
import {inject, injectable} from "inversify";
|
||||
|
||||
@injectable()
|
||||
export class ProcessTorrentsJob implements IProcessTorrentsJob {
|
||||
@inject(IocTypes.ITorrentProcessingService) torrentProcessingService: ITorrentProcessingService;
|
||||
@inject(IocTypes.ILoggingService) logger: ILoggingService;
|
||||
|
||||
private readonly assertQueueOptions: Options.AssertQueue = {durable: true};
|
||||
private readonly consumeQueueOptions: Options.Consume = {noAck: false};
|
||||
|
||||
async listenToQueue(): Promise<void> {
|
||||
if (!configurationService.jobConfig.JOBS_ENABLED) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const connection: Connection = await client.connect(configurationService.rabbitConfig.RABBIT_URI);
|
||||
const channel: Channel = await connection.createChannel();
|
||||
await this.assertAndConsumeQueue(channel);
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to connect and setup channel', error);
|
||||
}
|
||||
}
|
||||
|
||||
private processMessage = (msg: ConsumeMessage | null): Promise<void> => {
|
||||
const ingestedTorrent: IIngestedTorrentAttributes = this.getMessageAsJson(msg);
|
||||
return this.torrentProcessingService.processTorrentRecord(ingestedTorrent);
|
||||
};
|
||||
|
||||
private getMessageAsJson = (msg: ConsumeMessage | null): IIngestedTorrentAttributes => {
|
||||
const content = msg?.content.toString('utf8') ?? "{}";
|
||||
const receivedObject: IIngestedRabbitMessage = JSON.parse(content);
|
||||
const receivedTorrent: IIngestedRabbitTorrent = receivedObject.message;
|
||||
return {...receivedTorrent, info_hash: receivedTorrent.infoHash};
|
||||
};
|
||||
|
||||
private assertAndConsumeQueue = async (channel: Channel): Promise<void> => {
|
||||
this.logger.info('Worker is running! Waiting for new torrents...');
|
||||
|
||||
const ackMsg = async (msg: ConsumeMessage | null): Promise<void> => {
|
||||
await this.processMessage(msg)
|
||||
.then(() => this.logger.info('Processed torrent'))
|
||||
.then(() => msg && channel.ack(msg))
|
||||
.catch((error) => this.logger.error('Failed to process torrent', error));
|
||||
}
|
||||
|
||||
try {
|
||||
await channel.assertQueue(configurationService.rabbitConfig.QUEUE_NAME, this.assertQueueOptions);
|
||||
await channel.prefetch(configurationService.jobConfig.JOB_CONCURRENCY);
|
||||
await channel.consume(configurationService.rabbitConfig.QUEUE_NAME, ackMsg, this.consumeQueueOptions);
|
||||
} catch (error) {
|
||||
this.logger.error('Failed to setup channel', error);
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -1,5 +0,0 @@
|
||||
import pino from 'pino';
|
||||
|
||||
export const logger = pino({
|
||||
level: process.env.LOG_LEVEL || 'info'
|
||||
});
|
||||
@@ -1,165 +0,0 @@
|
||||
import axios from 'axios';
|
||||
import { search } from 'google-sr';
|
||||
import nameToImdb from 'name-to-imdb';
|
||||
import { cacheWrapImdbId, cacheWrapKitsuId, cacheWrapMetadata } from './cache.js';
|
||||
import { TorrentType } from './types.js';
|
||||
|
||||
const CINEMETA_URL = 'https://v3-cinemeta.strem.io';
|
||||
const KITSU_URL = 'https://anime-kitsu.strem.fun';
|
||||
const TIMEOUT = 20000;
|
||||
|
||||
export function getMetadata(id, type = TorrentType.SERIES) {
|
||||
if (!id) {
|
||||
return Promise.reject("no valid id provided");
|
||||
}
|
||||
|
||||
const key = Number.isInteger(id) || id.match(/^\d+$/) ? `kitsu:${id}` : id;
|
||||
const metaType = type === TorrentType.MOVIE ? TorrentType.MOVIE : TorrentType.SERIES;
|
||||
return cacheWrapMetadata(key, () => _requestMetadata(`${KITSU_URL}/meta/${metaType}/${key}.json`)
|
||||
.catch(() => _requestMetadata(`${CINEMETA_URL}/meta/${metaType}/${key}.json`))
|
||||
.catch(() => {
|
||||
// try different type in case there was a mismatch
|
||||
const otherType = metaType === TorrentType.MOVIE ? TorrentType.SERIES : TorrentType.MOVIE;
|
||||
return _requestMetadata(`${CINEMETA_URL}/meta/${otherType}/${key}.json`)
|
||||
})
|
||||
.catch((error) => {
|
||||
throw new Error(`failed metadata query ${key} due: ${error.message}`);
|
||||
}));
|
||||
}
|
||||
|
||||
function _requestMetadata(url) {
|
||||
return axios.get(url, { timeout: TIMEOUT })
|
||||
.then((response) => {
|
||||
const body = response.data;
|
||||
if (body && body.meta && (body.meta.imdb_id || body.meta.kitsu_id)) {
|
||||
return {
|
||||
kitsuId: body.meta.kitsu_id,
|
||||
imdbId: body.meta.imdb_id,
|
||||
type: body.meta.type,
|
||||
title: body.meta.name,
|
||||
year: body.meta.year,
|
||||
country: body.meta.country,
|
||||
genres: body.meta.genres,
|
||||
status: body.meta.status,
|
||||
videos: (body.meta.videos || [])
|
||||
.map((video) => Number.isInteger(video.imdbSeason)
|
||||
? {
|
||||
name: video.name || video.title,
|
||||
season: video.season,
|
||||
episode: video.episode,
|
||||
imdbSeason: video.imdbSeason,
|
||||
imdbEpisode: video.imdbEpisode
|
||||
}
|
||||
: {
|
||||
name: video.name || video.title,
|
||||
season: video.season,
|
||||
episode: video.episode,
|
||||
kitsuId: video.kitsu_id,
|
||||
kitsuEpisode: video.kitsuEpisode,
|
||||
released: video.released
|
||||
}
|
||||
),
|
||||
episodeCount: Object.values((body.meta.videos || [])
|
||||
.filter((entry) => entry.season !== 0 && entry.episode !== 0)
|
||||
.sort((a, b) => a.season - b.season)
|
||||
.reduce((map, next) => {
|
||||
map[next.season] = map[next.season] + 1 || 1;
|
||||
return map;
|
||||
}, {})),
|
||||
totalCount: body.meta.videos && body.meta.videos
|
||||
.filter((entry) => entry.season !== 0 && entry.episode !== 0).length
|
||||
};
|
||||
} else {
|
||||
throw new Error('No search results');
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export function escapeTitle(title) {
|
||||
return title.toLowerCase()
|
||||
.normalize('NFKD') // normalize non-ASCII characters
|
||||
.replace(/[\u0300-\u036F]/g, '')
|
||||
.replace(/&/g, 'and')
|
||||
.replace(/[;, ~./]+/g, ' ') // replace dots, commas or underscores with spaces
|
||||
.replace(/[^\w \-()×+#@!'\u0400-\u04ff]+/g, '') // remove all non-alphanumeric chars
|
||||
.replace(/^\d{1,2}[.#\s]+(?=(?:\d+[.\s]*)?[\u0400-\u04ff])/i, '') // remove russian movie numbering
|
||||
.replace(/\s{2,}/, ' ') // replace multiple spaces
|
||||
.trim();
|
||||
}
|
||||
|
||||
export async function getImdbId(info, type) {
|
||||
const name = escapeTitle(info.title);
|
||||
const year = info.year || (info.date && info.date.slice(0, 4));
|
||||
const key = `${name}_${year || 'NA'}_${type}`;
|
||||
const query = `${name} ${year || ''} ${type} imdb`;
|
||||
const fallbackQuery = `${name} ${type} imdb`;
|
||||
const googleQuery = year ? query : fallbackQuery;
|
||||
|
||||
try {
|
||||
const imdbId = await cacheWrapImdbId(key,
|
||||
() => getIMDbIdFromNameToImdb(name, info.year, type)
|
||||
);
|
||||
return imdbId && 'tt' + imdbId.replace(/tt0*([1-9][0-9]*)$/, '$1').padStart(7, '0');
|
||||
} catch (error) {
|
||||
const imdbIdFallback = await getIMDbIdFromGoogle(googleQuery);
|
||||
return imdbIdFallback && 'tt' + imdbIdFallback.replace(/tt0*([1-9][0-9]*)$/, '$1').padStart(7, '0');
|
||||
}
|
||||
}
|
||||
|
||||
function getIMDbIdFromNameToImdb(name, year, type) {
|
||||
return new Promise((resolve, reject) => {
|
||||
nameToImdb({ name, year, type }, function(err, res) {
|
||||
if (res) {
|
||||
resolve(res);
|
||||
} else {
|
||||
reject(err || new Error('Failed IMDbId search'));
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function getIMDbIdFromGoogle(query) {
|
||||
try {
|
||||
const searchResults = await search({ query: query });
|
||||
for (const result of searchResults) {
|
||||
if (result.link.includes('imdb.com/title/')) {
|
||||
const match = result.link.match(/imdb\.com\/title\/(tt\d+)/);
|
||||
if (match) {
|
||||
return match[1];
|
||||
}
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
catch (error) {
|
||||
throw new Error('Failed to find IMDb ID from Google search');
|
||||
}
|
||||
}
|
||||
|
||||
export async function getKitsuId(info) {
|
||||
const title = escapeTitle(info.title.replace(/\s\|\s.*/, ''));
|
||||
const year = info.year ? ` ${info.year}` : '';
|
||||
const season = info.season > 1 ? ` S${info.season}` : '';
|
||||
const key = `${title}${year}${season}`;
|
||||
const query = encodeURIComponent(key);
|
||||
|
||||
return cacheWrapKitsuId(key,
|
||||
() => axios.get(`${KITSU_URL}/catalog/series/kitsu-anime-list/search=${query}.json`, { timeout: 60000 })
|
||||
.then((response) => {
|
||||
const body = response.data;
|
||||
if (body && body.metas && body.metas.length) {
|
||||
return body.metas[0].id.replace('kitsu:', '');
|
||||
} else {
|
||||
throw new Error('No search results');
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
export async function isEpisodeImdbId(imdbId) {
|
||||
if (!imdbId) {
|
||||
return false;
|
||||
}
|
||||
return axios.get(`https://www.imdb.com/title/${imdbId}/`, { timeout: 10000 })
|
||||
.then(response => !!(response.data && response.data.includes('video.episode')))
|
||||
.catch(() => false);
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
import {BooleanHelpers} from "@helpers/boolean_helpers";
|
||||
|
||||
export const cacheConfig = {
|
||||
MONGODB_HOST: process.env.MONGODB_HOST || 'mongodb',
|
||||
MONGODB_PORT: process.env.MONGODB_PORT || '27017',
|
||||
MONGODB_DB: process.env.MONGODB_DB || 'knightcrawler',
|
||||
MONGO_INITDB_ROOT_USERNAME: process.env.MONGO_INITDB_ROOT_USERNAME || 'mongo',
|
||||
MONGO_INITDB_ROOT_PASSWORD: process.env.MONGO_INITDB_ROOT_PASSWORD || 'mongo',
|
||||
NO_CACHE: BooleanHelpers.parseBool(process.env.NO_CACHE, false),
|
||||
COLLECTION_NAME: process.env.MONGODB_COLLECTION || 'knightcrawler_consumer_collection',
|
||||
|
||||
get MONGO_URI(): string {
|
||||
return `mongodb://${this.MONGO_INITDB_ROOT_USERNAME}:${this.MONGO_INITDB_ROOT_PASSWORD}@${this.MONGODB_HOST}:${this.MONGODB_PORT}/${this.MONGODB_DB}?authSource=admin`;
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,14 @@
|
||||
import {BooleanHelpers} from "@helpers/boolean_helpers";
|
||||
|
||||
export const databaseConfig = {
|
||||
POSTGRES_HOST: process.env.POSTGRES_HOST || 'postgres',
|
||||
POSTGRES_PORT: parseInt(process.env.POSTGRES_PORT || '5432'),
|
||||
POSTGRES_DB: process.env.POSTGRES_DB || 'knightcrawler',
|
||||
POSTGRES_USER: process.env.POSTGRES_USER || 'postgres',
|
||||
POSTGRES_PASSWORD: process.env.POSTGRES_PASSWORD || 'postgres',
|
||||
AUTO_CREATE_AND_APPLY_MIGRATIONS: BooleanHelpers.parseBool(process.env.AUTO_CREATE_AND_APPLY_MIGRATIONS, false),
|
||||
|
||||
get POSTGRES_URI(): string {
|
||||
return `postgres://${this.POSTGRES_USER}:${this.POSTGRES_PASSWORD}@${this.POSTGRES_HOST}:${this.POSTGRES_PORT}/${this.POSTGRES_DB}`;
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,6 @@
|
||||
import {BooleanHelpers} from "@helpers/boolean_helpers";
|
||||
|
||||
export const jobConfig = {
|
||||
JOB_CONCURRENCY: parseInt(process.env.JOB_CONCURRENCY || "1", 10),
|
||||
JOBS_ENABLED: BooleanHelpers.parseBool(process.env.JOBS_ENABLED, true)
|
||||
};
|
||||
@@ -0,0 +1,4 @@
|
||||
export const metadataConfig = {
|
||||
IMDB_CONCURRENT: parseInt(process.env.IMDB_CONCURRENT || "1", 10),
|
||||
IMDB_INTERVAL_MS: parseInt(process.env.IMDB_INTERVAL_MS || "1000", 10)
|
||||
};
|
||||
@@ -0,0 +1,4 @@
|
||||
export const rabbitConfig = {
|
||||
RABBIT_URI: process.env.RABBIT_URI || 'amqp://localhost',
|
||||
QUEUE_NAME: process.env.QUEUE_NAME || 'test-queue'
|
||||
};
|
||||
@@ -0,0 +1,4 @@
|
||||
export const torrentConfig = {
|
||||
MAX_CONNECTIONS_PER_TORRENT: parseInt(process.env.MAX_CONNECTIONS_PER_TORRENT || "20", 10),
|
||||
TIMEOUT: parseInt(process.env.TORRENT_TIMEOUT || "30000", 10)
|
||||
};
|
||||
@@ -0,0 +1,6 @@
|
||||
import {BooleanHelpers} from "@helpers/boolean_helpers";
|
||||
|
||||
export const trackerConfig = {
|
||||
TRACKERS_URL: process.env.TRACKERS_URL || 'https://ngosang.github.io/trackerslist/trackers_all.txt',
|
||||
UDP_ENABLED: BooleanHelpers.parseBool(process.env.UDP_TRACKERS_ENABLED, false)
|
||||
};
|
||||
@@ -1,98 +0,0 @@
|
||||
import { parse } from 'parse-torrent-title';
|
||||
import { TorrentType } from './types.js';
|
||||
|
||||
const MULTIPLE_FILES_SIZE = 4 * 1024 * 1024 * 1024; // 4 GB
|
||||
|
||||
export function parseSeriesVideos(torrent, videos) {
|
||||
const parsedTorrentName = parse(torrent.title);
|
||||
const hasMovies = parsedTorrentName.complete || !!torrent.title.match(/movies?(?:\W|$)/i);
|
||||
const parsedVideos = videos.map(video => parseSeriesVideo(video, parsedTorrentName));
|
||||
return parsedVideos.map(video => ({ ...video, isMovie: isMovieVideo(video, parsedVideos, torrent.type, hasMovies) }));
|
||||
}
|
||||
|
||||
function parseSeriesVideo(video, parsedTorrentName) {
|
||||
const videoInfo = parse(video.name);
|
||||
// the episode may be in a folder containing season number
|
||||
if (!Number.isInteger(videoInfo.season) && video.path.includes('/')) {
|
||||
const folders = video.path.split('/');
|
||||
const pathInfo = parse(folders[folders.length - 2]);
|
||||
videoInfo.season = pathInfo.season;
|
||||
}
|
||||
if (!Number.isInteger(videoInfo.season) && parsedTorrentName.season) {
|
||||
videoInfo.season = parsedTorrentName.season;
|
||||
}
|
||||
if (!Number.isInteger(videoInfo.season) && videoInfo.seasons && videoInfo.seasons.length > 1) {
|
||||
// in case single file was interpreted as having multiple seasons
|
||||
videoInfo.season = videoInfo.seasons[0];
|
||||
}
|
||||
if (!Number.isInteger(videoInfo.season) && video.path.includes('/') && parsedTorrentName.seasons
|
||||
&& parsedTorrentName.seasons.length > 1) {
|
||||
// russian season are usually named with 'series name-2` i.e. Улицы разбитых фонарей-6/22. Одиночный выстрел.mkv
|
||||
const folderPathSeasonMatch = video.path.match(/[\u0400-\u04ff]-(\d{1,2})(?=.*\/)/);
|
||||
videoInfo.season = folderPathSeasonMatch && parseInt(folderPathSeasonMatch[1], 10) || undefined;
|
||||
}
|
||||
// sometimes video file does not have correct date format as in torrent title
|
||||
if (!videoInfo.episodes && !videoInfo.date && parsedTorrentName.date) {
|
||||
videoInfo.date = parsedTorrentName.date;
|
||||
}
|
||||
// limit number of episodes in case of incorrect parsing
|
||||
if (videoInfo.episodes && videoInfo.episodes.length > 20) {
|
||||
videoInfo.episodes = [videoInfo.episodes[0]];
|
||||
videoInfo.episode = videoInfo.episodes[0];
|
||||
}
|
||||
// force episode to any found number if it was not parsed
|
||||
if (!videoInfo.episodes && !videoInfo.date) {
|
||||
const epMatcher = videoInfo.title.match(
|
||||
/(?<!season\W*|disk\W*|movie\W*|film\W*)(?:^|\W|_)(\d{1,4})(?:a|b|c|v\d)?(?:_|\W|$)(?!disk|movie|film)/i);
|
||||
videoInfo.episodes = epMatcher && [parseInt(epMatcher[1], 10)];
|
||||
videoInfo.episode = videoInfo.episodes && videoInfo.episodes[0];
|
||||
}
|
||||
if (!videoInfo.episodes && !videoInfo.date) {
|
||||
const epMatcher = video.name.match(new RegExp(`(?:\\(${videoInfo.year}\\)|part)[._ ]?(\\d{1,3})(?:\\b|_)`, "i"));
|
||||
videoInfo.episodes = epMatcher && [parseInt(epMatcher[1], 10)];
|
||||
videoInfo.episode = videoInfo.episodes && videoInfo.episodes[0];
|
||||
}
|
||||
|
||||
return { ...video, ...videoInfo };
|
||||
}
|
||||
|
||||
function isMovieVideo(video, otherVideos, type, hasMovies) {
|
||||
if (Number.isInteger(video.season) && Array.isArray(video.episodes)) {
|
||||
// not movie if video has season
|
||||
return false;
|
||||
}
|
||||
if (video.name.match(/\b(?:\d+[ .]movie|movie[ .]\d+)\b/i)) {
|
||||
// movie if video explicitly has numbered movie keyword in the name, ie. 1 Movie or Movie 1
|
||||
return true;
|
||||
}
|
||||
if (!hasMovies && type !== TorrentType.ANIME) {
|
||||
// not movie if torrent name does not contain movies keyword or is not a pack torrent and is not anime
|
||||
return false;
|
||||
}
|
||||
if (!video.episodes) {
|
||||
// movie if there's no episode info it could be a movie
|
||||
return true;
|
||||
}
|
||||
// movie if contains year info and there aren't more than 3 video with same title and year
|
||||
// as some series titles might contain year in it.
|
||||
return !!video.year
|
||||
&& otherVideos.length > 3
|
||||
&& otherVideos.filter(other => other.title === video.title && other.year === video.year) < 3;
|
||||
}
|
||||
|
||||
export function isPackTorrent(torrent) {
|
||||
if (torrent.pack) {
|
||||
return true;
|
||||
}
|
||||
const parsedInfo = parse(torrent.title);
|
||||
if (torrent.type === TorrentType.MOVIE) {
|
||||
return parsedInfo.complete || typeof parsedInfo.year === 'string' || /movies/i.test(torrent.title);
|
||||
}
|
||||
const hasMultipleEpisodes = parsedInfo.complete ||
|
||||
torrent.size > MULTIPLE_FILES_SIZE ||
|
||||
(parsedInfo.seasons && parsedInfo.seasons.length > 1) ||
|
||||
(parsedInfo.episodes && parsedInfo.episodes.length > 1) ||
|
||||
(parsedInfo.seasons && !parsedInfo.episodes);
|
||||
const hasSingleEpisode = Number.isInteger(parsedInfo.episode) || (!parsedInfo.episodes && parsedInfo.date);
|
||||
return hasMultipleEpisodes && !hasSingleEpisode;
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
/**
|
||||
* Execute promises in sequence one after another.
|
||||
*/
|
||||
export async function sequence(promises) {
|
||||
return promises.reduce((promise, func) =>
|
||||
promise.then(result => func().then(Array.prototype.concat.bind(result))), Promise.resolve([]));
|
||||
}
|
||||
|
||||
/**
|
||||
* Return first resolved promise as the result.
|
||||
*/
|
||||
export async function first(promises) {
|
||||
return Promise.all(promises.map((p) => {
|
||||
// If a request fails, count that as a resolution so it will keep
|
||||
// waiting for other possible successes. If a request succeeds,
|
||||
// treat it as a rejection so Promise.all immediately bails out.
|
||||
return p.then(
|
||||
(val) => Promise.reject(val),
|
||||
(err) => Promise.resolve(err)
|
||||
);
|
||||
})).then(
|
||||
// If '.all' resolved, we've just got an array of errors.
|
||||
(errors) => Promise.reject(errors),
|
||||
// If '.all' rejected, we've got the result we wanted.
|
||||
(val) => Promise.resolve(val)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Delay promise
|
||||
*/
|
||||
export async function delay(duration) {
|
||||
return new Promise((resolve) => setTimeout(resolve, duration));
|
||||
}
|
||||
|
||||
/**
|
||||
* Timeout promise after a set time in ms
|
||||
*/
|
||||
export async function timeout(timeoutMs, promise, message = 'Timed out') {
|
||||
return Promise.race([
|
||||
promise,
|
||||
new Promise(function (resolve, reject) {
|
||||
setTimeout(function () {
|
||||
reject(message);
|
||||
}, timeoutMs);
|
||||
})
|
||||
]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return most common value from given array.
|
||||
*/
|
||||
export function mostCommonValue(array) {
|
||||
return array.sort((a, b) => array.filter(v => v === a).length - array.filter(v => v === b).length).pop();
|
||||
}
|
||||
@@ -1,382 +0,0 @@
|
||||
import moment from 'moment';
|
||||
import { Sequelize, Op, DataTypes, fn, col, literal } from 'sequelize';
|
||||
import { databaseConfig } from './config.js';
|
||||
import { logger } from "./logger.js";
|
||||
import * as Promises from './promises.js';
|
||||
|
||||
const database = new Sequelize(
|
||||
databaseConfig.POSTGRES_URI,
|
||||
{
|
||||
logging: false
|
||||
}
|
||||
);
|
||||
|
||||
const Provider = database.define('provider', {
|
||||
name: { type: DataTypes.STRING(32), primaryKey: true },
|
||||
lastScraped: { type: DataTypes.DATE },
|
||||
lastScrapedId: { type: DataTypes.STRING(128) }
|
||||
});
|
||||
|
||||
const IngestedTorrent = database.define('ingested_torrent', {
|
||||
id: { type: DataTypes.BIGINT, autoIncrement: true, primaryKey: true },
|
||||
name: DataTypes.STRING,
|
||||
source: DataTypes.STRING,
|
||||
category: DataTypes.STRING,
|
||||
info_hash: DataTypes.STRING,
|
||||
size: DataTypes.STRING,
|
||||
seeders: DataTypes.INTEGER,
|
||||
leechers: DataTypes.INTEGER,
|
||||
imdb: DataTypes.STRING,
|
||||
processed: {
|
||||
type: DataTypes.BOOLEAN,
|
||||
defaultValue: false
|
||||
}
|
||||
},
|
||||
{
|
||||
indexes: [
|
||||
{
|
||||
unique: true,
|
||||
fields: ['source', 'info_hash']
|
||||
}
|
||||
]
|
||||
})
|
||||
|
||||
/* eslint-disable no-unused-vars */
|
||||
const IngestedPage = database.define('ingested_page', {
|
||||
id: { type: DataTypes.BIGINT, autoIncrement: true, primaryKey: true },
|
||||
url: { type: DataTypes.STRING, allowNull: false },
|
||||
},
|
||||
{
|
||||
indexes: [
|
||||
{
|
||||
unique: true,
|
||||
fields: ['url']
|
||||
}
|
||||
]
|
||||
})
|
||||
/* eslint-enable no-unused-vars */
|
||||
|
||||
const Torrent = database.define('torrent',
|
||||
{
|
||||
infoHash: { type: DataTypes.STRING(64), primaryKey: true },
|
||||
provider: { type: DataTypes.STRING(32), allowNull: false },
|
||||
torrentId: { type: DataTypes.STRING(512) },
|
||||
title: { type: DataTypes.STRING(512), allowNull: false },
|
||||
size: { type: DataTypes.BIGINT },
|
||||
type: { type: DataTypes.STRING(16), allowNull: false },
|
||||
uploadDate: { type: DataTypes.DATE, allowNull: false },
|
||||
seeders: { type: DataTypes.SMALLINT },
|
||||
trackers: { type: DataTypes.STRING(8000) },
|
||||
languages: { type: DataTypes.STRING(4096) },
|
||||
resolution: { type: DataTypes.STRING(16) },
|
||||
reviewed: { type: DataTypes.BOOLEAN, allowNull: false, defaultValue: false },
|
||||
opened: { type: DataTypes.BOOLEAN, allowNull: false, defaultValue: false }
|
||||
}
|
||||
);
|
||||
|
||||
const File = database.define('file',
|
||||
{
|
||||
id: { type: DataTypes.BIGINT, autoIncrement: true, primaryKey: true },
|
||||
infoHash: {
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
references: { model: Torrent, key: 'infoHash' },
|
||||
onDelete: 'CASCADE'
|
||||
},
|
||||
fileIndex: { type: DataTypes.INTEGER },
|
||||
title: { type: DataTypes.STRING(512), allowNull: false },
|
||||
size: { type: DataTypes.BIGINT },
|
||||
imdbId: { type: DataTypes.STRING(32) },
|
||||
imdbSeason: { type: DataTypes.INTEGER },
|
||||
imdbEpisode: { type: DataTypes.INTEGER },
|
||||
kitsuId: { type: DataTypes.INTEGER },
|
||||
kitsuEpisode: { type: DataTypes.INTEGER }
|
||||
},
|
||||
{
|
||||
indexes: [
|
||||
{
|
||||
unique: true,
|
||||
name: 'files_unique_file_constraint',
|
||||
fields: [
|
||||
col('infoHash'),
|
||||
fn('COALESCE', (col('fileIndex')), -1),
|
||||
fn('COALESCE', (col('imdbId')), 'null'),
|
||||
fn('COALESCE', (col('imdbSeason')), -1),
|
||||
fn('COALESCE', (col('imdbEpisode')), -1),
|
||||
fn('COALESCE', (col('kitsuId')), -1),
|
||||
fn('COALESCE', (col('kitsuEpisode')), -1)
|
||||
]
|
||||
},
|
||||
{ unique: false, fields: ['imdbId', 'imdbSeason', 'imdbEpisode'] },
|
||||
{ unique: false, fields: ['kitsuId', 'kitsuEpisode'] }
|
||||
]
|
||||
}
|
||||
);
|
||||
|
||||
const Subtitle = database.define('subtitle',
|
||||
{
|
||||
infoHash: {
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
references: { model: Torrent, key: 'infoHash' },
|
||||
onDelete: 'CASCADE'
|
||||
},
|
||||
fileIndex: {
|
||||
type: DataTypes.INTEGER,
|
||||
allowNull: false
|
||||
},
|
||||
fileId: {
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: true,
|
||||
references: { model: File, key: 'id' },
|
||||
onDelete: 'SET NULL'
|
||||
},
|
||||
title: { type: DataTypes.STRING(512), allowNull: false },
|
||||
},
|
||||
{
|
||||
timestamps: false,
|
||||
indexes: [
|
||||
{
|
||||
unique: true,
|
||||
name: 'subtitles_unique_subtitle_constraint',
|
||||
fields: [
|
||||
col('infoHash'),
|
||||
col('fileIndex'),
|
||||
fn('COALESCE', (col('fileId')), -1)
|
||||
]
|
||||
},
|
||||
{ unique: false, fields: ['fileId'] }
|
||||
]
|
||||
}
|
||||
);
|
||||
|
||||
const Content = database.define('content',
|
||||
{
|
||||
infoHash: {
|
||||
type: DataTypes.STRING(64),
|
||||
primaryKey: true,
|
||||
allowNull: false,
|
||||
references: { model: Torrent, key: 'infoHash' },
|
||||
onDelete: 'CASCADE'
|
||||
},
|
||||
fileIndex: {
|
||||
type: DataTypes.INTEGER,
|
||||
primaryKey: true,
|
||||
allowNull: false
|
||||
},
|
||||
path: { type: DataTypes.STRING(512), allowNull: false },
|
||||
size: { type: DataTypes.BIGINT },
|
||||
},
|
||||
{
|
||||
timestamps: false,
|
||||
}
|
||||
);
|
||||
|
||||
const SkipTorrent = database.define('skip_torrent', {
|
||||
infoHash: { type: DataTypes.STRING(64), primaryKey: true },
|
||||
});
|
||||
|
||||
Torrent.hasMany(File, { foreignKey: 'infoHash', constraints: false });
|
||||
File.belongsTo(Torrent, { foreignKey: 'infoHash', constraints: false });
|
||||
Torrent.hasMany(Content, { foreignKey: 'infoHash', constraints: false });
|
||||
Content.belongsTo(Torrent, { foreignKey: 'infoHash', constraints: false });
|
||||
File.hasMany(Subtitle, { foreignKey: 'fileId', constraints: false });
|
||||
Subtitle.belongsTo(File, { foreignKey: 'fileId', constraints: false });
|
||||
|
||||
export function connect() {
|
||||
return database.sync({ alter: databaseConfig.AUTO_CREATE_AND_APPLY_MIGRATIONS })
|
||||
.catch(error => {
|
||||
console.error('Failed syncing database: ', error.message);
|
||||
throw error;
|
||||
});
|
||||
// I'm not convinced this code is needed. If anyone can see where it leads, or what it does, please inform me.
|
||||
// For now, I'm commenting it out. I don't think we ever reach this at the moment anyway as the previous ENABLE_SYNC
|
||||
// was always on.
|
||||
// return Promise.resolve();
|
||||
}
|
||||
|
||||
export function getProvider(provider) {
|
||||
return Provider.findOrCreate({ where: { name: { [Op.eq]: provider.name } }, defaults: provider })
|
||||
.then((result) => result[0])
|
||||
.catch(() => provider);
|
||||
}
|
||||
|
||||
export function getTorrent(torrent) {
|
||||
const where = torrent.infoHash
|
||||
? { infoHash: torrent.infoHash }
|
||||
: { provider: torrent.provider, torrentId: torrent.torrentId }
|
||||
return Torrent.findOne({ where: where });
|
||||
}
|
||||
|
||||
export function getTorrentsBasedOnTitle(titleQuery, type) {
|
||||
return getTorrentsBasedOnQuery({ title: { [Op.regexp]: `${titleQuery}` }, type: type });
|
||||
}
|
||||
|
||||
export function getTorrentsBasedOnQuery(where) {
|
||||
return Torrent.findAll({ where: where });
|
||||
}
|
||||
|
||||
export function getFilesBasedOnQuery(where) {
|
||||
return File.findAll({ where: where });
|
||||
}
|
||||
|
||||
export function getUnprocessedIngestedTorrents() {
|
||||
return IngestedTorrent.findAll({
|
||||
where: {
|
||||
processed: false,
|
||||
category: {
|
||||
[Op.or]: ['tv', 'movies']
|
||||
}
|
||||
},
|
||||
|
||||
});
|
||||
}
|
||||
|
||||
export function setIngestedTorrentsProcessed(ingestedTorrents) {
|
||||
return Promises.sequence(ingestedTorrents
|
||||
.map(ingestedTorrent => () => {
|
||||
ingestedTorrent.processed = true;
|
||||
return ingestedTorrent.save();
|
||||
}));
|
||||
}
|
||||
|
||||
export function getTorrentsWithoutSize() {
|
||||
return Torrent.findAll({
|
||||
where: literal(
|
||||
'exists (select 1 from files where files."infoHash" = torrent."infoHash" and files.size = 300000000)'),
|
||||
order: [
|
||||
['seeders', 'DESC']
|
||||
]
|
||||
});
|
||||
}
|
||||
|
||||
export function getUpdateSeedersTorrents(limit = 50) {
|
||||
const until = moment().subtract(7, 'days').format('YYYY-MM-DD');
|
||||
return Torrent.findAll({
|
||||
where: literal(`torrent."updatedAt" < '${until}'`),
|
||||
limit: limit,
|
||||
order: [
|
||||
['seeders', 'DESC'],
|
||||
['updatedAt', 'ASC']
|
||||
]
|
||||
});
|
||||
}
|
||||
|
||||
export function getUpdateSeedersNewTorrents(limit = 50) {
|
||||
const lastUpdate = moment().subtract(12, 'hours').format('YYYY-MM-DD');
|
||||
const createdAfter = moment().subtract(4, 'days').format('YYYY-MM-DD');
|
||||
return Torrent.findAll({
|
||||
where: literal(`torrent."updatedAt" < '${lastUpdate}' AND torrent."createdAt" > '${createdAfter}'`),
|
||||
limit: limit,
|
||||
order: [
|
||||
['seeders', 'ASC'],
|
||||
['updatedAt', 'ASC']
|
||||
]
|
||||
});
|
||||
}
|
||||
|
||||
export function getNoContentsTorrents() {
|
||||
return Torrent.findAll({
|
||||
where: { opened: false, seeders: { [Op.gte]: 1 } },
|
||||
limit: 500,
|
||||
order: [[fn('RANDOM')]]
|
||||
});
|
||||
}
|
||||
|
||||
export function createTorrent(torrent) {
|
||||
return Torrent.upsert(torrent)
|
||||
.then(() => createContents(torrent.infoHash, torrent.contents))
|
||||
.then(() => createSubtitles(torrent.infoHash, torrent.subtitles));
|
||||
}
|
||||
|
||||
export function setTorrentSeeders(torrent, seeders) {
|
||||
const where = torrent.infoHash
|
||||
? { infoHash: torrent.infoHash }
|
||||
: { provider: torrent.provider, torrentId: torrent.torrentId }
|
||||
return Torrent.update(
|
||||
{ seeders: seeders },
|
||||
{ where: where }
|
||||
);
|
||||
}
|
||||
|
||||
export function deleteTorrent(torrent) {
|
||||
return Torrent.destroy({ where: { infoHash: torrent.infoHash } })
|
||||
}
|
||||
|
||||
export function createFile(file) {
|
||||
if (file.id) {
|
||||
return (file.dataValues ? file.save() : File.upsert(file))
|
||||
.then(() => upsertSubtitles(file, file.subtitles));
|
||||
}
|
||||
if (file.subtitles && file.subtitles.length) {
|
||||
file.subtitles = file.subtitles.map(subtitle => ({ infoHash: file.infoHash, title: subtitle.path, ...subtitle }));
|
||||
}
|
||||
return File.create(file, { include: [Subtitle], ignoreDuplicates: true });
|
||||
}
|
||||
|
||||
export function getFiles(torrent) {
|
||||
return File.findAll({ where: { infoHash: torrent.infoHash } });
|
||||
}
|
||||
|
||||
export function getFilesBasedOnTitle(titleQuery) {
|
||||
return File.findAll({ where: { title: { [Op.regexp]: `${titleQuery}` } } });
|
||||
}
|
||||
|
||||
export function deleteFile(file) {
|
||||
return File.destroy({ where: { id: file.id } })
|
||||
}
|
||||
|
||||
export function createSubtitles(infoHash, subtitles) {
|
||||
if (subtitles && subtitles.length) {
|
||||
return Subtitle.bulkCreate(subtitles.map(subtitle => ({ infoHash, title: subtitle.path, ...subtitle })));
|
||||
}
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
export function upsertSubtitles(file, subtitles) {
|
||||
if (file.id && subtitles && subtitles.length) {
|
||||
return Promises.sequence(subtitles
|
||||
.map(subtitle => {
|
||||
subtitle.fileId = file.id;
|
||||
subtitle.infoHash = subtitle.infoHash || file.infoHash;
|
||||
subtitle.title = subtitle.title || subtitle.path;
|
||||
return subtitle;
|
||||
})
|
||||
.map(subtitle => () => subtitle.dataValues ? subtitle.save() : Subtitle.create(subtitle)));
|
||||
}
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
export function getSubtitles(torrent) {
|
||||
return Subtitle.findAll({ where: { infoHash: torrent.infoHash } });
|
||||
}
|
||||
|
||||
export function getUnassignedSubtitles() {
|
||||
return Subtitle.findAll({ where: { fileId: null } });
|
||||
}
|
||||
|
||||
export function createContents(infoHash, contents) {
|
||||
if (contents && contents.length) {
|
||||
return Content.bulkCreate(contents.map(content => ({ infoHash, ...content })), { ignoreDuplicates: true })
|
||||
.then(() => Torrent.update({ opened: true }, { where: { infoHash: infoHash }, silent: true }));
|
||||
}
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
export function getContents(torrent) {
|
||||
return Content.findAll({ where: { infoHash: torrent.infoHash } });
|
||||
}
|
||||
|
||||
export function getSkipTorrent(torrent) {
|
||||
return SkipTorrent.findByPk(torrent.infoHash)
|
||||
.then((result) => {
|
||||
if (!result) {
|
||||
throw new Error(`torrent not found: ${torrent.infoHash}`);
|
||||
}
|
||||
return result.dataValues;
|
||||
})
|
||||
}
|
||||
|
||||
export function createSkipTorrent(torrent) {
|
||||
return SkipTorrent.upsert({ infoHash: torrent.infoHash });
|
||||
}
|
||||
260
src/node/consumer/src/lib/repository/database_repository.ts
Normal file
260
src/node/consumer/src/lib/repository/database_repository.ts
Normal file
@@ -0,0 +1,260 @@
|
||||
import {PromiseHelpers} from '@helpers/promises_helpers';
|
||||
import {ILoggingService} from "@interfaces/logging_service";
|
||||
import {IContentCreationAttributes} from "@repository/interfaces/content_attributes";
|
||||
import {IDatabaseRepository} from "@repository/interfaces/database_repository";
|
||||
import {IFileAttributes, IFileCreationAttributes} from "@repository/interfaces/file_attributes";
|
||||
import {ISubtitleAttributes, ISubtitleCreationAttributes} from "@repository/interfaces/subtitle_attributes";
|
||||
import {ITorrentAttributes, ITorrentCreationAttributes} from "@repository/interfaces/torrent_attributes";
|
||||
import {Content} from "@repository/models/content";
|
||||
import {File} from "@repository/models/file";
|
||||
import {IngestedPage} from "@repository/models/ingestedPage";
|
||||
import {IngestedTorrent} from "@repository/models/ingestedTorrent";
|
||||
import {Provider} from "@repository/models/provider";
|
||||
import {SkipTorrent} from "@repository/models/skipTorrent";
|
||||
import {Subtitle} from "@repository/models/subtitle";
|
||||
import {Torrent} from "@repository/models/torrent";
|
||||
import {configurationService} from '@services/configuration_service';
|
||||
import {IocTypes} from "@setup/ioc_types";
|
||||
import {inject, injectable} from "inversify";
|
||||
import moment from 'moment';
|
||||
import {literal, Op, WhereOptions} from "sequelize";
|
||||
import {Model, Sequelize} from 'sequelize-typescript';
|
||||
|
||||
@injectable()
|
||||
export class DatabaseRepository implements IDatabaseRepository {
|
||||
@inject(IocTypes.ILoggingService) logger: ILoggingService;
|
||||
|
||||
private readonly database: Sequelize;
|
||||
|
||||
private models = [
|
||||
Torrent,
|
||||
Provider,
|
||||
File,
|
||||
Subtitle,
|
||||
Content,
|
||||
SkipTorrent,
|
||||
IngestedTorrent,
|
||||
IngestedPage];
|
||||
|
||||
constructor() {
|
||||
this.database = this.createDatabase();
|
||||
}
|
||||
|
||||
async connect(): Promise<void> {
|
||||
try {
|
||||
await this.database.sync({alter: configurationService.databaseConfig.AUTO_CREATE_AND_APPLY_MIGRATIONS});
|
||||
} catch (error) {
|
||||
this.logger.debug('Failed to sync database', error);
|
||||
this.logger.error('Failed syncing database');
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
async getProvider(provider: Provider): Promise<Provider> {
|
||||
try {
|
||||
const [result] = await Provider.findOrCreate({where: {name: {[Op.eq]: provider.name}}, defaults: provider});
|
||||
return result;
|
||||
} catch {
|
||||
return provider as Provider;
|
||||
}
|
||||
}
|
||||
|
||||
async getTorrent(torrent: ITorrentAttributes): Promise<Torrent | null> {
|
||||
const where = torrent.infoHash
|
||||
? {infoHash: torrent.infoHash}
|
||||
: {provider: torrent.provider, torrentId: torrent.torrentId};
|
||||
return await Torrent.findOne({where});
|
||||
}
|
||||
|
||||
async getTorrentsBasedOnTitle(titleQuery: string, type: string): Promise<Torrent[]> {
|
||||
return this.getTorrentsBasedOnQuery({
|
||||
title: {[Op.regexp]: `${titleQuery}`},
|
||||
type
|
||||
});
|
||||
}
|
||||
|
||||
async getTorrentsBasedOnQuery(where: WhereOptions<ITorrentAttributes>): Promise<Torrent[]> {
|
||||
return await Torrent.findAll({where});
|
||||
}
|
||||
|
||||
async getFilesBasedOnQuery(where: WhereOptions<IFileAttributes>): Promise<File[]> {
|
||||
return await File.findAll({where});
|
||||
}
|
||||
|
||||
async getTorrentsWithoutSize(): Promise<Torrent[]> {
|
||||
return await Torrent.findAll({
|
||||
where: literal(
|
||||
'exists (select 1 from files where files."infoHash" = torrent."infoHash" and files.size = 300000000)'),
|
||||
order: [
|
||||
['seeders', 'DESC']
|
||||
]
|
||||
});
|
||||
}
|
||||
|
||||
async getUpdateSeedersTorrents(limit = 50): Promise<Torrent[]> {
|
||||
const until = moment().subtract(7, 'days').format('YYYY-MM-DD');
|
||||
return await Torrent.findAll({
|
||||
where: literal(`torrent."updatedAt" < '${until}'`),
|
||||
limit: limit,
|
||||
order: [
|
||||
['seeders', 'DESC'],
|
||||
['updatedAt', 'ASC']
|
||||
]
|
||||
});
|
||||
}
|
||||
|
||||
async getUpdateSeedersNewTorrents(limit = 50): Promise<Torrent[]> {
|
||||
const lastUpdate = moment().subtract(12, 'hours').format('YYYY-MM-DD');
|
||||
const createdAfter = moment().subtract(4, 'days').format('YYYY-MM-DD');
|
||||
return await Torrent.findAll({
|
||||
where: literal(`torrent."updatedAt" < '${lastUpdate}' AND torrent."createdAt" > '${createdAfter}'`),
|
||||
limit: limit,
|
||||
order: [
|
||||
['seeders', 'ASC'],
|
||||
['updatedAt', 'ASC']
|
||||
]
|
||||
});
|
||||
}
|
||||
|
||||
async getNoContentsTorrents(): Promise<Torrent[]> {
|
||||
return await Torrent.findAll({
|
||||
where: {opened: false, seeders: {[Op.gte]: 1}},
|
||||
limit: 500,
|
||||
order: literal('random()')
|
||||
});
|
||||
}
|
||||
|
||||
async createTorrent(torrent: ITorrentCreationAttributes): Promise<void> {
|
||||
try {
|
||||
await Torrent.upsert(torrent);
|
||||
await this.createContents(torrent.infoHash, torrent.contents);
|
||||
await this.createSubtitles(torrent.infoHash, torrent.subtitles);
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to create torrent: ${torrent.infoHash}`);
|
||||
this.logger.debug("Error: ", error);
|
||||
}
|
||||
}
|
||||
|
||||
async setTorrentSeeders(torrent: ITorrentAttributes, seeders: number): Promise<[number]> {
|
||||
const where = torrent.infoHash
|
||||
? {infoHash: torrent.infoHash}
|
||||
: {provider: torrent.provider, torrentId: torrent.torrentId};
|
||||
|
||||
return await Torrent.update(
|
||||
{seeders: seeders},
|
||||
{where: where}
|
||||
);
|
||||
}
|
||||
|
||||
async deleteTorrent(infoHash: string): Promise<number> {
|
||||
return await Torrent.destroy({where: {infoHash: infoHash}});
|
||||
}
|
||||
|
||||
async createFile(file: IFileCreationAttributes): Promise<void> {
|
||||
try {
|
||||
const operatingFile = File.build(file);
|
||||
if (operatingFile.id) {
|
||||
if (operatingFile.dataValues) {
|
||||
await operatingFile.save();
|
||||
} else {
|
||||
await File.upsert(file);
|
||||
}
|
||||
await this.upsertSubtitles(operatingFile, file.subtitles);
|
||||
} else {
|
||||
if (operatingFile.subtitles && operatingFile.subtitles.length) {
|
||||
operatingFile.subtitles = operatingFile.subtitles.map(subtitle => {
|
||||
subtitle.title = subtitle.path || '';
|
||||
return subtitle;
|
||||
});
|
||||
}
|
||||
await File.create(file, {include: [Subtitle], ignoreDuplicates: true});
|
||||
}
|
||||
} catch (error) {
|
||||
this.logger.error(`Failed to create file: ${file.infoHash}`);
|
||||
this.logger.debug("Error: ", error);
|
||||
}
|
||||
}
|
||||
|
||||
async getFiles(infoHash: string): Promise<File[]> {
|
||||
return File.findAll({where: {infoHash: infoHash}});
|
||||
}
|
||||
|
||||
async getFilesBasedOnTitle(titleQuery: string): Promise<File[]> {
|
||||
return File.findAll({where: {title: {[Op.regexp]: `${titleQuery}`}}});
|
||||
}
|
||||
|
||||
async deleteFile(id: number): Promise<number> {
|
||||
return File.destroy({where: {id: id}});
|
||||
}
|
||||
|
||||
async createSubtitles(infoHash: string, subtitles: ISubtitleCreationAttributes[] | undefined): Promise<void | Model<ISubtitleAttributes, ISubtitleCreationAttributes>[]> {
|
||||
if (subtitles && subtitles.length) {
|
||||
return Subtitle.bulkCreate(subtitles.map(subtitle => ({...subtitle, infoHash: infoHash, title: subtitle.path})));
|
||||
}
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
async upsertSubtitles(file: File, subtitles: ISubtitleCreationAttributes[] | undefined): Promise<void> {
|
||||
if (file.id && subtitles && subtitles.length) {
|
||||
await PromiseHelpers.sequence(subtitles
|
||||
.map(subtitle => {
|
||||
subtitle.fileId = file.id;
|
||||
subtitle.infoHash = subtitle.infoHash || file.infoHash;
|
||||
subtitle.title = subtitle.title || subtitle.path || '';
|
||||
return subtitle;
|
||||
})
|
||||
.map(subtitle => async () => {
|
||||
const operatingInstance = Subtitle.build(subtitle);
|
||||
if (operatingInstance.dataValues) {
|
||||
await operatingInstance.save();
|
||||
} else {
|
||||
await Subtitle.create(subtitle);
|
||||
}
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
async getSubtitles(infoHash: string): Promise<Subtitle[]> {
|
||||
return Subtitle.findAll({where: {infoHash: infoHash}});
|
||||
}
|
||||
|
||||
async getUnassignedSubtitles(): Promise<Subtitle[]> {
|
||||
return Subtitle.findAll({where: {fileId: null}});
|
||||
}
|
||||
|
||||
async createContents(infoHash: string, contents: IContentCreationAttributes[] | undefined): Promise<void> {
|
||||
if (contents && contents.length) {
|
||||
await Content.bulkCreate(contents.map(content => ({...content, infoHash})), {ignoreDuplicates: true});
|
||||
await Torrent.update({opened: true}, {where: {infoHash: infoHash}, silent: true});
|
||||
}
|
||||
}
|
||||
|
||||
async getContents(infoHash: string): Promise<Content[]> {
|
||||
return Content.findAll({where: {infoHash: infoHash}});
|
||||
}
|
||||
|
||||
async getSkipTorrent(infoHash: string): Promise<SkipTorrent> {
|
||||
const result = await SkipTorrent.findByPk(infoHash);
|
||||
if (!result) {
|
||||
throw new Error(`torrent not found: ${infoHash}`);
|
||||
}
|
||||
return result.dataValues as SkipTorrent;
|
||||
}
|
||||
|
||||
async createSkipTorrent(torrent: ITorrentCreationAttributes): Promise<[SkipTorrent, boolean | null]> {
|
||||
return SkipTorrent.upsert({infoHash: torrent.infoHash});
|
||||
}
|
||||
|
||||
private createDatabase = (): Sequelize => {
|
||||
const newDatabase = new Sequelize(
|
||||
configurationService.databaseConfig.POSTGRES_URI,
|
||||
{
|
||||
logging: false
|
||||
}
|
||||
);
|
||||
|
||||
newDatabase.addModels(this.models);
|
||||
|
||||
return newDatabase;
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
import {Optional} from "sequelize";
|
||||
|
||||
export interface IContentAttributes {
|
||||
infoHash: string;
|
||||
fileIndex: number;
|
||||
path: string;
|
||||
size: number;
|
||||
}
|
||||
|
||||
export interface IContentCreationAttributes extends Optional<IContentAttributes, 'fileIndex' | 'size'> {
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
import {IContentCreationAttributes} from "@repository/interfaces/content_attributes";
|
||||
import {IFileAttributes, IFileCreationAttributes} from "@repository/interfaces/file_attributes";
|
||||
import {ISubtitleAttributes, ISubtitleCreationAttributes} from "@repository/interfaces/subtitle_attributes";
|
||||
import {ITorrentAttributes, ITorrentCreationAttributes} from "@repository/interfaces/torrent_attributes";
|
||||
import {Content} from "@repository/models/content";
|
||||
import {File} from "@repository/models/file";
|
||||
import {Provider} from "@repository/models/provider";
|
||||
import {SkipTorrent} from "@repository/models/skipTorrent";
|
||||
import {Subtitle} from "@repository/models/subtitle";
|
||||
import {Torrent} from "@repository/models/torrent";
|
||||
import {WhereOptions} from "sequelize";
|
||||
import {Model} from "sequelize-typescript";
|
||||
|
||||
export interface IDatabaseRepository {
|
||||
connect(): Promise<void>;
|
||||
|
||||
getProvider(provider: Provider): Promise<Provider>;
|
||||
|
||||
getTorrent(torrent: ITorrentAttributes): Promise<Torrent | null>;
|
||||
|
||||
getTorrentsBasedOnTitle(titleQuery: string, type: string): Promise<Torrent[]>;
|
||||
|
||||
getTorrentsBasedOnQuery(where: WhereOptions<ITorrentAttributes>): Promise<Torrent[]>;
|
||||
|
||||
getFilesBasedOnQuery(where: WhereOptions<IFileAttributes>): Promise<File[]>;
|
||||
|
||||
getTorrentsWithoutSize(): Promise<Torrent[]>;
|
||||
|
||||
getUpdateSeedersTorrents(limit: number): Promise<Torrent[]>;
|
||||
|
||||
getUpdateSeedersNewTorrents(limit: number): Promise<Torrent[]>;
|
||||
|
||||
getNoContentsTorrents(): Promise<Torrent[]>;
|
||||
|
||||
createTorrent(torrent: ITorrentCreationAttributes): Promise<void>;
|
||||
|
||||
setTorrentSeeders(torrent: ITorrentAttributes, seeders: number): Promise<[number]>;
|
||||
|
||||
deleteTorrent(infoHash: string): Promise<number>;
|
||||
|
||||
createFile(file: IFileCreationAttributes): Promise<void>;
|
||||
|
||||
getFiles(infoHash: string): Promise<File[]>;
|
||||
|
||||
getFilesBasedOnTitle(titleQuery: string): Promise<File[]>;
|
||||
|
||||
deleteFile(id: number): Promise<number>;
|
||||
|
||||
createSubtitles(infoHash: string, subtitles: ISubtitleCreationAttributes[]): Promise<void | Model<ISubtitleAttributes, ISubtitleCreationAttributes>[]>;
|
||||
|
||||
upsertSubtitles(file: File, subtitles: ISubtitleCreationAttributes[] | undefined): Promise<void>;
|
||||
|
||||
getSubtitles(infoHash: string): Promise<Subtitle[]>;
|
||||
|
||||
getUnassignedSubtitles(): Promise<Subtitle[]>;
|
||||
|
||||
createContents(infoHash: string, contents: IContentCreationAttributes[]): Promise<void>;
|
||||
|
||||
getContents(infoHash: string): Promise<Content[]>;
|
||||
|
||||
getSkipTorrent(infoHash: string): Promise<SkipTorrent>;
|
||||
|
||||
createSkipTorrent(torrent: ITorrentCreationAttributes): Promise<[SkipTorrent, boolean | null]>;
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import {IParseTorrentTitleResult} from "@interfaces/parse_torrent_title_result";
|
||||
import {ISubtitleAttributes} from "@repository/interfaces/subtitle_attributes";
|
||||
import {Optional} from "sequelize";
|
||||
|
||||
export interface IFileAttributes extends IParseTorrentTitleResult {
|
||||
id?: number;
|
||||
infoHash?: string;
|
||||
fileIndex?: number;
|
||||
title: string;
|
||||
size?: number;
|
||||
imdbId?: string;
|
||||
imdbSeason?: number;
|
||||
imdbEpisode?: number;
|
||||
kitsuId?: number;
|
||||
kitsuEpisode?: number;
|
||||
subtitles?: ISubtitleAttributes[];
|
||||
path?: string;
|
||||
isMovie?: boolean;
|
||||
}
|
||||
|
||||
export interface IFileCreationAttributes extends Optional<IFileAttributes, 'fileIndex' | 'size' | 'imdbId' | 'imdbSeason' | 'imdbEpisode' | 'kitsuId' | 'kitsuEpisode'> {
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
export interface IIngestedPageAttributes {
|
||||
url: string;
|
||||
}
|
||||
|
||||
export interface IIngestedPageCreationAttributes extends IIngestedPageAttributes {
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
import {Optional} from "sequelize";
|
||||
|
||||
export interface IIngestedTorrentAttributes {
|
||||
name: string;
|
||||
source: string;
|
||||
category: string;
|
||||
info_hash: string;
|
||||
size: string;
|
||||
seeders: number;
|
||||
leechers: number;
|
||||
imdb: string;
|
||||
processed: boolean;
|
||||
createdAt?: Date;
|
||||
}
|
||||
|
||||
export interface IIngestedTorrentCreationAttributes extends Optional<IIngestedTorrentAttributes, 'processed'> {
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
import {Optional} from "sequelize";
|
||||
|
||||
export interface IProviderAttributes {
|
||||
name: string;
|
||||
lastScraped: Date;
|
||||
lastScrapedId: string;
|
||||
}
|
||||
|
||||
export interface IProviderCreationAttributes extends Optional<IProviderAttributes, 'lastScraped' | 'lastScrapedId'> {
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
import {Optional} from "sequelize";
|
||||
|
||||
export interface ISkipTorrentAttributes {
|
||||
infoHash: string;
|
||||
}
|
||||
|
||||
export interface ISkipTorrentCreationAttributes extends Optional<ISkipTorrentAttributes, never> {
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
import {Optional} from "sequelize";
|
||||
|
||||
export interface ISubtitleAttributes {
|
||||
infoHash: string;
|
||||
fileIndex: number;
|
||||
fileId?: number | null;
|
||||
title: string;
|
||||
path: string;
|
||||
}
|
||||
|
||||
export interface ISubtitleCreationAttributes extends Optional<ISubtitleAttributes, 'fileId'> {
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import {IContentAttributes} from "@repository/interfaces/content_attributes";
|
||||
import {IFileAttributes} from "@repository/interfaces/file_attributes";
|
||||
import {ISubtitleAttributes} from "@repository/interfaces/subtitle_attributes";
|
||||
import {Optional} from "sequelize";
|
||||
|
||||
export interface ITorrentAttributes {
|
||||
infoHash: string;
|
||||
provider?: string | null;
|
||||
torrentId?: string;
|
||||
title?: string;
|
||||
size?: number;
|
||||
type?: string;
|
||||
uploadDate?: Date;
|
||||
seeders?: number;
|
||||
trackers?: string;
|
||||
languages?: string;
|
||||
resolution?: string;
|
||||
reviewed?: boolean;
|
||||
opened?: boolean;
|
||||
contents?: IContentAttributes[];
|
||||
files?: IFileAttributes[];
|
||||
subtitles?: ISubtitleAttributes[];
|
||||
}
|
||||
|
||||
export interface ITorrentCreationAttributes extends Optional<ITorrentAttributes, 'torrentId' | 'size' | 'seeders' | 'trackers' | 'languages' | 'resolution' | 'reviewed' | 'opened'> {
|
||||
}
|
||||
22
src/node/consumer/src/lib/repository/models/content.ts
Normal file
22
src/node/consumer/src/lib/repository/models/content.ts
Normal file
@@ -0,0 +1,22 @@
|
||||
import {IContentAttributes, IContentCreationAttributes} from "@repository/interfaces/content_attributes";
|
||||
import {Torrent} from "@repository/models/torrent";
|
||||
import {BelongsTo, Column, DataType, ForeignKey, Model, Table} from 'sequelize-typescript';
|
||||
|
||||
@Table({modelName: 'content', timestamps: false})
|
||||
export class Content extends Model<IContentAttributes, IContentCreationAttributes> {
|
||||
@Column({type: DataType.STRING(64), primaryKey: true, allowNull: false, onDelete: 'CASCADE'})
|
||||
@ForeignKey(() => Torrent)
|
||||
declare infoHash: string;
|
||||
|
||||
@Column({type: DataType.INTEGER, primaryKey: true, allowNull: false})
|
||||
declare fileIndex: number;
|
||||
|
||||
@Column({type: DataType.STRING(512), allowNull: false})
|
||||
declare path: string;
|
||||
|
||||
@Column({type: DataType.BIGINT})
|
||||
declare size: number;
|
||||
|
||||
@BelongsTo(() => Torrent, {constraints: false, foreignKey: 'infoHash'})
|
||||
torrent?: Torrent;
|
||||
}
|
||||
59
src/node/consumer/src/lib/repository/models/file.ts
Normal file
59
src/node/consumer/src/lib/repository/models/file.ts
Normal file
@@ -0,0 +1,59 @@
|
||||
import {IFileAttributes, IFileCreationAttributes} from "@repository/interfaces/file_attributes";
|
||||
import {Subtitle} from "@repository/models/subtitle";
|
||||
import {Torrent} from "@repository/models/torrent";
|
||||
import {BelongsTo, Column, DataType, ForeignKey, HasMany, Model, Table} from 'sequelize-typescript';
|
||||
|
||||
const indexes = [
|
||||
{
|
||||
unique: true,
|
||||
name: 'files_unique_file_constraint',
|
||||
fields: [
|
||||
'infoHash',
|
||||
'fileIndex',
|
||||
'imdbId',
|
||||
'imdbSeason',
|
||||
'imdbEpisode',
|
||||
'kitsuId',
|
||||
'kitsuEpisode'
|
||||
]
|
||||
},
|
||||
{unique: false, fields: ['imdbId', 'imdbSeason', 'imdbEpisode']},
|
||||
{unique: false, fields: ['kitsuId', 'kitsuEpisode']}
|
||||
];
|
||||
|
||||
@Table({modelName: 'file', timestamps: true, indexes: indexes})
|
||||
export class File extends Model<IFileAttributes, IFileCreationAttributes> {
|
||||
@Column({type: DataType.STRING(64), allowNull: false, onDelete: 'CASCADE'})
|
||||
@ForeignKey(() => Torrent)
|
||||
declare infoHash: string;
|
||||
|
||||
@Column({type: DataType.INTEGER})
|
||||
declare fileIndex: number;
|
||||
|
||||
@Column({type: DataType.STRING(512), allowNull: false})
|
||||
declare title: string;
|
||||
|
||||
@Column({type: DataType.BIGINT})
|
||||
declare size: number;
|
||||
|
||||
@Column({type: DataType.STRING(32)})
|
||||
declare imdbId: string;
|
||||
|
||||
@Column({type: DataType.INTEGER})
|
||||
declare imdbSeason: number;
|
||||
|
||||
@Column({type: DataType.INTEGER})
|
||||
declare imdbEpisode: number;
|
||||
|
||||
@Column({type: DataType.INTEGER})
|
||||
declare kitsuId: number;
|
||||
|
||||
@Column({type: DataType.INTEGER})
|
||||
declare kitsuEpisode: number;
|
||||
|
||||
@HasMany(() => Subtitle, {constraints: false, foreignKey: 'fileId'})
|
||||
declare subtitles?: Subtitle[];
|
||||
|
||||
@BelongsTo(() => Torrent, {constraints: false, foreignKey: 'infoHash'})
|
||||
torrent?: Torrent;
|
||||
}
|
||||
16
src/node/consumer/src/lib/repository/models/ingestedPage.ts
Normal file
16
src/node/consumer/src/lib/repository/models/ingestedPage.ts
Normal file
@@ -0,0 +1,16 @@
|
||||
import {IIngestedPageAttributes, IIngestedPageCreationAttributes} from "@repository/interfaces/ingested_page_attributes";
|
||||
import {Column, DataType, Model, Table} from 'sequelize-typescript';
|
||||
|
||||
const indexes = [
|
||||
{
|
||||
unique: true,
|
||||
name: 'ingested_page_unique_url_constraint',
|
||||
fields: ['url']
|
||||
}
|
||||
];
|
||||
|
||||
@Table({modelName: 'ingested_page', timestamps: true, indexes: indexes})
|
||||
export class IngestedPage extends Model<IIngestedPageAttributes, IIngestedPageCreationAttributes> {
|
||||
@Column({type: DataType.STRING(512), allowNull: false})
|
||||
declare url: string;
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
import {IIngestedTorrentAttributes, IIngestedTorrentCreationAttributes} from "@repository/interfaces/ingested_torrent_attributes";
|
||||
import {Column, DataType, Model, Table} from 'sequelize-typescript';
|
||||
|
||||
const indexes = [
|
||||
{
|
||||
unique: true,
|
||||
name: 'ingested_torrent_unique_source_info_hash_constraint',
|
||||
fields: ['source', 'info_hash']
|
||||
}
|
||||
];
|
||||
|
||||
@Table({modelName: 'ingested_torrent', timestamps: true, indexes: indexes})
|
||||
export class IngestedTorrent extends Model<IIngestedTorrentAttributes, IIngestedTorrentCreationAttributes> {
|
||||
@Column({type: DataType.STRING(512)})
|
||||
declare name: string;
|
||||
|
||||
@Column({type: DataType.STRING(512)})
|
||||
declare source: string;
|
||||
|
||||
@Column({type: DataType.STRING(32)})
|
||||
declare category: string;
|
||||
|
||||
@Column({type: DataType.STRING(64)})
|
||||
declare info_hash: string;
|
||||
|
||||
@Column({type: DataType.STRING(32)})
|
||||
declare size: string;
|
||||
|
||||
@Column({type: DataType.INTEGER})
|
||||
declare seeders: number;
|
||||
|
||||
@Column({type: DataType.INTEGER})
|
||||
declare leechers: number;
|
||||
|
||||
@Column({type: DataType.STRING(32)})
|
||||
declare imdb: string;
|
||||
|
||||
@Column({type: DataType.BOOLEAN, defaultValue: false})
|
||||
declare processed: boolean;
|
||||
}
|
||||
15
src/node/consumer/src/lib/repository/models/provider.ts
Normal file
15
src/node/consumer/src/lib/repository/models/provider.ts
Normal file
@@ -0,0 +1,15 @@
|
||||
import {IProviderAttributes, IProviderCreationAttributes} from "@repository/interfaces/provider_attributes";
|
||||
import {Column, DataType, Model, Table} from 'sequelize-typescript';
|
||||
|
||||
@Table({modelName: 'provider', timestamps: false})
|
||||
export class Provider extends Model<IProviderAttributes, IProviderCreationAttributes> {
|
||||
|
||||
@Column({type: DataType.STRING(32), primaryKey: true})
|
||||
declare name: string;
|
||||
|
||||
@Column({type: DataType.DATE})
|
||||
declare lastScraped: Date;
|
||||
|
||||
@Column({type: DataType.STRING(128)})
|
||||
declare lastScrapedId: string;
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
import {ISkipTorrentAttributes, ISkipTorrentCreationAttributes} from "@repository/interfaces/skip_torrent_attributes";
|
||||
import {Column, DataType, Model, Table} from 'sequelize-typescript';
|
||||
|
||||
@Table({modelName: 'skip_torrent', timestamps: false})
|
||||
export class SkipTorrent extends Model<ISkipTorrentAttributes, ISkipTorrentCreationAttributes> {
|
||||
|
||||
@Column({type: DataType.STRING(64), primaryKey: true})
|
||||
declare infoHash: string;
|
||||
}
|
||||
38
src/node/consumer/src/lib/repository/models/subtitle.ts
Normal file
38
src/node/consumer/src/lib/repository/models/subtitle.ts
Normal file
@@ -0,0 +1,38 @@
|
||||
import {ISubtitleAttributes, ISubtitleCreationAttributes} from "@repository/interfaces/subtitle_attributes";
|
||||
import {File} from "@repository/models/file";
|
||||
import {BelongsTo, Column, DataType, ForeignKey, Model, Table} from 'sequelize-typescript';
|
||||
|
||||
const indexes = [
|
||||
{
|
||||
unique: true,
|
||||
name: 'subtitles_unique_subtitle_constraint',
|
||||
fields: [
|
||||
'infoHash',
|
||||
'fileIndex',
|
||||
'fileId'
|
||||
]
|
||||
},
|
||||
{unique: false, fields: ['fileId']}
|
||||
];
|
||||
|
||||
@Table({modelName: 'subtitle', timestamps: false, indexes: indexes})
|
||||
export class Subtitle extends Model<ISubtitleAttributes, ISubtitleCreationAttributes> {
|
||||
|
||||
@Column({type: DataType.STRING(64), allowNull: false, onDelete: 'CASCADE'})
|
||||
declare infoHash: string;
|
||||
|
||||
@Column({type: DataType.INTEGER, allowNull: false})
|
||||
declare fileIndex: number;
|
||||
|
||||
@Column({type: DataType.BIGINT, allowNull: true, onDelete: 'SET NULL'})
|
||||
@ForeignKey(() => File)
|
||||
declare fileId?: number | null;
|
||||
|
||||
@Column({type: DataType.STRING(512), allowNull: false})
|
||||
declare title: string;
|
||||
|
||||
@BelongsTo(() => File, {constraints: false, foreignKey: 'fileId'})
|
||||
file?: File;
|
||||
|
||||
path?: string;
|
||||
}
|
||||
56
src/node/consumer/src/lib/repository/models/torrent.ts
Normal file
56
src/node/consumer/src/lib/repository/models/torrent.ts
Normal file
@@ -0,0 +1,56 @@
|
||||
import {ITorrentAttributes, ITorrentCreationAttributes} from "@repository/interfaces/torrent_attributes";
|
||||
import {Content} from "@repository/models/content";
|
||||
import {File} from "@repository/models/file";
|
||||
import {Subtitle} from "@repository/models/subtitle";
|
||||
import {Column, DataType, HasMany, Model, Table} from 'sequelize-typescript';
|
||||
|
||||
@Table({modelName: 'torrent', timestamps: true})
|
||||
|
||||
export class Torrent extends Model<ITorrentAttributes, ITorrentCreationAttributes> {
|
||||
@Column({type: DataType.STRING(64), primaryKey: true})
|
||||
declare infoHash: string;
|
||||
|
||||
@Column({type: DataType.STRING(32), allowNull: false})
|
||||
declare provider: string;
|
||||
|
||||
@Column({type: DataType.STRING(512)})
|
||||
declare torrentId: string;
|
||||
|
||||
@Column({type: DataType.STRING(512), allowNull: false})
|
||||
declare title: string;
|
||||
|
||||
@Column({type: DataType.BIGINT})
|
||||
declare size: number;
|
||||
|
||||
@Column({type: DataType.STRING(16), allowNull: false})
|
||||
declare type: string;
|
||||
|
||||
@Column({type: DataType.DATE, allowNull: false})
|
||||
declare uploadDate: Date;
|
||||
|
||||
@Column({type: DataType.SMALLINT})
|
||||
declare seeders: number;
|
||||
|
||||
@Column({type: DataType.STRING(8000)})
|
||||
declare trackers: string;
|
||||
|
||||
@Column({type: DataType.STRING(4096)})
|
||||
declare languages: string;
|
||||
|
||||
@Column({type: DataType.STRING(16)})
|
||||
declare resolution: string;
|
||||
|
||||
@Column({type: DataType.BOOLEAN, allowNull: false, defaultValue: false})
|
||||
declare reviewed: boolean;
|
||||
|
||||
@Column({type: DataType.BOOLEAN, allowNull: false, defaultValue: false})
|
||||
declare opened: boolean;
|
||||
|
||||
@HasMany(() => Content, {foreignKey: 'infoHash', constraints: false})
|
||||
contents?: Content[];
|
||||
|
||||
@HasMany(() => File, {foreignKey: 'infoHash', constraints: false})
|
||||
files?: File[];
|
||||
|
||||
subtitles?: Subtitle[];
|
||||
}
|
||||
111
src/node/consumer/src/lib/services/cache_service.ts
Normal file
111
src/node/consumer/src/lib/services/cache_service.ts
Normal file
@@ -0,0 +1,111 @@
|
||||
import {CacheType} from "@enums/cache_types";
|
||||
import {ICacheOptions} from "@interfaces/cache_options";
|
||||
import {ICacheService} from "@interfaces/cache_service";
|
||||
import {ILoggingService} from "@interfaces/logging_service";
|
||||
import {configurationService} from '@services/configuration_service';
|
||||
import {IocTypes} from "@setup/ioc_types";
|
||||
import {mongoDbStore} from '@tirke/node-cache-manager-mongodb'
|
||||
import {Cache, createCache, MemoryCache, memoryStore} from 'cache-manager';
|
||||
import {inject, injectable} from "inversify";
|
||||
|
||||
const GLOBAL_KEY_PREFIX = 'knightcrawler-consumer';
|
||||
const IMDB_ID_PREFIX = `${GLOBAL_KEY_PREFIX}|imdb_id`;
|
||||
const KITSU_ID_PREFIX = `${GLOBAL_KEY_PREFIX}|kitsu_id`;
|
||||
const METADATA_PREFIX = `${GLOBAL_KEY_PREFIX}|metadata`;
|
||||
const TRACKERS_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|trackers`;
|
||||
|
||||
const GLOBAL_TTL: number = Number(process.env.METADATA_TTL) || 7 * 24 * 60 * 60; // 7 days
|
||||
const MEMORY_TTL: number = Number(process.env.METADATA_TTL) || 2 * 60 * 60; // 2 hours
|
||||
const TRACKERS_TTL: number = 2 * 24 * 60 * 60; // 2 days
|
||||
|
||||
/* eslint-disable-next-line @typescript-eslint/no-explicit-any */
|
||||
export type CacheMethod = () => any;
|
||||
|
||||
@injectable()
|
||||
export class CacheService implements ICacheService {
|
||||
@inject(IocTypes.ILoggingService) private logger: ILoggingService;
|
||||
|
||||
private readonly memoryCache: MemoryCache | undefined;
|
||||
private readonly remoteCache: Cache | MemoryCache | undefined;
|
||||
|
||||
constructor() {
|
||||
if (configurationService.cacheConfig.NO_CACHE) {
|
||||
this.logger.info('Cache is disabled');
|
||||
return;
|
||||
}
|
||||
|
||||
this.memoryCache = this.initiateMemoryCache();
|
||||
this.remoteCache = this.initiateRemoteCache();
|
||||
}
|
||||
|
||||
cacheWrapImdbId(key: string, method: CacheMethod): Promise<CacheMethod> {
|
||||
return this.cacheWrap(CacheType.MongoDb, `${IMDB_ID_PREFIX}:${key}`, method, {ttl: GLOBAL_TTL});
|
||||
}
|
||||
|
||||
cacheWrapKitsuId(key: string, method: CacheMethod): Promise<CacheMethod> {
|
||||
return this.cacheWrap(CacheType.MongoDb, `${KITSU_ID_PREFIX}:${key}`, method, {ttl: GLOBAL_TTL});
|
||||
}
|
||||
|
||||
cacheWrapMetadata(id: string, method: CacheMethod): Promise<CacheMethod> {
|
||||
return this.cacheWrap(CacheType.Memory, `${METADATA_PREFIX}:${id}`, method, {ttl: MEMORY_TTL});
|
||||
}
|
||||
|
||||
cacheTrackers(method: CacheMethod): Promise<CacheMethod> {
|
||||
return this.cacheWrap(CacheType.Memory, `${TRACKERS_KEY_PREFIX}`, method, {ttl: TRACKERS_TTL});
|
||||
}
|
||||
|
||||
private initiateMemoryCache = (): MemoryCache =>
|
||||
createCache(memoryStore(), {
|
||||
ttl: MEMORY_TTL
|
||||
});
|
||||
|
||||
private initiateMongoCache = (): Cache => {
|
||||
const store = mongoDbStore({
|
||||
collectionName: configurationService.cacheConfig.COLLECTION_NAME,
|
||||
ttl: GLOBAL_TTL,
|
||||
url: configurationService.cacheConfig.MONGO_URI,
|
||||
mongoConfig: {
|
||||
socketTimeoutMS: 120000,
|
||||
appName: 'knightcrawler-consumer',
|
||||
}
|
||||
});
|
||||
|
||||
return createCache(store, {
|
||||
ttl: GLOBAL_TTL,
|
||||
});
|
||||
}
|
||||
|
||||
private initiateRemoteCache = (): Cache | undefined => {
|
||||
if (configurationService.cacheConfig.NO_CACHE) {
|
||||
this.logger.debug('Cache is disabled');
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return configurationService.cacheConfig.MONGO_URI ? this.initiateMongoCache() : this.initiateMemoryCache();
|
||||
}
|
||||
|
||||
private getCacheType = (cacheType: CacheType): MemoryCache | Cache | undefined => {
|
||||
switch (cacheType) {
|
||||
case CacheType.Memory:
|
||||
return this.memoryCache;
|
||||
case CacheType.MongoDb:
|
||||
return this.remoteCache;
|
||||
default:
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
private cacheWrap = async (cacheType: CacheType, key: string, method: CacheMethod, options: ICacheOptions): Promise<CacheMethod> => {
|
||||
const cache = this.getCacheType(cacheType);
|
||||
|
||||
if (configurationService.cacheConfig.NO_CACHE || !cache) {
|
||||
return method();
|
||||
}
|
||||
|
||||
this.logger.debug(`Cache type: ${cacheType}`);
|
||||
this.logger.debug(`Cache key: ${key}`);
|
||||
this.logger.debug(`Cache options: ${JSON.stringify(options)}`);
|
||||
|
||||
return cache.wrap(key, method, options.ttl);
|
||||
};
|
||||
}
|
||||
17
src/node/consumer/src/lib/services/configuration_service.ts
Normal file
17
src/node/consumer/src/lib/services/configuration_service.ts
Normal file
@@ -0,0 +1,17 @@
|
||||
import {cacheConfig} from "@models/configuration/cache_config";
|
||||
import {databaseConfig} from "@models/configuration/database_config";
|
||||
import {jobConfig} from "@models/configuration/job_config";
|
||||
import {metadataConfig} from "@models/configuration/metadata_config";
|
||||
import {rabbitConfig} from "@models/configuration/rabbit_config";
|
||||
import {torrentConfig} from "@models/configuration/torrent_config";
|
||||
import {trackerConfig} from "@models/configuration/tracker_config";
|
||||
|
||||
export const configurationService = {
|
||||
rabbitConfig: rabbitConfig,
|
||||
cacheConfig: cacheConfig,
|
||||
databaseConfig: databaseConfig,
|
||||
jobConfig: jobConfig,
|
||||
metadataConfig: metadataConfig,
|
||||
trackerConfig: trackerConfig,
|
||||
torrentConfig: torrentConfig
|
||||
};
|
||||
32
src/node/consumer/src/lib/services/logging_service.ts
Normal file
32
src/node/consumer/src/lib/services/logging_service.ts
Normal file
@@ -0,0 +1,32 @@
|
||||
import {ILoggingService} from "@interfaces/logging_service";
|
||||
import {injectable} from "inversify";
|
||||
import {Logger, pino} from "pino";
|
||||
|
||||
/* eslint-disable @typescript-eslint/no-explicit-any */
|
||||
@injectable()
|
||||
export class LoggingService implements ILoggingService {
|
||||
private readonly logger: Logger;
|
||||
|
||||
constructor() {
|
||||
this.logger = pino({
|
||||
level: process.env.LOG_LEVEL || 'info'
|
||||
});
|
||||
}
|
||||
|
||||
public info = (message: string, ...args: any[]): void => {
|
||||
this.logger.info(message, args);
|
||||
};
|
||||
|
||||
public error = (message: string, ...args: any[]): void => {
|
||||
this.logger.error(message, args);
|
||||
};
|
||||
|
||||
public debug = (message: string, ...args: any[]): void => {
|
||||
this.logger.debug(message, args);
|
||||
};
|
||||
|
||||
public warn = (message: string, ...args: any[]): void => {
|
||||
this.logger.warn(message, args);
|
||||
};
|
||||
}
|
||||
/* eslint-enable @typescript-eslint/no-explicit-any */
|
||||
234
src/node/consumer/src/lib/services/metadata_service.ts
Normal file
234
src/node/consumer/src/lib/services/metadata_service.ts
Normal file
@@ -0,0 +1,234 @@
|
||||
import {TorrentType} from '@enums/torrent_types';
|
||||
import {ICacheService} from "@interfaces/cache_service";
|
||||
import {ICinemetaJsonResponse} from "@interfaces/cinemeta_metadata";
|
||||
import {ICommonVideoMetadata} from "@interfaces/common_video_metadata";
|
||||
import {IKitsuCatalogJsonResponse} from "@interfaces/kitsu_catalog_metadata";
|
||||
import {IKitsuJsonResponse} from "@interfaces/kitsu_metadata";
|
||||
import {IMetaDataQuery} from "@interfaces/metadata_query";
|
||||
import {IMetadataResponse} from "@interfaces/metadata_response";
|
||||
import {IMetadataService} from "@interfaces/metadata_service";
|
||||
import {IocTypes} from "@setup/ioc_types";
|
||||
import axios from 'axios';
|
||||
import {ResultTypes, search} from 'google-sr';
|
||||
import {inject, injectable} from "inversify";
|
||||
import nameToImdb from 'name-to-imdb';
|
||||
|
||||
const CINEMETA_URL = 'https://v3-cinemeta.strem.io';
|
||||
const KITSU_URL = 'https://anime-kitsu.strem.fun';
|
||||
const TIMEOUT = 60000;
|
||||
|
||||
@injectable()
|
||||
export class MetadataService implements IMetadataService {
|
||||
@inject(IocTypes.ICacheService) private cacheService: ICacheService;
|
||||
|
||||
async getKitsuId(info: IMetaDataQuery): Promise<number | Error> {
|
||||
const title = this.escapeTitle(info.title!.replace(/\s\|\s.*/, ''));
|
||||
const year = info.year ? ` ${info.year}` : '';
|
||||
const season = info.season || 0 > 1 ? ` S${info.season}` : '';
|
||||
const key = `${title}${year}${season}`;
|
||||
const query = encodeURIComponent(key);
|
||||
|
||||
return this.cacheService.cacheWrapKitsuId(key,
|
||||
() => axios.get(`${KITSU_URL}/catalog/series/kitsu-anime-list/search=${query}.json`, {timeout: TIMEOUT})
|
||||
.then((response) => {
|
||||
const body = response.data as IKitsuCatalogJsonResponse;
|
||||
if (body && body.metas && body.metas.length) {
|
||||
return body.metas[0].id.replace('kitsu:', '');
|
||||
} else {
|
||||
throw new Error('No search results');
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
async getImdbId(info: IMetaDataQuery): Promise<string | undefined> {
|
||||
const name = this.escapeTitle(info.title!);
|
||||
const year = info.year || (info.date && info.date.slice(0, 4));
|
||||
const key = `${name}_${year || 'NA'}_${info.type}`;
|
||||
const query = `${name} ${year || ''} ${info.type} imdb`;
|
||||
const fallbackQuery = `${name} ${info.type} imdb`;
|
||||
const googleQuery = year ? query : fallbackQuery;
|
||||
|
||||
try {
|
||||
const imdbId = await this.cacheService.cacheWrapImdbId(key,
|
||||
() => this.getIMDbIdFromNameToImdb(name, info)
|
||||
);
|
||||
return imdbId && 'tt' + imdbId.replace(/tt0*([1-9][0-9]*)$/, '$1').padStart(7, '0');
|
||||
} catch (error) {
|
||||
const imdbIdFallback = await this.getIMDbIdFromGoogle(googleQuery);
|
||||
return imdbIdFallback && 'tt' + imdbIdFallback.toString().replace(/tt0*([1-9][0-9]*)$/, '$1').padStart(7, '0');
|
||||
}
|
||||
}
|
||||
|
||||
async getMetadata(query: IMetaDataQuery): Promise<IMetadataResponse | Error> {
|
||||
if (!query.id) {
|
||||
return Promise.reject("no valid id provided");
|
||||
}
|
||||
|
||||
const key = Number.isInteger(query.id) || query.id.toString().match(/^\d+$/) ? `kitsu:${query.id}` : query.id;
|
||||
const metaType = query.type === TorrentType.Movie ? TorrentType.Movie : TorrentType.Series;
|
||||
const isImdbId = Boolean(key.toString().match(/^tt\d+$/));
|
||||
|
||||
try {
|
||||
try {
|
||||
return await this.cacheService.cacheWrapMetadata(key.toString(), () => {
|
||||
switch (isImdbId) {
|
||||
case true:
|
||||
return this.requestMetadata(`${CINEMETA_URL}/meta/imdb/${key}.json`, this.handleCinemetaResponse);
|
||||
default:
|
||||
return this.requestMetadata(`${KITSU_URL}/meta/${metaType}/${key}.json`, this.handleKitsuResponse)
|
||||
}
|
||||
});
|
||||
} catch (e) {
|
||||
// try different type in case there was a mismatch
|
||||
const otherType = metaType === TorrentType.Movie ? TorrentType.Series : TorrentType.Movie;
|
||||
return this.requestMetadata(`${CINEMETA_URL}/meta/${otherType}/${key}.json`, this.handleCinemetaResponse)
|
||||
}
|
||||
} catch (error) {
|
||||
throw new Error(`failed metadata query ${key} due: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
async isEpisodeImdbId(imdbId: string | undefined): Promise<boolean> {
|
||||
if (!imdbId || !imdbId.toString().match(/^tt\d+$/)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await axios.get(`https://www.imdb.com/title/${imdbId}/`, {timeout: TIMEOUT});
|
||||
return response.data.includes('video.episode');
|
||||
} catch (error) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
escapeTitle(title: string): string {
|
||||
return title.toLowerCase()
|
||||
.normalize('NFKD') // normalize non-ASCII characters
|
||||
.replace(/[\u0300-\u036F]/g, '')
|
||||
.replace(/&/g, 'and')
|
||||
.replace(/[;, ~./]+/g, ' ') // replace dots, commas or underscores with spaces
|
||||
.replace(/[^\w \-()×+#@!'\u0400-\u04ff]+/g, '') // remove all non-alphanumeric chars
|
||||
.replace(/^\d{1,2}[.#\s]+(?=(?:\d+[.\s]*)?[\u0400-\u04ff])/i, '') // remove russian movie numbering
|
||||
.replace(/\s{2,}/, ' ') // replace multiple spaces
|
||||
.trim();
|
||||
}
|
||||
|
||||
private requestMetadata = async (url: string, handler: (body: unknown) => IMetadataResponse): Promise<IMetadataResponse> => {
|
||||
try {
|
||||
const response = await axios.get(url, {timeout: TIMEOUT});
|
||||
const body = response.data;
|
||||
return handler(body);
|
||||
} catch (error) {
|
||||
throw new Error(`HTTP error! status: ${error.response?.status}`);
|
||||
}
|
||||
};
|
||||
|
||||
private handleCinemetaResponse = (response: unknown): IMetadataResponse => {
|
||||
const body = response as ICinemetaJsonResponse
|
||||
|
||||
return ({
|
||||
imdbId: parseInt(body.meta?.id || '0'),
|
||||
type: body.meta?.type,
|
||||
title: body.meta?.name,
|
||||
year: parseInt(body.meta?.year || '0'),
|
||||
country: body.meta?.country,
|
||||
genres: body.meta?.genres,
|
||||
status: body.meta?.status,
|
||||
videos: body.meta?.videos
|
||||
? body.meta.videos.map(video => ({
|
||||
name: video.name,
|
||||
season: video.season,
|
||||
episode: video.episode,
|
||||
imdbSeason: video.season,
|
||||
imdbEpisode: video.episode,
|
||||
}))
|
||||
: [],
|
||||
episodeCount: body.meta?.videos
|
||||
? this.getEpisodeCount(body.meta.videos)
|
||||
: [],
|
||||
totalCount: body.meta?.videos
|
||||
? body.meta.videos.filter(
|
||||
entry => entry.season !== 0 && entry.episode !== 0
|
||||
).length
|
||||
: 0,
|
||||
});
|
||||
};
|
||||
|
||||
private handleKitsuResponse = (response: unknown): IMetadataResponse => {
|
||||
const body = response as IKitsuJsonResponse;
|
||||
|
||||
return ({
|
||||
kitsuId: parseInt(body.meta?.kitsu_id || '0'),
|
||||
type: body.meta?.type,
|
||||
title: body.meta?.name,
|
||||
year: parseInt(body.meta?.year || '0'),
|
||||
country: body.meta?.country,
|
||||
genres: body.meta?.genres,
|
||||
status: body.meta?.status,
|
||||
videos: body.meta?.videos
|
||||
? body.meta?.videos.map(video => ({
|
||||
name: video.title,
|
||||
season: video.season,
|
||||
episode: video.episode,
|
||||
kitsuId: video.id,
|
||||
kitsuEpisode: video.episode,
|
||||
released: video.released,
|
||||
}))
|
||||
: [],
|
||||
episodeCount: body.meta?.videos
|
||||
? this.getEpisodeCount(body.meta.videos)
|
||||
: [],
|
||||
totalCount: body.meta?.videos
|
||||
? body.meta.videos.filter(
|
||||
entry => entry.season !== 0 && entry.episode !== 0
|
||||
).length
|
||||
: 0,
|
||||
});
|
||||
};
|
||||
|
||||
private getEpisodeCount = (videos: ICommonVideoMetadata[]): number[] =>
|
||||
Object.values(
|
||||
videos
|
||||
.filter(entry => entry.season !== null && entry.season !== 0 && entry.episode !== 0)
|
||||
.sort((a, b) => (a.season || 0) - (b.season || 0))
|
||||
.reduce((map: Record<number, number>, next) => {
|
||||
if (next.season || next.season === 0) {
|
||||
map[next.season] = (map[next.season] || 0) + 1;
|
||||
}
|
||||
return map;
|
||||
}, {})
|
||||
);
|
||||
|
||||
private getIMDbIdFromNameToImdb = (name: string, info: IMetaDataQuery): Promise<string | Error> => {
|
||||
const {year} = info;
|
||||
const {type} = info;
|
||||
return new Promise((resolve, reject) => {
|
||||
nameToImdb({name, year, type}, function (err: Error, res: string) {
|
||||
if (res) {
|
||||
resolve(res);
|
||||
} else {
|
||||
reject(err || new Error('Failed IMDbId search'));
|
||||
}
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
private getIMDbIdFromGoogle = async (query: string): Promise<string | undefined> => {
|
||||
try {
|
||||
const searchResults = await search({query: query});
|
||||
for (const result of searchResults) {
|
||||
if (result.type === ResultTypes.SearchResult) {
|
||||
if (result.link.includes('imdb.com/title/')) {
|
||||
const match = result.link.match(/imdb\.com\/title\/(tt\d+)/);
|
||||
if (match) {
|
||||
return match[1];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
} catch (error) {
|
||||
throw new Error('Failed to find IMDb ID from Google search');
|
||||
}
|
||||
};
|
||||
}
|
||||
164
src/node/consumer/src/lib/services/torrent_download_service.ts
Normal file
164
src/node/consumer/src/lib/services/torrent_download_service.ts
Normal file
@@ -0,0 +1,164 @@
|
||||
import {ExtensionHelpers} from '@helpers/extension_helpers';
|
||||
import {ILoggingService} from "@interfaces/logging_service";
|
||||
import {IParsedTorrent} from "@interfaces/parsed_torrent";
|
||||
import {ITorrentDownloadService} from "@interfaces/torrent_download_service";
|
||||
import {ITorrentFileCollection} from "@interfaces/torrent_file_collection";
|
||||
import {IContentAttributes} from "@repository/interfaces/content_attributes";
|
||||
import {IFileAttributes} from "@repository/interfaces/file_attributes";
|
||||
import {ISubtitleAttributes} from "@repository/interfaces/subtitle_attributes";
|
||||
import {configurationService} from '@services/configuration_service';
|
||||
import {IocTypes} from "@setup/ioc_types";
|
||||
import {inject, injectable} from "inversify";
|
||||
import {encode} from 'magnet-uri';
|
||||
import {parse} from "parse-torrent-title";
|
||||
// eslint-disable-next-line import/no-extraneous-dependencies
|
||||
import * as torrentStream from "torrent-stream";
|
||||
import TorrentEngine = TorrentStream.TorrentEngine;
|
||||
import TorrentEngineOptions = TorrentStream.TorrentEngineOptions;
|
||||
|
||||
interface ITorrentFile {
|
||||
name: string;
|
||||
path: string;
|
||||
length: number;
|
||||
fileIndex: number;
|
||||
}
|
||||
|
||||
@injectable()
|
||||
export class TorrentDownloadService implements ITorrentDownloadService {
|
||||
@inject(IocTypes.ILoggingService) private logger: ILoggingService;
|
||||
|
||||
private engineOptions: TorrentEngineOptions = {
|
||||
connections: configurationService.torrentConfig.MAX_CONNECTIONS_PER_TORRENT,
|
||||
uploads: 0,
|
||||
verify: false,
|
||||
dht: false,
|
||||
tracker: true,
|
||||
};
|
||||
|
||||
async getTorrentFiles(torrent: IParsedTorrent, timeout: number = 30000): Promise<ITorrentFileCollection> {
|
||||
const torrentFiles: ITorrentFile[] = await this.filesFromTorrentStream(torrent, timeout);
|
||||
|
||||
const videos = this.filterVideos(torrent, torrentFiles);
|
||||
const subtitles = this.filterSubtitles(torrent, torrentFiles);
|
||||
const contents = this.createContent(torrent, torrentFiles);
|
||||
|
||||
return {
|
||||
contents: contents,
|
||||
videos: videos,
|
||||
subtitles: subtitles,
|
||||
};
|
||||
}
|
||||
|
||||
private filesFromTorrentStream = async (torrent: IParsedTorrent, timeout: number): Promise<ITorrentFile[]> => {
|
||||
if (!torrent.infoHash) {
|
||||
return Promise.reject(new Error("No infoHash..."));
|
||||
}
|
||||
const magnet = encode({infoHash: torrent.infoHash, announce: torrent.trackers!.split(',')});
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
this.logger.debug(`Adding torrent with infoHash ${torrent.infoHash} to torrent engine...`);
|
||||
|
||||
const timeoutId = setTimeout(() => {
|
||||
engine.destroy(() => {
|
||||
});
|
||||
reject(new Error('No available connections for torrent!'));
|
||||
}, timeout);
|
||||
|
||||
const engine: TorrentEngine = torrentStream.default(magnet, this.engineOptions);
|
||||
|
||||
engine.on("ready", () => {
|
||||
const files: ITorrentFile[] = engine.files.map((file, fileId) => ({
|
||||
fileIndex: fileId,
|
||||
length: file.length,
|
||||
name: file.name,
|
||||
path: file.path,
|
||||
}));
|
||||
|
||||
resolve(files);
|
||||
clearTimeout(timeoutId);
|
||||
engine.destroy(() => {
|
||||
});
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
private filterVideos = (torrent: IParsedTorrent, torrentFiles: ITorrentFile[]): IFileAttributes[] => {
|
||||
if (torrentFiles.length === 1 && !Number.isInteger(torrentFiles[0].fileIndex)) {
|
||||
return [this.mapTorrentFileToFileAttributes(torrent, torrentFiles[0])];
|
||||
}
|
||||
const videos = torrentFiles.filter(file => ExtensionHelpers.isVideo(file.path || ''));
|
||||
const maxSize = Math.max(...videos.map((video: ITorrentFile) => video.length));
|
||||
const minSampleRatio = videos.length <= 3 ? 3 : 10;
|
||||
const minAnimeExtraRatio = 5;
|
||||
const minRedundantRatio = videos.length <= 3 ? 30 : Number.MAX_VALUE;
|
||||
|
||||
const isSample = (video: ITorrentFile): boolean => video.path?.toString()?.match(/sample|bonus|promo/i) && maxSize / video.length > minSampleRatio || false;
|
||||
const isRedundant = (video: ITorrentFile): boolean => maxSize / video.length > minRedundantRatio;
|
||||
const isExtra = (video: ITorrentFile): boolean => /extras?\//i.test(video.path?.toString() || "");
|
||||
const isAnimeExtra = (video: ITorrentFile): boolean => {
|
||||
if (!video.path || !video.length) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return video.path.toString()?.match(/(?:\b|_)(?:NC)?(?:ED|OP|PV)(?:v?\d\d?)?(?:\b|_)/i)
|
||||
&& maxSize / parseInt(video.length.toString()) > minAnimeExtraRatio || false;
|
||||
};
|
||||
const isWatermark = (video: ITorrentFile): boolean => {
|
||||
if (!video.path || !video.length) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return video.path.toString()?.match(/^[A-Z-]+(?:\.[A-Z]+)?\.\w{3,4}$/)
|
||||
&& maxSize / parseInt(video.length.toString()) > minAnimeExtraRatio || false;
|
||||
}
|
||||
|
||||
return videos
|
||||
.filter(video => !isSample(video))
|
||||
.filter(video => !isExtra(video))
|
||||
.filter(video => !isAnimeExtra(video))
|
||||
.filter(video => !isRedundant(video))
|
||||
.filter(video => !isWatermark(video))
|
||||
.map(video => this.mapTorrentFileToFileAttributes(torrent, video));
|
||||
};
|
||||
|
||||
private filterSubtitles = (torrent: IParsedTorrent, torrentFiles: ITorrentFile[]): ISubtitleAttributes[] => torrentFiles.filter(file => ExtensionHelpers.isSubtitle(file.name || ''))
|
||||
.map(file => this.mapTorrentFileToSubtitleAttributes(torrent, file));
|
||||
|
||||
private createContent = (torrent: IParsedTorrent, torrentFiles: ITorrentFile[]): IContentAttributes[] => torrentFiles.map(file => this.mapTorrentFileToContentAttributes(torrent, file));
|
||||
|
||||
private mapTorrentFileToFileAttributes = (torrent: IParsedTorrent, file: ITorrentFile): IFileAttributes => {
|
||||
try {
|
||||
const videoFile: IFileAttributes = {
|
||||
title: file.name,
|
||||
size: file.length,
|
||||
fileIndex: file.fileIndex || 0,
|
||||
path: file.path,
|
||||
infoHash: torrent.infoHash?.toString(),
|
||||
imdbId: torrent.imdbId?.toString() || '',
|
||||
imdbSeason: torrent.season || 0,
|
||||
imdbEpisode: torrent.episode || 0,
|
||||
kitsuId: parseInt(torrent.kitsuId?.toString() || '0') || 0,
|
||||
kitsuEpisode: torrent.episode || 0,
|
||||
};
|
||||
|
||||
return {...videoFile, ...parse(file.name)};
|
||||
} catch (error) {
|
||||
throw new Error(`Error parsing file ${file.name} from torrent ${torrent.infoHash}: ${error}`);
|
||||
}
|
||||
};
|
||||
|
||||
private mapTorrentFileToSubtitleAttributes = (torrent: IParsedTorrent, file: ITorrentFile): ISubtitleAttributes => ({
|
||||
title: file.name,
|
||||
infoHash: torrent.infoHash,
|
||||
fileIndex: file.fileIndex,
|
||||
fileId: file.fileIndex,
|
||||
path: file.path,
|
||||
});
|
||||
|
||||
private mapTorrentFileToContentAttributes = (torrent: IParsedTorrent, file: ITorrentFile): IContentAttributes => ({
|
||||
infoHash: torrent.infoHash,
|
||||
fileIndex: file.fileIndex,
|
||||
path: file.path,
|
||||
size: file.length,
|
||||
});
|
||||
}
|
||||
288
src/node/consumer/src/lib/services/torrent_entries_service.ts
Normal file
288
src/node/consumer/src/lib/services/torrent_entries_service.ts
Normal file
@@ -0,0 +1,288 @@
|
||||
import {TorrentType} from '@enums/torrent_types';
|
||||
import {PromiseHelpers} from '@helpers/promises_helpers';
|
||||
import {ILoggingService} from "@interfaces/logging_service";
|
||||
import {IMetaDataQuery} from "@interfaces/metadata_query";
|
||||
import {IMetadataService} from "@interfaces/metadata_service";
|
||||
import {IParsedTorrent} from "@interfaces/parsed_torrent";
|
||||
import {ITorrentEntriesService} from "@interfaces/torrent_entries_service";
|
||||
import {ITorrentFileCollection} from "@interfaces/torrent_file_collection";
|
||||
import {ITorrentFileService} from "@interfaces/torrent_file_service";
|
||||
import {ITorrentSubtitleService} from "@interfaces/torrent_subtitle_service";
|
||||
import {IDatabaseRepository} from "@repository/interfaces/database_repository";
|
||||
import {IFileCreationAttributes} from "@repository/interfaces/file_attributes";
|
||||
import {ISubtitleAttributes} from "@repository/interfaces/subtitle_attributes";
|
||||
import {ITorrentAttributes, ITorrentCreationAttributes} from "@repository/interfaces/torrent_attributes";
|
||||
import {File} from "@repository/models/file";
|
||||
import {SkipTorrent} from "@repository/models/skipTorrent";
|
||||
import {Subtitle} from "@repository/models/subtitle";
|
||||
import {Torrent} from "@repository/models/torrent";
|
||||
import {IocTypes} from "@setup/ioc_types";
|
||||
import {inject, injectable} from "inversify";
|
||||
import {parse} from 'parse-torrent-title';
|
||||
|
||||
@injectable()
|
||||
export class TorrentEntriesService implements ITorrentEntriesService {
|
||||
@inject(IocTypes.IMetadataService) private metadataService: IMetadataService;
|
||||
@inject(IocTypes.ILoggingService) private logger: ILoggingService;
|
||||
@inject(IocTypes.ITorrentFileService) private fileService: ITorrentFileService;
|
||||
@inject(IocTypes.ITorrentSubtitleService) private subtitleService: ITorrentSubtitleService;
|
||||
@inject(IocTypes.IDatabaseRepository) private repository: IDatabaseRepository;
|
||||
|
||||
async createTorrentEntry(torrent: IParsedTorrent, overwrite = false): Promise<void> {
|
||||
if (!torrent.title) {
|
||||
this.logger.warn(`No title found for ${torrent.provider} [${torrent.infoHash}]`);
|
||||
return;
|
||||
}
|
||||
|
||||
const titleInfo = parse(torrent.title);
|
||||
|
||||
if (!torrent.imdbId && torrent.type !== TorrentType.Anime) {
|
||||
const imdbQuery = {
|
||||
title: titleInfo.title,
|
||||
year: titleInfo.year,
|
||||
type: torrent.type
|
||||
};
|
||||
torrent.imdbId = await this.metadataService.getImdbId(imdbQuery)
|
||||
.catch(() => undefined);
|
||||
}
|
||||
if (torrent.imdbId && torrent.imdbId.toString().length < 9) {
|
||||
// pad zeros to imdbId if missing
|
||||
torrent.imdbId = 'tt' + torrent.imdbId.toString().replace('tt', '').padStart(7, '0');
|
||||
}
|
||||
if (torrent.imdbId && torrent.imdbId.toString().length > 9 && torrent.imdbId.toString().startsWith('tt0')) {
|
||||
// sanitize imdbId from redundant zeros
|
||||
torrent.imdbId = torrent.imdbId.toString().replace(/tt0+([0-9]{7,})$/, 'tt$1');
|
||||
}
|
||||
if (!torrent.kitsuId && torrent.type === TorrentType.Anime) {
|
||||
const kitsuQuery = {
|
||||
title: titleInfo.title,
|
||||
year: titleInfo.year,
|
||||
season: titleInfo.season,
|
||||
};
|
||||
|
||||
await this.assignKitsuId(kitsuQuery, torrent);
|
||||
}
|
||||
|
||||
if (!torrent.imdbId && !torrent.kitsuId && !this.fileService.isPackTorrent(torrent)) {
|
||||
this.logger.warn(`imdbId or kitsuId not found: ${torrent.provider} ${torrent.title}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const fileCollection: ITorrentFileCollection = await this.fileService.parseTorrentFiles(torrent)
|
||||
.then((torrentContents: ITorrentFileCollection) => overwrite ? this.overwriteExistingFiles(torrent, torrentContents) : torrentContents)
|
||||
.then((torrentContents: ITorrentFileCollection) => this.subtitleService.assignSubtitles(torrentContents))
|
||||
.catch(error => {
|
||||
this.logger.warn(`Failed getting files for ${torrent.title}`, error.message);
|
||||
return {};
|
||||
});
|
||||
|
||||
if (!fileCollection.videos || !fileCollection.videos.length) {
|
||||
this.logger.warn(`no video files found for ${torrent.provider} [${torrent.infoHash}] ${torrent.title}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const newTorrent: ITorrentCreationAttributes = ({
|
||||
...torrent,
|
||||
contents: fileCollection.contents,
|
||||
subtitles: fileCollection.subtitles
|
||||
});
|
||||
|
||||
return this.repository.createTorrent(newTorrent)
|
||||
.then(() => PromiseHelpers.sequence(fileCollection.videos!.map(video => () => {
|
||||
const newVideo: IFileCreationAttributes = {...video, infoHash: video.infoHash, title: video.title};
|
||||
if (!newVideo.kitsuId) {
|
||||
newVideo.kitsuId = 0;
|
||||
}
|
||||
return this.repository.createFile(newVideo)
|
||||
})))
|
||||
.then(() => this.logger.info(`Created ${torrent.provider} entry for [${torrent.infoHash}] ${torrent.title}`));
|
||||
}
|
||||
|
||||
async createSkipTorrentEntry(torrent: ITorrentCreationAttributes): Promise<[SkipTorrent, boolean | null]> {
|
||||
return this.repository.createSkipTorrent(torrent);
|
||||
}
|
||||
|
||||
async getStoredTorrentEntry(torrent: Torrent): Promise<Torrent | SkipTorrent | null | undefined> {
|
||||
return this.repository.getSkipTorrent(torrent.infoHash)
|
||||
.catch(() => this.repository.getTorrent(torrent.dataValues))
|
||||
.catch(() => undefined);
|
||||
}
|
||||
|
||||
async checkAndUpdateTorrent(torrent: IParsedTorrent): Promise<boolean> {
|
||||
const query: ITorrentAttributes = {
|
||||
infoHash: torrent.infoHash,
|
||||
provider: torrent.provider,
|
||||
}
|
||||
|
||||
const existingTorrent = await this.repository.getTorrent(query).catch(() => undefined);
|
||||
|
||||
if (!existingTorrent) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (existingTorrent.provider === 'RARBG') {
|
||||
return true;
|
||||
}
|
||||
if (existingTorrent.provider === 'KickassTorrents' && torrent.provider) {
|
||||
existingTorrent.provider = torrent.provider;
|
||||
existingTorrent.torrentId = torrent.torrentId!;
|
||||
}
|
||||
|
||||
if (!existingTorrent.languages && torrent.languages && existingTorrent.provider !== 'RARBG') {
|
||||
existingTorrent.languages = torrent.languages;
|
||||
await existingTorrent.save();
|
||||
this.logger.debug(`Updated [${existingTorrent.infoHash}] ${existingTorrent.title} language to ${torrent.languages}`);
|
||||
}
|
||||
|
||||
return this.createTorrentContents(existingTorrent)
|
||||
.then(() => this.updateTorrentSeeders(existingTorrent.dataValues))
|
||||
.then(() => Promise.resolve(true))
|
||||
.catch(() => Promise.reject(false));
|
||||
}
|
||||
|
||||
async createTorrentContents(torrent: Torrent): Promise<void> {
|
||||
if (torrent.opened) {
|
||||
return;
|
||||
}
|
||||
|
||||
const storedVideos: File[] = await this.repository.getFiles(torrent.infoHash).catch(() => []);
|
||||
if (!storedVideos || !storedVideos.length) {
|
||||
return;
|
||||
}
|
||||
const notOpenedVideo = storedVideos.length === 1 && !Number.isInteger(storedVideos[0].fileIndex);
|
||||
const imdbId: string | undefined = PromiseHelpers.mostCommonValue(storedVideos.map(stored => stored.imdbId));
|
||||
const kitsuId: number = PromiseHelpers.mostCommonValue(storedVideos.map(stored => stored.kitsuId || 0));
|
||||
|
||||
const fileCollection: ITorrentFileCollection = await this.fileService.parseTorrentFiles(torrent)
|
||||
.then(torrentContents => notOpenedVideo ? torrentContents : {
|
||||
...torrentContents,
|
||||
videos: storedVideos.map(video => video.dataValues)
|
||||
})
|
||||
.then(torrentContents => this.subtitleService.assignSubtitles(torrentContents))
|
||||
.then(torrentContents => this.assignMetaIds(torrentContents, imdbId, kitsuId))
|
||||
.catch(error => {
|
||||
this.logger.warn(`Failed getting contents for [${torrent.infoHash}] ${torrent.title}`, error.message);
|
||||
return {};
|
||||
});
|
||||
|
||||
if (!fileCollection.contents || !fileCollection.contents.length) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (notOpenedVideo && fileCollection.videos?.length === 1) {
|
||||
// if both have a single video and stored one was not opened, update stored one to true metadata and use that
|
||||
storedVideos[0].fileIndex = fileCollection?.videos[0]?.fileIndex || 0;
|
||||
storedVideos[0].title = fileCollection.videos[0].title;
|
||||
storedVideos[0].size = fileCollection.videos[0].size || 0;
|
||||
const subtitles: ISubtitleAttributes[] = fileCollection.videos[0]?.subtitles || [];
|
||||
storedVideos[0].subtitles = subtitles.map(subtitle => Subtitle.build(subtitle));
|
||||
fileCollection.videos[0] = {...storedVideos[0], subtitles: subtitles};
|
||||
}
|
||||
// no videos available or more than one new videos were in the torrent
|
||||
const shouldDeleteOld = notOpenedVideo && fileCollection.videos?.every(video => !video.id) || false;
|
||||
|
||||
const newTorrent: ITorrentCreationAttributes = {
|
||||
...torrent,
|
||||
files: fileCollection.videos,
|
||||
contents: fileCollection.contents,
|
||||
subtitles: fileCollection.subtitles
|
||||
};
|
||||
|
||||
return this.repository.createTorrent(newTorrent)
|
||||
.then(() => {
|
||||
if (shouldDeleteOld) {
|
||||
this.logger.debug(`Deleting old video for [${torrent.infoHash}] ${torrent.title}`)
|
||||
return storedVideos[0].destroy();
|
||||
}
|
||||
return Promise.resolve();
|
||||
})
|
||||
.then(() => {
|
||||
const promises = fileCollection.videos!.map(video => {
|
||||
const newVideo: IFileCreationAttributes = {...video, infoHash: video.infoHash, title: video.title};
|
||||
return this.repository.createFile(newVideo);
|
||||
});
|
||||
return Promise.all(promises);
|
||||
})
|
||||
.then(() => this.logger.info(`Created contents for ${torrent.provider} [${torrent.infoHash}] ${torrent.title}`))
|
||||
.catch(error => this.logger.error(`Failed saving contents for [${torrent.infoHash}] ${torrent.title}`, error));
|
||||
}
|
||||
|
||||
async updateTorrentSeeders(torrent: ITorrentAttributes): Promise<[number]> {
|
||||
if (!(torrent.infoHash || (torrent.provider && torrent.torrentId)) || !Number.isInteger(torrent.seeders)) {
|
||||
return [0];
|
||||
}
|
||||
|
||||
if (torrent.seeders === undefined) {
|
||||
this.logger.warn(`Seeders not found for ${torrent.provider} [${torrent.infoHash}] ${torrent.title}`);
|
||||
return [0];
|
||||
}
|
||||
|
||||
|
||||
return this.repository.setTorrentSeeders(torrent, torrent.seeders)
|
||||
.catch(error => {
|
||||
this.logger.warn('Failed updating seeders:', error);
|
||||
return [0];
|
||||
});
|
||||
}
|
||||
|
||||
private assignKitsuId = async (kitsuQuery: IMetaDataQuery, torrent: IParsedTorrent): Promise<void> => {
|
||||
await this.metadataService.getKitsuId(kitsuQuery)
|
||||
.then((result: number | Error) => {
|
||||
if (typeof result === 'number') {
|
||||
torrent.kitsuId = result;
|
||||
} else {
|
||||
torrent.kitsuId = 0;
|
||||
}
|
||||
})
|
||||
.catch((error: Error) => {
|
||||
this.logger.debug(`Failed getting kitsuId for ${torrent.title}`, error.message);
|
||||
torrent.kitsuId = 0;
|
||||
});
|
||||
};
|
||||
|
||||
private assignMetaIds = (fileCollection: ITorrentFileCollection, imdbId: string | undefined, kitsuId: number): ITorrentFileCollection => {
|
||||
if (fileCollection && fileCollection.videos && fileCollection.videos.length) {
|
||||
fileCollection.videos.forEach(video => {
|
||||
video.imdbId = imdbId || '';
|
||||
video.kitsuId = kitsuId || 0;
|
||||
});
|
||||
}
|
||||
|
||||
return fileCollection;
|
||||
};
|
||||
|
||||
private overwriteExistingFiles = async (torrent: IParsedTorrent, torrentContents: ITorrentFileCollection): Promise<ITorrentFileCollection> => {
|
||||
const videos = torrentContents && torrentContents.videos;
|
||||
if (videos && videos.length) {
|
||||
const existingFiles = await this.repository.getFiles(torrent.infoHash)
|
||||
.then((existing) => existing.reduce<{ [key: number]: File[] }>((map, next) => {
|
||||
const fileIndex = next.fileIndex !== undefined ? next.fileIndex : null;
|
||||
if (fileIndex !== null) {
|
||||
map[fileIndex] = (map[fileIndex] || []).concat(next);
|
||||
}
|
||||
return map;
|
||||
}, {}))
|
||||
.catch(() => undefined);
|
||||
if (existingFiles && Object.keys(existingFiles).length) {
|
||||
const overwrittenVideos = videos
|
||||
.map(file => {
|
||||
const index = file.fileIndex !== undefined ? file.fileIndex : null;
|
||||
let mapping;
|
||||
if (index !== null) {
|
||||
mapping = videos.length === 1 && Object.keys(existingFiles).length === 1
|
||||
? Object.values(existingFiles)[0]
|
||||
: existingFiles[index];
|
||||
}
|
||||
if (mapping) {
|
||||
const originalFile = mapping.shift();
|
||||
return {id: originalFile!.id, ...file};
|
||||
}
|
||||
return file;
|
||||
});
|
||||
return {...torrentContents, videos: overwrittenVideos};
|
||||
}
|
||||
return torrentContents;
|
||||
}
|
||||
return Promise.reject(`No video files found for: ${torrent.title}`);
|
||||
};
|
||||
}
|
||||
733
src/node/consumer/src/lib/services/torrent_file_service.ts
Normal file
733
src/node/consumer/src/lib/services/torrent_file_service.ts
Normal file
@@ -0,0 +1,733 @@
|
||||
import {TorrentType} from '@enums/torrent_types';
|
||||
import {ExtensionHelpers} from '@helpers/extension_helpers';
|
||||
import {PromiseHelpers} from '@helpers/promises_helpers';
|
||||
import {ICommonVideoMetadata} from "@interfaces/common_video_metadata";
|
||||
import {ILoggingService} from "@interfaces/logging_service";
|
||||
import {IMetaDataQuery} from "@interfaces/metadata_query";
|
||||
import {IMetadataResponse} from "@interfaces/metadata_response";
|
||||
import {IMetadataService} from "@interfaces/metadata_service";
|
||||
import {IParsedTorrent} from "@interfaces/parsed_torrent";
|
||||
import {ITorrentDownloadService} from "@interfaces/torrent_download_service";
|
||||
import {ITorrentFileCollection} from "@interfaces/torrent_file_collection";
|
||||
import {ITorrentFileService} from "@interfaces/torrent_file_service";
|
||||
import {IContentAttributes} from "@repository/interfaces/content_attributes";
|
||||
import {IFileAttributes} from "@repository/interfaces/file_attributes";
|
||||
import {configurationService} from '@services/configuration_service';
|
||||
import {IocTypes} from "@setup/ioc_types";
|
||||
import Bottleneck from 'bottleneck';
|
||||
import {inject, injectable} from "inversify";
|
||||
import moment from 'moment';
|
||||
import {parse} from 'parse-torrent-title';
|
||||
|
||||
const MIN_SIZE: number = 5 * 1024 * 1024; // 5 MB
|
||||
const MULTIPLE_FILES_SIZE = 4 * 1024 * 1024 * 1024; // 4 GB
|
||||
|
||||
type SeasonEpisodeMap = Record<number, Record<number, ICommonVideoMetadata>>;
|
||||
|
||||
@injectable()
|
||||
export class TorrentFileService implements ITorrentFileService {
|
||||
@inject(IocTypes.IMetadataService) metadataService: IMetadataService;
|
||||
@inject(IocTypes.ITorrentDownloadService) torrentDownloadService: ITorrentDownloadService;
|
||||
@inject(IocTypes.ILoggingService) logger: ILoggingService;
|
||||
|
||||
private readonly imdb_limiter: Bottleneck = new Bottleneck({
|
||||
maxConcurrent: configurationService.metadataConfig.IMDB_CONCURRENT,
|
||||
minTime: configurationService.metadataConfig.IMDB_INTERVAL_MS
|
||||
});
|
||||
|
||||
async parseTorrentFiles(torrent: IParsedTorrent): Promise<ITorrentFileCollection> {
|
||||
if (!torrent.title) {
|
||||
return Promise.reject(new Error('Torrent title is missing'));
|
||||
}
|
||||
|
||||
if (!torrent.infoHash) {
|
||||
return Promise.reject(new Error('Torrent infoHash is missing'));
|
||||
}
|
||||
|
||||
const parsedTorrentName = parse(torrent.title);
|
||||
const query: IMetaDataQuery = {
|
||||
id: torrent.kitsuId || torrent.imdbId,
|
||||
type: torrent.type || TorrentType.Movie,
|
||||
};
|
||||
const metadata = await this.metadataService.getMetadata(query)
|
||||
.then(meta => Object.assign({}, meta))
|
||||
.catch(() => undefined);
|
||||
|
||||
if (metadata === undefined || metadata instanceof Error) {
|
||||
return Promise.reject(new Error('Failed to retrieve metadata'));
|
||||
}
|
||||
|
||||
if (torrent.type !== TorrentType.Anime && metadata && metadata.type && metadata.type !== torrent.type) {
|
||||
// it's actually a movie/series
|
||||
torrent.type = metadata.type;
|
||||
}
|
||||
|
||||
if (torrent.type === TorrentType.Movie && (!parsedTorrentName.seasons ||
|
||||
parsedTorrentName.season === 5 && [1, 5].includes(parsedTorrentName.episode || 0))) {
|
||||
return this.parseMovieFiles(torrent, metadata);
|
||||
}
|
||||
|
||||
return this.parseSeriesFiles(torrent, metadata)
|
||||
}
|
||||
|
||||
isPackTorrent(torrent: IParsedTorrent): boolean {
|
||||
if (torrent.isPack) {
|
||||
return true;
|
||||
}
|
||||
if (!torrent.title) {
|
||||
return false;
|
||||
}
|
||||
const parsedInfo = parse(torrent.title);
|
||||
if (torrent.type === TorrentType.Movie) {
|
||||
return parsedInfo.complete || typeof parsedInfo.year === 'string' || /movies/i.test(torrent.title);
|
||||
}
|
||||
|
||||
const hasMultipleEpisodes = Boolean(parsedInfo.complete || torrent.size || 0 > MULTIPLE_FILES_SIZE ||
|
||||
(parsedInfo.seasons && parsedInfo.seasons.length > 1) ||
|
||||
(parsedInfo.episodes && parsedInfo.episodes.length > 1) ||
|
||||
(parsedInfo.seasons && !parsedInfo.episodes));
|
||||
|
||||
const hasSingleEpisode: boolean = Boolean(Number.isInteger(parsedInfo.episode) || (!parsedInfo.episodes && parsedInfo.date));
|
||||
|
||||
return hasMultipleEpisodes && !hasSingleEpisode;
|
||||
}
|
||||
|
||||
private parseSeriesVideos = (torrent: IParsedTorrent, videos: IFileAttributes[]): IFileAttributes[] => {
|
||||
const parsedTorrentName = parse(torrent.title!);
|
||||
const hasMovies = parsedTorrentName.complete || !!torrent.title!.match(/movies?(?:\W|$)/i);
|
||||
const parsedVideos = videos.map(video => this.parseSeriesVideo(video));
|
||||
|
||||
return parsedVideos.map(video => ({...video, isMovie: this.isMovieVideo(torrent, video, parsedVideos, hasMovies)}));
|
||||
};
|
||||
|
||||
private parseMovieFiles = async (torrent: IParsedTorrent, metadata: IMetadataResponse): Promise<ITorrentFileCollection> => {
|
||||
const fileCollection: ITorrentFileCollection = await this.getMoviesTorrentContent(torrent);
|
||||
if (fileCollection.videos === undefined || fileCollection.videos.length === 0) {
|
||||
return {...fileCollection, videos: this.getDefaultFileEntries(torrent)};
|
||||
}
|
||||
|
||||
const filteredVideos = fileCollection.videos
|
||||
.filter(video => video.size! > MIN_SIZE)
|
||||
.filter(video => !this.isFeaturette(video));
|
||||
if (this.isSingleMovie(filteredVideos)) {
|
||||
const parsedVideos = filteredVideos.map(video => ({
|
||||
infoHash: torrent.infoHash,
|
||||
fileIndex: video.fileIndex,
|
||||
title: video.title || video.path || video.fileName || '',
|
||||
size: video.size || torrent.size,
|
||||
imdbId: torrent.imdbId?.toString() || metadata && metadata.imdbId?.toString(),
|
||||
kitsuId: parseInt(torrent.kitsuId?.toString() || metadata && metadata.kitsuId?.toString() || '0')
|
||||
}));
|
||||
return {...fileCollection, videos: parsedVideos};
|
||||
}
|
||||
|
||||
const parsedVideos = await PromiseHelpers.sequence(filteredVideos.map(video => () => this.isFeaturette(video)
|
||||
? Promise.resolve(video)
|
||||
: this.findMovieImdbId(video.title).then(imdbId => ({...video, imdbId: imdbId?.toString() || ''}))))
|
||||
.then(videos => videos.map((video: IFileAttributes) => ({
|
||||
infoHash: torrent.infoHash,
|
||||
fileIndex: video.fileIndex,
|
||||
title: video.title || video.path,
|
||||
size: video.size,
|
||||
imdbId: video.imdbId,
|
||||
})));
|
||||
return {...fileCollection, videos: parsedVideos};
|
||||
};
|
||||
|
||||
private parseSeriesFiles = async (torrent: IParsedTorrent, metadata: IMetadataResponse): Promise<ITorrentFileCollection> => {
|
||||
const fileCollection: ITorrentFileCollection = await this.getSeriesTorrentContent(torrent);
|
||||
if (fileCollection.videos === undefined || fileCollection.videos.length === 0) {
|
||||
return {...fileCollection, videos: this.getDefaultFileEntries(torrent)};
|
||||
}
|
||||
|
||||
const parsedVideos: IFileAttributes[] = await Promise.resolve(fileCollection.videos)
|
||||
.then(videos => videos.filter(video => videos?.length === 1 || video.size! > MIN_SIZE))
|
||||
.then(videos => this.parseSeriesVideos(torrent, videos))
|
||||
.then(videos => this.decomposeEpisodes(torrent, videos, metadata))
|
||||
.then(videos => this.assignKitsuOrImdbEpisodes(torrent, videos, metadata))
|
||||
.then(videos => Promise.all(videos.map(video => video.isMovie
|
||||
? this.mapSeriesMovie(torrent, video)
|
||||
: this.mapSeriesEpisode(torrent, video, videos))))
|
||||
.then(videos => videos
|
||||
.reduce((a, b) => a.concat(b), [])
|
||||
.map(video => this.isFeaturette(video) ? this.clearInfoFields(video) : video));
|
||||
return {...torrent.fileCollection, videos: parsedVideos};
|
||||
};
|
||||
|
||||
private getMoviesTorrentContent = async (torrent: IParsedTorrent): Promise<ITorrentFileCollection> => {
|
||||
const files = await this.torrentDownloadService.getTorrentFiles(torrent, configurationService.torrentConfig.TIMEOUT)
|
||||
.catch(error => {
|
||||
if (!this.isPackTorrent(torrent)) {
|
||||
const entries = this.getDefaultFileEntries(torrent);
|
||||
return {videos: entries, contents: [], subtitles: [], files: entries}
|
||||
}
|
||||
return Promise.reject(error);
|
||||
});
|
||||
|
||||
if (files.contents && files.contents.length && !files.videos?.length && this.isDiskTorrent(files.contents)) {
|
||||
files.videos = this.getDefaultFileEntries(torrent);
|
||||
}
|
||||
|
||||
return files;
|
||||
};
|
||||
|
||||
private getDefaultFileEntries = (torrent: IParsedTorrent): IFileAttributes[] => [{
|
||||
title: torrent.title!,
|
||||
path: torrent.title,
|
||||
size: torrent.size,
|
||||
fileIndex: 0,
|
||||
}];
|
||||
|
||||
private getSeriesTorrentContent = async (torrent: IParsedTorrent): Promise<ITorrentFileCollection> => this.torrentDownloadService.getTorrentFiles(torrent, configurationService.torrentConfig.TIMEOUT)
|
||||
.catch(error => {
|
||||
if (!this.isPackTorrent(torrent)) {
|
||||
return {videos: this.getDefaultFileEntries(torrent), subtitles: [], contents: []}
|
||||
}
|
||||
return Promise.reject(error);
|
||||
});
|
||||
|
||||
private mapSeriesEpisode = async (torrent: IParsedTorrent, file: IFileAttributes, files: IFileAttributes[]): Promise<IFileAttributes[]> => {
|
||||
if (!file.episodes && !file.episodes) {
|
||||
if (files.length === 1 || files.some(f => f.episodes || f.episodes) || parse(torrent.title!).seasons) {
|
||||
return Promise.resolve([{
|
||||
infoHash: torrent.infoHash,
|
||||
fileIndex: file.fileIndex,
|
||||
title: file.path || file.title,
|
||||
size: file.size,
|
||||
imdbId: torrent?.imdbId?.toString() || file?.imdbId?.toString() || '',
|
||||
}]);
|
||||
}
|
||||
return Promise.resolve([]);
|
||||
}
|
||||
const episodeIndexes = [...(file.episodes || file.episodes).keys()];
|
||||
return Promise.resolve(episodeIndexes.map((index) => ({
|
||||
infoHash: torrent.infoHash,
|
||||
fileIndex: file.fileIndex,
|
||||
title: file.path || file.title,
|
||||
size: file.size,
|
||||
imdbId: file?.imdbId?.toString() || torrent?.imdbId?.toString() || '',
|
||||
imdbSeason: file.season,
|
||||
season: file.season,
|
||||
imdbEpisode: file.episodes && file.episodes[index],
|
||||
episode: file.episodes && file.episodes[index],
|
||||
kitsuEpisode: file.episodes && file.episodes[index],
|
||||
episodes: file.episodes,
|
||||
kitsuId: parseInt(file.kitsuId?.toString() || torrent.kitsuId?.toString() || '0') || 0,
|
||||
})))
|
||||
};
|
||||
|
||||
private mapSeriesMovie = async (torrent: IParsedTorrent, file: IFileAttributes): Promise<IFileAttributes[]> => {
|
||||
const kitsuId = torrent.type === TorrentType.Anime ? await this.findMovieKitsuId(file)
|
||||
.then(result => {
|
||||
if (result instanceof Error) {
|
||||
this.logger.warn(`Failed to retrieve kitsuId due to error: ${result.message}`);
|
||||
return undefined;
|
||||
}
|
||||
return result;
|
||||
}) : undefined;
|
||||
|
||||
const imdbId = !kitsuId ? await this.findMovieImdbId(file) : undefined;
|
||||
|
||||
const query: IMetaDataQuery = {
|
||||
id: kitsuId || imdbId,
|
||||
type: TorrentType.Movie
|
||||
};
|
||||
|
||||
const metadataOrError = await this.metadataService.getMetadata(query);
|
||||
if (metadataOrError instanceof Error) {
|
||||
this.logger.warn(`Failed to retrieve metadata due to error: ${metadataOrError.message}`);
|
||||
// return default result or throw error, depending on your use case
|
||||
return [{
|
||||
infoHash: torrent.infoHash,
|
||||
fileIndex: file.fileIndex,
|
||||
title: file.path || file.title,
|
||||
size: file.size,
|
||||
imdbId: imdbId,
|
||||
kitsuId: parseInt(kitsuId?.toString() || '0') || 0,
|
||||
episodes: undefined,
|
||||
imdbSeason: undefined,
|
||||
imdbEpisode: undefined,
|
||||
kitsuEpisode: undefined
|
||||
}];
|
||||
}
|
||||
// at this point, TypeScript infers that metadataOrError is actually MetadataResponse
|
||||
const metadata = metadataOrError;
|
||||
const hasEpisode = metadata.videos && metadata.videos.length && (file.episode || metadata.videos.length === 1);
|
||||
const episodeVideo = hasEpisode && metadata.videos && metadata.videos[(file.episode || 1) - 1];
|
||||
return [{
|
||||
infoHash: torrent.infoHash,
|
||||
fileIndex: file.fileIndex,
|
||||
title: file.path || file.title,
|
||||
size: file.size,
|
||||
imdbId: metadata.imdbId?.toString() || imdbId || '',
|
||||
kitsuId: parseInt(metadata.kitsuId?.toString() || kitsuId?.toString() || '0') || 0,
|
||||
imdbSeason: episodeVideo && metadata.imdbId ? episodeVideo.season : undefined,
|
||||
imdbEpisode: episodeVideo && metadata.imdbId || episodeVideo && metadata.kitsuId ? episodeVideo.episode || episodeVideo.episode : undefined,
|
||||
kitsuEpisode: episodeVideo && metadata.imdbId || episodeVideo && metadata.kitsuId ? episodeVideo.episode || episodeVideo.episode : undefined,
|
||||
}];
|
||||
};
|
||||
|
||||
private decomposeEpisodes = async (torrent: IParsedTorrent, files: IFileAttributes[], metadata: IMetadataResponse = {episodeCount: []}): Promise<IFileAttributes[]> => {
|
||||
if (files.every(file => !file.episodes && !file.date)) {
|
||||
return files;
|
||||
}
|
||||
|
||||
this.preprocessEpisodes(files);
|
||||
|
||||
if (torrent.type === TorrentType.Anime && torrent.kitsuId) {
|
||||
if (this.needsCinemetaMetadataForAnime(files, metadata)) {
|
||||
// In some cases anime could be resolved to wrong kitsuId
|
||||
// because of imdb season naming/absolute per series naming/multiple seasons
|
||||
// So in these cases we need to fetch cinemeta based metadata and decompose episodes using that
|
||||
await this.updateToCinemetaMetadata(metadata);
|
||||
if (files.some(file => Number.isInteger(file.season))) {
|
||||
// sometimes multi season anime torrents don't include season 1 naming
|
||||
files
|
||||
.filter(file => !Number.isInteger(file.season) && file.episodes)
|
||||
.forEach(file => file.season = 1);
|
||||
}
|
||||
} else {
|
||||
// otherwise for anime type episodes are always absolute and for a single season
|
||||
files
|
||||
.filter(file => file.episodes && file.season !== 0)
|
||||
.forEach(file => file.season = 1);
|
||||
return files;
|
||||
}
|
||||
}
|
||||
|
||||
const sortedEpisodes = files
|
||||
.map(file => !file.isMovie && file.episodes || [])
|
||||
.reduce((a, b) => a.concat(b), [])
|
||||
.sort((a, b) => a - b);
|
||||
|
||||
if (this.isConcatSeasonAndEpisodeFiles(files, sortedEpisodes, metadata)) {
|
||||
this.decomposeConcatSeasonAndEpisodeFiles(files, metadata);
|
||||
} else if (this.isDateEpisodeFiles(files, metadata)) {
|
||||
this.decomposeDateEpisodeFiles(files, metadata);
|
||||
} else if (this.isAbsoluteEpisodeFiles(torrent, files, metadata)) {
|
||||
this.decomposeAbsoluteEpisodeFiles(torrent, files, metadata);
|
||||
}
|
||||
// decomposeEpisodeTitleFiles(torrent, files, metadata);
|
||||
|
||||
return files;
|
||||
};
|
||||
|
||||
private preprocessEpisodes = (files: IFileAttributes[]): void => {
|
||||
// reverse special episode naming when they named with 0 episode, ie. S02E00
|
||||
files
|
||||
.filter(file => Number.isInteger(file.season) && file.episode === 0)
|
||||
.forEach(file => {
|
||||
file.episode = file.season
|
||||
file.episodes = [file.season || 0];
|
||||
file.season = 0;
|
||||
})
|
||||
};
|
||||
|
||||
private isConcatSeasonAndEpisodeFiles = (files: IFileAttributes[], sortedEpisodes: number[], metadata: IMetadataResponse): boolean => {
|
||||
if (metadata.kitsuId !== undefined) {
|
||||
// anime does not use this naming scheme in 99% of cases;
|
||||
return false;
|
||||
}
|
||||
// decompose concat season and episode files (ex. 101=S01E01) in case:
|
||||
// 1. file has a season, but individual files are concatenated with that season (ex. path Season 5/511 - Prize
|
||||
// Fighters.avi)
|
||||
// 2. file does not have a season and the episode does not go out of range for the concat season
|
||||
// episode count
|
||||
const thresholdAbove = Math.max(Math.ceil(files.length * 0.05), 5);
|
||||
const thresholdSorted = Math.max(Math.ceil(files.length * 0.8), 8);
|
||||
const threshold = Math.max(Math.ceil(files.length * 0.8), 5);
|
||||
const sortedConcatEpisodes = sortedEpisodes
|
||||
.filter(ep => ep > 100)
|
||||
.filter(ep => metadata.episodeCount && metadata.episodeCount[this.div100(ep) - 1] < ep)
|
||||
.filter(ep => metadata.episodeCount && metadata.episodeCount[this.div100(ep) - 1] >= this.mod100(ep));
|
||||
const concatFileEpisodes = files
|
||||
.filter(file => !file.isMovie && file.episodes)
|
||||
.filter(file => !file.season || file.episodes?.every(ep => this.div100(ep) === file.season));
|
||||
const concatAboveTotalEpisodeCount = files
|
||||
.filter(file => !file.isMovie && file.episodes && file.episodes.every(ep => ep > 100))
|
||||
.filter(file => file.episodes?.every(ep => ep > metadata.totalCount!));
|
||||
return sortedConcatEpisodes.length >= thresholdSorted && concatFileEpisodes.length >= threshold
|
||||
|| concatAboveTotalEpisodeCount.length >= thresholdAbove;
|
||||
};
|
||||
|
||||
private isDateEpisodeFiles = (files: IFileAttributes[], metadata: IMetadataResponse): boolean => files.every(file => (!file.season || metadata.episodeCount && !metadata.episodeCount[file.season - 1]) && file.date);
|
||||
|
||||
private isAbsoluteEpisodeFiles = (torrent: IParsedTorrent, files: IFileAttributes[], metadata: IMetadataResponse): boolean => {
|
||||
const threshold = Math.ceil(files.length / 5);
|
||||
const isAnime = torrent.type === TorrentType.Anime && torrent.kitsuId;
|
||||
const nonMovieEpisodes = files.filter(file => !file.isMovie && file.episodes);
|
||||
const absoluteEpisodes = files
|
||||
.filter(file => file.season && file.episodes)
|
||||
.filter(file => file.episodes?.every(ep =>
|
||||
metadata.episodeCount && file.season && metadata.episodeCount[file.season - 1] < ep));
|
||||
return nonMovieEpisodes.every(file => !file.season)
|
||||
|| (isAnime && nonMovieEpisodes.every(file =>
|
||||
metadata.episodeCount && file.season && file.season > metadata.episodeCount.length))
|
||||
|| absoluteEpisodes.length >= threshold;
|
||||
};
|
||||
|
||||
private isNewEpisodeNotInMetadata = (torrent: IParsedTorrent, video: IFileAttributes, metadata: IMetadataResponse): boolean => {
|
||||
const isAnime = torrent.type === TorrentType.Anime && torrent.kitsuId;
|
||||
return !!(!isAnime && !video.isMovie && video.episodes && video.season !== 1
|
||||
&& metadata.status && /continuing|current/i.test(metadata.status)
|
||||
&& metadata.episodeCount && video.season && video.season >= metadata.episodeCount.length
|
||||
&& video.episodes.every(ep => metadata.episodeCount && video.season && ep > (metadata.episodeCount[video.season - 1] || 0)));
|
||||
};
|
||||
|
||||
private decomposeConcatSeasonAndEpisodeFiles = (files: IFileAttributes[], metadata: IMetadataResponse): void => {
|
||||
files
|
||||
.filter(file => file.episodes && file.season !== 0 && file.episodes.every(ep => ep > 100))
|
||||
.filter(file => file.episodes && metadata?.episodeCount &&
|
||||
((file.season || this.div100(file.episodes[0])) - 1) >= 0 &&
|
||||
metadata.episodeCount[(file.season || this.div100(file.episodes[0])) - 1] < 100)
|
||||
.filter(file => (file.season && file.episodes && file.episodes.every(ep => this.div100(ep) === file.season)) || !file.season)
|
||||
.forEach(file => {
|
||||
if (file.episodes) {
|
||||
file.season = this.div100(file.episodes[0]);
|
||||
file.episodes = file.episodes.map(ep => this.mod100(ep));
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
private decomposeAbsoluteEpisodeFiles = (torrent: IParsedTorrent, videos: IFileAttributes[], metadata: IMetadataResponse): void => {
|
||||
if (metadata.episodeCount?.length === 0) {
|
||||
videos
|
||||
.filter(file => !Number.isInteger(file.season) && file.episodes && !file.isMovie)
|
||||
.forEach(file => {
|
||||
file.season = 1;
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (!metadata.episodeCount) return;
|
||||
|
||||
videos
|
||||
.filter(file => file.episodes && !file.isMovie && file.season !== 0)
|
||||
.filter(file => !this.isNewEpisodeNotInMetadata(torrent, file, metadata))
|
||||
.filter(file => {
|
||||
if (!file.episodes || !metadata.episodeCount) return false;
|
||||
return !file.season || (metadata.episodeCount[file.season - 1] || 0) < file.episodes[0];
|
||||
})
|
||||
.forEach(file => {
|
||||
if (!file.episodes || !metadata.episodeCount) return;
|
||||
|
||||
let seasonIdx = metadata.episodeCount
|
||||
.map((_, i) => i)
|
||||
.find(i => metadata.episodeCount && file.episodes && metadata.episodeCount.slice(0, i + 1).reduce((a, b) => a + b) >= file.episodes[0]);
|
||||
|
||||
seasonIdx = (seasonIdx || 1 || metadata.episodeCount.length) - 1;
|
||||
|
||||
file.season = seasonIdx + 1;
|
||||
file.episodes = file.episodes
|
||||
.map(ep => ep - (metadata.episodeCount?.slice(0, seasonIdx).reduce((a, b) => a + b, 0) || 0));
|
||||
});
|
||||
};
|
||||
|
||||
private decomposeDateEpisodeFiles = (files: IFileAttributes[], metadata: IMetadataResponse): void => {
|
||||
if (!metadata || !metadata.videos || !metadata.videos.length) {
|
||||
return;
|
||||
}
|
||||
|
||||
const timeZoneOffset = this.getTimeZoneOffset(metadata.country);
|
||||
const offsetVideos: { [key: string]: ICommonVideoMetadata } = metadata.videos
|
||||
.reduce((map: { [key: string]: ICommonVideoMetadata }, video: ICommonVideoMetadata) => {
|
||||
const releaseDate = moment(video.released).utcOffset(timeZoneOffset).format('YYYY-MM-DD');
|
||||
map[releaseDate] = video;
|
||||
return map;
|
||||
}, {});
|
||||
|
||||
files
|
||||
.filter(file => file.date)
|
||||
.forEach(file => {
|
||||
const video = offsetVideos[file.date!];
|
||||
if (video) {
|
||||
file.season = video.season;
|
||||
file.episodes = [video.episode || 0];
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
private getTimeZoneOffset = (country: string | undefined): string => {
|
||||
switch (country) {
|
||||
case 'United States':
|
||||
case 'USA':
|
||||
return '-08:00';
|
||||
default:
|
||||
return '00:00';
|
||||
}
|
||||
};
|
||||
|
||||
private assignKitsuOrImdbEpisodes = (torrent: IParsedTorrent, files: IFileAttributes[], metadata: IMetadataResponse): IFileAttributes[] => {
|
||||
if (!metadata || !metadata.videos || !metadata.videos.length) {
|
||||
if (torrent.type === TorrentType.Anime) {
|
||||
// assign episodes as kitsu episodes for anime when no metadata available for imdb mapping
|
||||
files
|
||||
.filter(file => file.season && file.episodes)
|
||||
.forEach(file => {
|
||||
file.season = undefined;
|
||||
file.episodes = undefined;
|
||||
})
|
||||
if (metadata.type === TorrentType.Movie && files.every(file => !file.imdbId)) {
|
||||
// sometimes a movie has episode naming, thus not recognized as a movie and imdbId not assigned
|
||||
files.forEach(file => file.imdbId = metadata.imdbId?.toString());
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
const seriesMapping = metadata.videos
|
||||
.filter(video => video.season !== undefined && Number.isInteger(video.season) && video.episode !== undefined && Number.isInteger(video.episode))
|
||||
.reduce<SeasonEpisodeMap>((map, video) => {
|
||||
if (video.season !== undefined && video.episode !== undefined) {
|
||||
const episodeMap = map[video.season] || {};
|
||||
episodeMap[video.episode] = video;
|
||||
map[video.season] = episodeMap;
|
||||
}
|
||||
return map;
|
||||
}, {});
|
||||
|
||||
|
||||
if (metadata.videos.some(video => Number.isInteger(video.season)) || !metadata.imdbId) {
|
||||
files.filter(file => file && Number.isInteger(file.season) && file.episodes)
|
||||
.map(file => {
|
||||
const seasonMapping = file && file.season && seriesMapping[file.season] || null;
|
||||
const episodeMapping = seasonMapping && file && file.episodes && file.episodes[0] && seasonMapping[file.episodes[0]] || null;
|
||||
|
||||
if (episodeMapping && Number.isInteger(episodeMapping.season)) {
|
||||
file.imdbId = metadata.imdbId?.toString();
|
||||
file.season = episodeMapping.season;
|
||||
file.episodes = file.episodes && file.episodes.map(ep => (seasonMapping && seasonMapping[ep]) ? Number(seasonMapping[ep].episode) : 0);
|
||||
} else {
|
||||
file.season = undefined;
|
||||
file.episodes = undefined;
|
||||
}
|
||||
});
|
||||
} else if (metadata.videos.some(video => video.episode)) {
|
||||
// imdb episode info is base
|
||||
files
|
||||
.filter(file => Number.isInteger(file.season) && file.episodes)
|
||||
.forEach(file => {
|
||||
if (!file.season || !file.episodes) {
|
||||
return;
|
||||
}
|
||||
if (seriesMapping[file.season]) {
|
||||
|
||||
const seasonMapping = seriesMapping[file.season];
|
||||
file.imdbId = metadata.imdbId?.toString();
|
||||
file.kitsuId = seasonMapping[file.episodes[0]] && parseInt(seasonMapping[file.episodes[0]].id || '0') || 0;
|
||||
file.episodes = file.episodes.map(ep => seasonMapping[ep]?.episode)
|
||||
.filter((ep): ep is number => ep !== undefined);
|
||||
} else if (seriesMapping[file.season - 1]) {
|
||||
// sometimes a second season might be a continuation of the previous season
|
||||
const seasonMapping = seriesMapping[file.season - 1] as ICommonVideoMetadata;
|
||||
const episodes = Object.values(seasonMapping);
|
||||
const firstKitsuId = episodes.length && episodes[0];
|
||||
const differentTitlesCount = new Set(episodes.map(ep => ep.id)).size
|
||||
const skippedCount = episodes.filter(ep => ep.id === firstKitsuId).length;
|
||||
const emptyArray: number[] = [];
|
||||
const seasonEpisodes = files
|
||||
.filter((otherFile: IFileAttributes) => otherFile.season === file.season && otherFile.episodes)
|
||||
.reduce((a, b) => a.concat(b.episodes || []), emptyArray);
|
||||
const isAbsoluteOrder = seasonEpisodes.every(ep => ep > skippedCount && ep <= episodes.length)
|
||||
const isNormalOrder = seasonEpisodes.every(ep => ep + skippedCount <= episodes.length)
|
||||
if (differentTitlesCount >= 1 && (isAbsoluteOrder || isNormalOrder)) {
|
||||
const {season} = file;
|
||||
const [episode] = file.episodes;
|
||||
file.imdbId = metadata.imdbId?.toString();
|
||||
file.season = file.season - 1;
|
||||
file.episodes = file.episodes.map(ep => isAbsoluteOrder ? ep : ep + skippedCount);
|
||||
const currentEpisode = seriesMapping[season][episode];
|
||||
file.kitsuId = currentEpisode ? parseInt(currentEpisode.id || '0') : 0;
|
||||
if (typeof season === 'number' && Array.isArray(file.episodes)) {
|
||||
file.episodes = file.episodes.map(ep =>
|
||||
seriesMapping[season]
|
||||
&& seriesMapping[season][ep]
|
||||
&& seriesMapping[season][ep].episode
|
||||
|| ep);
|
||||
}
|
||||
}
|
||||
} else if (Object.values(seriesMapping).length === 1 && seriesMapping[1]) {
|
||||
// sometimes series might be named with sequel season but it's not a season on imdb and a new title
|
||||
// eslint-disable-next-line prefer-destructuring
|
||||
const seasonMapping = seriesMapping[1];
|
||||
file.imdbId = metadata.imdbId?.toString();
|
||||
file.season = 1;
|
||||
file.kitsuId = parseInt(seasonMapping[file.episodes[0]].id || '0') || 0;
|
||||
file.episodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].episode)
|
||||
.filter((ep): ep is number => ep !== undefined);
|
||||
}
|
||||
});
|
||||
}
|
||||
return files;
|
||||
};
|
||||
|
||||
private needsCinemetaMetadataForAnime = (files: IFileAttributes[], metadata: IMetadataResponse): boolean => {
|
||||
if (!metadata || !metadata.imdbId || !metadata.videos || !metadata.videos.length) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const seasons = metadata.videos
|
||||
.map(video => video.season)
|
||||
.filter((season): season is number => season !== null && season !== undefined);
|
||||
|
||||
// Using || 0 instead of || Number.MAX_VALUE to match previous logic
|
||||
const minSeason = Math.min(...seasons) || 0;
|
||||
const maxSeason = Math.max(...seasons) || 0;
|
||||
const differentSeasons = new Set(seasons.filter(season => Number.isInteger(season))).size;
|
||||
|
||||
const total = metadata.totalCount || Number.MAX_VALUE;
|
||||
|
||||
return differentSeasons > 1 || files
|
||||
.filter(file => !file.isMovie && file.episodes)
|
||||
.some(file => file.season || 0 < minSeason || file.season || 0 > maxSeason || file.episodes?.every(ep => ep > total));
|
||||
};
|
||||
|
||||
private updateToCinemetaMetadata = async (metadata: IMetadataResponse): Promise<IMetadataResponse> => {
|
||||
const query: IMetaDataQuery = {
|
||||
id: metadata.imdbId,
|
||||
type: metadata.type
|
||||
};
|
||||
|
||||
return await this.metadataService.getMetadata(query)
|
||||
.then((newMetadataOrError) => {
|
||||
if (newMetadataOrError instanceof Error) {
|
||||
// handle error
|
||||
this.logger.warn(`Failed ${metadata.imdbId} metadata cinemeta update due: ${newMetadataOrError.message}`);
|
||||
return metadata; // or throw newMetadataOrError to propagate error up the call stack
|
||||
}
|
||||
// At this point TypeScript infers newMetadataOrError to be of type MetadataResponse
|
||||
const newMetadata = newMetadataOrError;
|
||||
if (!newMetadata.videos || !newMetadata.videos.length) {
|
||||
return metadata;
|
||||
} else {
|
||||
metadata.videos = newMetadata.videos;
|
||||
metadata.episodeCount = newMetadata.episodeCount;
|
||||
metadata.totalCount = newMetadata.totalCount;
|
||||
return metadata;
|
||||
}
|
||||
})
|
||||
};
|
||||
|
||||
private findMovieImdbId = (title: IFileAttributes | string): Promise<string | undefined> => {
|
||||
const parsedTitle = typeof title === 'string' ? parse(title) : title;
|
||||
this.logger.debug(`Finding movie imdbId for ${title}`);
|
||||
return this.imdb_limiter.schedule(async () => {
|
||||
const imdbQuery = {
|
||||
title: parsedTitle.title,
|
||||
year: parsedTitle.year,
|
||||
type: TorrentType.Movie
|
||||
};
|
||||
try {
|
||||
return await this.metadataService.getImdbId(imdbQuery);
|
||||
} catch (e) {
|
||||
return undefined;
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
private findMovieKitsuId = async (title: IFileAttributes | string): Promise<number | Error | undefined> => {
|
||||
const parsedTitle = typeof title === 'string' ? parse(title) : title;
|
||||
const kitsuQuery = {
|
||||
title: parsedTitle.title,
|
||||
year: parsedTitle.year,
|
||||
season: parsedTitle.season,
|
||||
type: TorrentType.Movie
|
||||
};
|
||||
try {
|
||||
return await this.metadataService.getKitsuId(kitsuQuery);
|
||||
} catch (e) {
|
||||
return undefined;
|
||||
}
|
||||
};
|
||||
|
||||
private isDiskTorrent = (contents: IContentAttributes[]): boolean => contents.some(content => ExtensionHelpers.isDisk(content.path));
|
||||
|
||||
private isSingleMovie = (videos: IFileAttributes[]): boolean => videos.length === 1 ||
|
||||
(videos.length === 2 &&
|
||||
videos.find(v => /\b(?:part|disc|cd)[ ._-]?0?1\b|^0?1\.\w{2,4}$/i.test(v.path!)) &&
|
||||
videos.find(v => /\b(?:part|disc|cd)[ ._-]?0?2\b|^0?2\.\w{2,4}$/i.test(v.path!))) !== undefined;
|
||||
|
||||
private isFeaturette = (video: IFileAttributes): boolean => /featurettes?\/|extras-grym/i.test(video.path!);
|
||||
|
||||
private parseSeriesVideo = (video: IFileAttributes): IFileAttributes => {
|
||||
const videoInfo = parse(video.title);
|
||||
// the episode may be in a folder containing season number
|
||||
if (!Number.isInteger(videoInfo.season) && video.path?.includes('/')) {
|
||||
const folders = video.path?.split('/');
|
||||
const pathInfo = parse(folders[folders.length - 2]);
|
||||
videoInfo.season = pathInfo.season;
|
||||
}
|
||||
if (!Number.isInteger(videoInfo.season) && video.season) {
|
||||
videoInfo.season = video.season;
|
||||
}
|
||||
if (!Number.isInteger(videoInfo.season) && videoInfo.seasons && videoInfo.seasons.length > 1) {
|
||||
// in case single file was interpreted as having multiple seasons
|
||||
[videoInfo.season] = videoInfo.seasons;
|
||||
}
|
||||
if (!Number.isInteger(videoInfo.season) && video.path?.includes('/') && video.seasons
|
||||
&& video.seasons.length > 1) {
|
||||
// russian season are usually named with 'series name-2` i.e. Улицы разбитых фонарей-6/22. Одиночный выстрел.mkv
|
||||
const folderPathSeasonMatch = video.path?.match(/[\u0400-\u04ff]-(\d{1,2})(?=.*\/)/);
|
||||
videoInfo.season = folderPathSeasonMatch && parseInt(folderPathSeasonMatch[1], 10) || undefined;
|
||||
}
|
||||
// sometimes video file does not have correct date format as in torrent title
|
||||
if (!videoInfo.episodes && !videoInfo.date && video.date) {
|
||||
videoInfo.date = video.date;
|
||||
}
|
||||
// limit number of episodes in case of incorrect parsing
|
||||
if (videoInfo.episodes && videoInfo.episodes.length > 20) {
|
||||
videoInfo.episodes = [videoInfo.episodes[0]];
|
||||
[videoInfo.episode] = videoInfo.episodes;
|
||||
}
|
||||
// force episode to any found number if it was not parsed
|
||||
if (!videoInfo.episodes && !videoInfo.date) {
|
||||
const epMatcher = videoInfo.title.match(
|
||||
/(?<!season\W*|disk\W*|movie\W*|film\W*)(?:^|\W|_)(\d{1,4})(?:a|b|c|v\d)?(?:_|\W|$)(?!disk|movie|film)/i);
|
||||
videoInfo.episodes = epMatcher && [parseInt(epMatcher[1], 10)] || undefined;
|
||||
videoInfo.episode = videoInfo.episodes && videoInfo.episodes[0];
|
||||
}
|
||||
if (!videoInfo.episodes && !videoInfo.date) {
|
||||
const epMatcher = video.title.match(new RegExp(`(?:\\(${videoInfo.year}\\)|part)[._ ]?(\\d{1,3})(?:\\b|_)`, "i"));
|
||||
videoInfo.episodes = epMatcher && [parseInt(epMatcher[1], 10)] || undefined;
|
||||
videoInfo.episode = videoInfo.episodes && videoInfo.episodes[0];
|
||||
}
|
||||
|
||||
return {...video, ...videoInfo};
|
||||
};
|
||||
|
||||
private isMovieVideo = (torrent: IParsedTorrent, video: IFileAttributes, otherVideos: IFileAttributes[], hasMovies: boolean): boolean => {
|
||||
if (Number.isInteger(torrent.season) && Array.isArray(torrent.episodes)) {
|
||||
// not movie if video has season
|
||||
return false;
|
||||
}
|
||||
if (torrent.title?.match(/\b(?:\d+[ .]movie|movie[ .]\d+)\b/i)) {
|
||||
// movie if video explicitly has numbered movie keyword in the name, ie. 1 Movie or Movie 1
|
||||
return true;
|
||||
}
|
||||
if (!hasMovies && torrent.type !== TorrentType.Anime) {
|
||||
// not movie if torrent name does not contain movies keyword or is not a pack torrent and is not anime
|
||||
return false;
|
||||
}
|
||||
if (!torrent.episodes) {
|
||||
// movie if there's no episode info it could be a movie
|
||||
return true;
|
||||
}
|
||||
// movie if contains year info and there aren't more than 3 video with same title and year
|
||||
// as some series titles might contain year in it.
|
||||
return !!torrent.year
|
||||
&& otherVideos.length > 3
|
||||
&& otherVideos.filter(other => other.title === video.title && other.year === video.year).length < 3;
|
||||
};
|
||||
|
||||
private clearInfoFields = (video: IFileAttributes): IFileAttributes => {
|
||||
video.imdbId = undefined;
|
||||
video.imdbSeason = undefined;
|
||||
video.imdbEpisode = undefined;
|
||||
video.kitsuId = undefined;
|
||||
video.kitsuEpisode = undefined;
|
||||
return video;
|
||||
};
|
||||
|
||||
private div100 = (episode: number): number => (episode / 100 >> 0);
|
||||
|
||||
private mod100 = (episode: number): number => episode % 100;
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
import {TorrentType} from "@enums/torrent_types";
|
||||
import {ILoggingService} from "@interfaces/logging_service";
|
||||
import {IParsedTorrent} from "@interfaces/parsed_torrent";
|
||||
import {ITorrentEntriesService} from "@interfaces/torrent_entries_service";
|
||||
import {ITorrentProcessingService} from "@interfaces/torrent_processing_service";
|
||||
import {ITrackerService} from "@interfaces/tracker_service";
|
||||
import {IIngestedTorrentAttributes} from "@repository/interfaces/ingested_torrent_attributes";
|
||||
import {IocTypes} from "@setup/ioc_types";
|
||||
import {inject, injectable} from "inversify";
|
||||
|
||||
@injectable()
|
||||
export class TorrentProcessingService implements ITorrentProcessingService {
|
||||
@inject(IocTypes.ITorrentEntriesService) torrentEntriesService: ITorrentEntriesService;
|
||||
@inject(IocTypes.ILoggingService) logger: ILoggingService;
|
||||
@inject(IocTypes.ITrackerService) trackerService: ITrackerService;
|
||||
|
||||
async processTorrentRecord(torrent: IIngestedTorrentAttributes): Promise<void> {
|
||||
const {category} = torrent;
|
||||
const type = category === 'tv' ? TorrentType.Series : TorrentType.Movie;
|
||||
const torrentInfo: IParsedTorrent = await this.parseTorrent(torrent, type);
|
||||
|
||||
this.logger.info(`Processing torrent ${torrentInfo.title} with infoHash ${torrentInfo.infoHash}`);
|
||||
|
||||
if (await this.torrentEntriesService.checkAndUpdateTorrent(torrentInfo)) {
|
||||
return;
|
||||
}
|
||||
|
||||
return this.torrentEntriesService.createTorrentEntry(torrentInfo, false);
|
||||
}
|
||||
|
||||
private assignTorrentTrackers = async (): Promise<string> => {
|
||||
const trackers = await this.trackerService.getTrackers();
|
||||
return trackers.join(',');
|
||||
}
|
||||
|
||||
private parseTorrent = async (torrent: IIngestedTorrentAttributes, category: string): Promise<IParsedTorrent> => {
|
||||
const infoHash = torrent.info_hash?.trim().toLowerCase()
|
||||
return {
|
||||
title: torrent.name,
|
||||
torrentId: `${torrent.name}_${infoHash}`,
|
||||
infoHash: infoHash,
|
||||
seeders: 100,
|
||||
size: parseInt(torrent.size),
|
||||
uploadDate: torrent.createdAt,
|
||||
imdbId: this.parseImdbId(torrent),
|
||||
type: category,
|
||||
provider: torrent.source,
|
||||
trackers: await this.assignTorrentTrackers(),
|
||||
}
|
||||
};
|
||||
|
||||
private parseImdbId = (torrent: IIngestedTorrentAttributes): string | undefined => {
|
||||
if (torrent.imdb === undefined || torrent.imdb === null) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
return torrent.imdb;
|
||||
};
|
||||
}
|
||||
107
src/node/consumer/src/lib/services/torrent_subtitle_service.ts
Normal file
107
src/node/consumer/src/lib/services/torrent_subtitle_service.ts
Normal file
@@ -0,0 +1,107 @@
|
||||
import {ITorrentFileCollection} from "@interfaces/torrent_file_collection";
|
||||
import {ITorrentSubtitleService} from "@interfaces/torrent_subtitle_service";
|
||||
import {IFileAttributes} from "@repository/interfaces/file_attributes";
|
||||
import {ISubtitleAttributes} from "@repository/interfaces/subtitle_attributes";
|
||||
import {injectable} from "inversify";
|
||||
import {parse} from 'parse-torrent-title';
|
||||
|
||||
@injectable()
|
||||
export class TorrentSubtitleService implements ITorrentSubtitleService {
|
||||
assignSubtitles(fileCollection: ITorrentFileCollection): ITorrentFileCollection {
|
||||
if (fileCollection.videos && fileCollection.videos.length && fileCollection.subtitles && fileCollection.subtitles.length) {
|
||||
if (fileCollection.videos.length === 1) {
|
||||
const matchingSubtitles = fileCollection.subtitles.filter(subtitle =>
|
||||
this.mostProbableSubtitleVideos(subtitle, [fileCollection.videos[0]]).length > 0
|
||||
);
|
||||
fileCollection.videos[0].subtitles = matchingSubtitles;
|
||||
const nonMatchingSubtitles = fileCollection.subtitles.filter(subtitle =>
|
||||
!matchingSubtitles.includes(subtitle)
|
||||
);
|
||||
return {...fileCollection, subtitles: nonMatchingSubtitles};
|
||||
}
|
||||
|
||||
const parsedVideos = fileCollection.videos.map(video => this.parseVideo(video));
|
||||
const assignedSubs = fileCollection.subtitles.map(subtitle => ({
|
||||
subtitle,
|
||||
videos: this.mostProbableSubtitleVideos(subtitle, parsedVideos)
|
||||
}));
|
||||
const unassignedSubs = assignedSubs.filter(assignedSub => !assignedSub.videos).map(assignedSub => assignedSub.subtitle);
|
||||
|
||||
assignedSubs
|
||||
.filter(assignedSub => assignedSub.videos)
|
||||
.forEach(assignedSub => assignedSub.videos.forEach(video => video.subtitles = (video.subtitles || []).concat(assignedSub.subtitle)));
|
||||
return {...fileCollection, subtitles: unassignedSubs};
|
||||
}
|
||||
return fileCollection;
|
||||
}
|
||||
|
||||
private parseVideo = (video: IFileAttributes): IFileAttributes => {
|
||||
const fileName = video.title?.split('/')?.pop()?.replace(/\.(\w{2,4})$/, '') || '';
|
||||
const folderName = video.title?.replace(/\/?[^/]+$/, '') || '';
|
||||
return Object.assign(video, {
|
||||
fileName: fileName,
|
||||
folderName: folderName,
|
||||
...this.parseFilename(video.title.toString() || '')
|
||||
});
|
||||
}
|
||||
|
||||
private mostProbableSubtitleVideos = (subtitle: ISubtitleAttributes, parsedVideos: IFileAttributes[]): IFileAttributes[] => {
|
||||
const subTitle = (subtitle.title || subtitle.path)?.split('/')?.pop()?.replace(/\.(\w{2,4})$/, '') || '';
|
||||
const parsedSub = this.parsePath(subtitle.title || subtitle.path);
|
||||
const byFileName = parsedVideos.filter(video => subTitle.includes(video.title!));
|
||||
if (byFileName.length === 1) {
|
||||
return byFileName.map(v => v);
|
||||
}
|
||||
const byTitleSeasonEpisode = parsedVideos.filter(video => video.title === parsedSub.title
|
||||
&& parsedSub.seasons && parsedSub.episodes
|
||||
&& this.arrayEquals(video.seasons || [], parsedSub.seasons)
|
||||
&& this.arrayEquals(video.episodes || [], parsedSub.episodes));
|
||||
if (this.singleVideoFile(byTitleSeasonEpisode)) {
|
||||
return byTitleSeasonEpisode.map(v => v);
|
||||
}
|
||||
const bySeasonEpisode = parsedVideos.filter(video => parsedSub.seasons && parsedSub.episodes
|
||||
&& this.arrayEquals(video.seasons || [], parsedSub.seasons)
|
||||
&& this.arrayEquals(video.episodes || [], parsedSub.episodes));
|
||||
if (this.singleVideoFile(bySeasonEpisode)) {
|
||||
return bySeasonEpisode.map(v => v);
|
||||
}
|
||||
const byTitle = parsedVideos.filter(video => video.title && video.title === parsedSub.title);
|
||||
if (this.singleVideoFile(byTitle)) {
|
||||
return byTitle.map(v => v);
|
||||
}
|
||||
const byEpisode = parsedVideos.filter(video => parsedSub.episodes
|
||||
&& this.arrayEquals(video.episodes || [], parsedSub.episodes || []));
|
||||
if (this.singleVideoFile(byEpisode)) {
|
||||
return byEpisode.map(v => v);
|
||||
}
|
||||
const byInfoHash = parsedVideos.filter(video => video.infoHash === subtitle.infoHash);
|
||||
if (this.singleVideoFile(byInfoHash)) {
|
||||
return byInfoHash.map(v => v);
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
private singleVideoFile = (videos: IFileAttributes[]): boolean => {
|
||||
return new Set(videos.map(v => v.fileIndex)).size === 1;
|
||||
}
|
||||
|
||||
private parsePath = (path: string): IFileAttributes => {
|
||||
const pathParts = path.split('/').map(part => this.parseFilename(part));
|
||||
const parsedWithEpisode = pathParts.find(parsed => parsed.season && parsed.episodes);
|
||||
return parsedWithEpisode || pathParts[pathParts.length - 1];
|
||||
}
|
||||
|
||||
private parseFilename = (filename: string): IFileAttributes => {
|
||||
const parsedInfo = parse(filename)
|
||||
const titleEpisode = parsedInfo.title.match(/(\d+)$/);
|
||||
if (!parsedInfo.episodes && titleEpisode) {
|
||||
parsedInfo.episodes = [parseInt(titleEpisode[1], 10)];
|
||||
}
|
||||
return parsedInfo;
|
||||
}
|
||||
|
||||
private arrayEquals = <T>(array1: T[], array2: T[]): boolean => {
|
||||
if (!array1 || !array2) return array1 === array2;
|
||||
return array1.length === array2.length && array1.every((value, index) => value === array2[index])
|
||||
}
|
||||
}
|
||||
36
src/node/consumer/src/lib/services/tracker_service.ts
Normal file
36
src/node/consumer/src/lib/services/tracker_service.ts
Normal file
@@ -0,0 +1,36 @@
|
||||
import {ICacheService} from "@interfaces/cache_service";
|
||||
import {ILoggingService} from "@interfaces/logging_service";
|
||||
import {ITrackerService} from "@interfaces/tracker_service";
|
||||
import {configurationService} from '@services/configuration_service';
|
||||
import {IocTypes} from "@setup/ioc_types";
|
||||
import axios, {AxiosResponse} from 'axios';
|
||||
import {inject, injectable} from "inversify";
|
||||
|
||||
@injectable()
|
||||
export class TrackerService implements ITrackerService {
|
||||
@inject(IocTypes.ICacheService) cacheService: ICacheService;
|
||||
@inject(IocTypes.ILoggingService) logger: ILoggingService;
|
||||
|
||||
async getTrackers(): Promise<string[]> {
|
||||
return this.cacheService.cacheTrackers(this.downloadTrackers);
|
||||
}
|
||||
|
||||
private downloadTrackers = async (): Promise<string[]> => {
|
||||
const response: AxiosResponse<string> = await axios.get(configurationService.trackerConfig.TRACKERS_URL);
|
||||
const trackersListText: string = response.data;
|
||||
// Trackers are separated by a newline character
|
||||
let urlTrackers = trackersListText.split("\n");
|
||||
// remove blank lines
|
||||
urlTrackers = urlTrackers.filter(line => line.trim() !== '');
|
||||
|
||||
if (!configurationService.trackerConfig.UDP_ENABLED) {
|
||||
// remove any udp trackers
|
||||
urlTrackers = urlTrackers.filter(line => !line.startsWith('udp://'));
|
||||
|
||||
}
|
||||
|
||||
this.logger.info(`Trackers updated at ${Date.now()}: ${urlTrackers.length} trackers`);
|
||||
|
||||
return urlTrackers;
|
||||
};
|
||||
}
|
||||
@@ -1,82 +0,0 @@
|
||||
import { decode } from 'magnet-uri';
|
||||
import torrentStream from 'torrent-stream';
|
||||
import { torrentConfig } from './config.js';
|
||||
import {isSubtitle, isVideo} from './extension.js';
|
||||
|
||||
export async function torrentFiles(torrent, timeout) {
|
||||
return filesFromTorrentStream(torrent, timeout)
|
||||
.then(files => ({
|
||||
contents: files,
|
||||
videos: filterVideos(files),
|
||||
subtitles: filterSubtitles(files)
|
||||
}));
|
||||
}
|
||||
|
||||
async function filesFromTorrentStream(torrent, timeout) {
|
||||
return filesAndSizeFromTorrentStream(torrent, timeout).then(result => result.files);
|
||||
}
|
||||
|
||||
const engineOptions = {
|
||||
connections: torrentConfig.MAX_CONNECTIONS_PER_TORRENT,
|
||||
uploads: 0,
|
||||
verify: false,
|
||||
dht: false,
|
||||
tracker: true
|
||||
}
|
||||
|
||||
function filesAndSizeFromTorrentStream(torrent, timeout = 30000) {
|
||||
if (!torrent.infoHash) {
|
||||
return Promise.reject(new Error("no infoHash..."));
|
||||
}
|
||||
const magnet = decode.encode({ infoHash: torrent.infoHash, announce: torrent.trackers });
|
||||
return new Promise((resolve, rejected) => {
|
||||
const timeoutId = setTimeout(() => {
|
||||
engine.destroy();
|
||||
rejected(new Error('No available connections for torrent!'));
|
||||
}, timeout);
|
||||
|
||||
const engine = new torrentStream(magnet, engineOptions);
|
||||
|
||||
engine.ready(() => {
|
||||
const files = engine.files
|
||||
.map((file, fileId) => ({
|
||||
fileIndex: fileId,
|
||||
name: file.name,
|
||||
path: file.path.replace(/^[^/]+\//, ''),
|
||||
size: file.length
|
||||
}));
|
||||
const size = engine.torrent.length;
|
||||
resolve({ files, size });
|
||||
engine.destroy();
|
||||
clearTimeout(timeoutId);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function filterVideos(files) {
|
||||
if (files.length === 1 && !Number.isInteger(files[0].fileIndex)) {
|
||||
return files;
|
||||
}
|
||||
const videos = files.filter(file => isVideo(file.path));
|
||||
const maxSize = Math.max(...videos.map(video => video.size));
|
||||
const minSampleRatio = videos.length <= 3 ? 3 : 10;
|
||||
const minAnimeExtraRatio = 5;
|
||||
const minRedundantRatio = videos.length <= 3 ? 30 : Number.MAX_VALUE;
|
||||
const isSample = video => video.path.match(/sample|bonus|promo/i) && maxSize / parseInt(video.size) > minSampleRatio;
|
||||
const isRedundant = video => maxSize / parseInt(video.size) > minRedundantRatio;
|
||||
const isExtra = video => video.path.match(/extras?\//i);
|
||||
const isAnimeExtra = video => video.path.match(/(?:\b|_)(?:NC)?(?:ED|OP|PV)(?:v?\d\d?)?(?:\b|_)/i)
|
||||
&& maxSize / parseInt(video.size) > minAnimeExtraRatio;
|
||||
const isWatermark = video => video.path.match(/^[A-Z-]+(?:\.[A-Z]+)?\.\w{3,4}$/)
|
||||
&& maxSize / parseInt(video.size) > minAnimeExtraRatio
|
||||
return videos
|
||||
.filter(video => !isSample(video))
|
||||
.filter(video => !isExtra(video))
|
||||
.filter(video => !isAnimeExtra(video))
|
||||
.filter(video => !isRedundant(video))
|
||||
.filter(video => !isWatermark(video));
|
||||
}
|
||||
|
||||
function filterSubtitles(files) {
|
||||
return files.filter(file => isSubtitle(file.path));
|
||||
}
|
||||
@@ -1,173 +0,0 @@
|
||||
import { parse } from 'parse-torrent-title';
|
||||
import { getImdbId, getKitsuId } from './metadata.js';
|
||||
import { isPackTorrent } from './parseHelper.js';
|
||||
import * as Promises from './promises.js';
|
||||
import * as repository from './repository.js';
|
||||
import { parseTorrentFiles } from './torrentFiles.js';
|
||||
import { assignSubtitles } from './torrentSubtitles.js';
|
||||
import { TorrentType } from './types.js';
|
||||
import {logger} from "./logger.js";
|
||||
|
||||
export async function createTorrentEntry(torrent, overwrite = false) {
|
||||
const titleInfo = parse(torrent.title);
|
||||
|
||||
if (!torrent.imdbId && torrent.type !== TorrentType.ANIME) {
|
||||
torrent.imdbId = await getImdbId(titleInfo, torrent.type)
|
||||
.catch(() => undefined);
|
||||
}
|
||||
if (torrent.imdbId && torrent.imdbId.length < 9) {
|
||||
// pad zeros to imdbId if missing
|
||||
torrent.imdbId = 'tt' + torrent.imdbId.replace('tt', '').padStart(7, '0');
|
||||
}
|
||||
if (torrent.imdbId && torrent.imdbId.length > 9 && torrent.imdbId.startsWith('tt0')) {
|
||||
// sanitize imdbId from redundant zeros
|
||||
torrent.imdbId = torrent.imdbId.replace(/tt0+([0-9]{7,})$/, 'tt$1');
|
||||
}
|
||||
if (!torrent.kitsuId && torrent.type === TorrentType.ANIME) {
|
||||
torrent.kitsuId = await getKitsuId(titleInfo)
|
||||
.catch(() => undefined);
|
||||
}
|
||||
|
||||
if (!torrent.imdbId && !torrent.kitsuId && !isPackTorrent(torrent)) {
|
||||
logger.warn(`imdbId or kitsuId not found: ${torrent.provider} ${torrent.title}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const { contents, videos, subtitles } = await parseTorrentFiles(torrent)
|
||||
.then(torrentContents => overwrite ? overwriteExistingFiles(torrent, torrentContents) : torrentContents)
|
||||
.then(torrentContents => assignSubtitles(torrentContents))
|
||||
.catch(error => {
|
||||
logger.warn(`Failed getting files for ${torrent.title}`, error.message);
|
||||
return {};
|
||||
});
|
||||
if (!videos || !videos.length) {
|
||||
logger.warn(`no video files found for ${torrent.provider} [${torrent.infoHash}] ${torrent.title}`);
|
||||
return;
|
||||
}
|
||||
|
||||
return repository.createTorrent({ ...torrent, contents, subtitles })
|
||||
.then(() => Promises.sequence(videos.map(video => () => repository.createFile(video))))
|
||||
.then(() => logger.info(`Created ${torrent.provider} entry for [${torrent.infoHash}] ${torrent.title}`));
|
||||
}
|
||||
|
||||
async function overwriteExistingFiles(torrent, torrentContents) {
|
||||
const videos = torrentContents && torrentContents.videos;
|
||||
if (videos && videos.length) {
|
||||
const existingFiles = await repository.getFiles({ infoHash: videos[0].infoHash })
|
||||
.then((existing) => existing
|
||||
.reduce((map, next) => {
|
||||
const fileIndex = next.fileIndex !== undefined ? next.fileIndex : null;
|
||||
map[fileIndex] = (map[fileIndex] || []).concat(next);
|
||||
return map;
|
||||
}, {}))
|
||||
.catch(() => undefined);
|
||||
if (existingFiles && Object.keys(existingFiles).length) {
|
||||
const overwrittenVideos = videos
|
||||
.map(file => {
|
||||
const mapping = videos.length === 1 && Object.keys(existingFiles).length === 1
|
||||
? Object.values(existingFiles)[0]
|
||||
: existingFiles[file.fileIndex !== undefined ? file.fileIndex : null];
|
||||
if (mapping) {
|
||||
const originalFile = mapping.shift();
|
||||
return { id: originalFile.id, ...file };
|
||||
}
|
||||
return file;
|
||||
});
|
||||
return { ...torrentContents, videos: overwrittenVideos };
|
||||
}
|
||||
return torrentContents;
|
||||
}
|
||||
return Promise.reject(`No video files found for: ${torrent.title}`);
|
||||
}
|
||||
|
||||
export async function createSkipTorrentEntry(torrent) {
|
||||
return repository.createSkipTorrent(torrent);
|
||||
}
|
||||
|
||||
export async function getStoredTorrentEntry(torrent) {
|
||||
return repository.getSkipTorrent(torrent)
|
||||
.catch(() => repository.getTorrent(torrent))
|
||||
.catch(() => undefined);
|
||||
}
|
||||
|
||||
export async function checkAndUpdateTorrent(torrent) {
|
||||
const storedTorrent = torrent.dataValues
|
||||
? torrent
|
||||
: await repository.getTorrent(torrent).catch(() => undefined);
|
||||
if (!storedTorrent) {
|
||||
return false;
|
||||
}
|
||||
if (storedTorrent.provider === 'RARBG') {
|
||||
return true;
|
||||
}
|
||||
if (storedTorrent.provider === 'KickassTorrents' && torrent.provider) {
|
||||
storedTorrent.provider = torrent.provider;
|
||||
storedTorrent.torrentId = torrent.torrentId;
|
||||
}
|
||||
if (!storedTorrent.languages && torrent.languages && storedTorrent.provider !== 'RARBG') {
|
||||
storedTorrent.languages = torrent.languages;
|
||||
await storedTorrent.save();
|
||||
logger.debug(`Updated [${storedTorrent.infoHash}] ${storedTorrent.title} language to ${torrent.languages}`);
|
||||
}
|
||||
return createTorrentContents({ ...storedTorrent.get(), torrentLink: torrent.torrentLink })
|
||||
.then(() => updateTorrentSeeders(torrent));
|
||||
}
|
||||
|
||||
export async function createTorrentContents(torrent) {
|
||||
if (torrent.opened) {
|
||||
return;
|
||||
}
|
||||
const storedVideos = await repository.getFiles(torrent).catch(() => []);
|
||||
if (!storedVideos || !storedVideos.length) {
|
||||
return;
|
||||
}
|
||||
const notOpenedVideo = storedVideos.length === 1 && !Number.isInteger(storedVideos[0].fileIndex);
|
||||
const imdbId = Promises.mostCommonValue(storedVideos.map(stored => stored.imdbId));
|
||||
const kitsuId = Promises.mostCommonValue(storedVideos.map(stored => stored.kitsuId));
|
||||
|
||||
const { contents, videos, subtitles } = await parseTorrentFiles({ ...torrent, imdbId, kitsuId })
|
||||
.then(torrentContents => notOpenedVideo ? torrentContents : { ...torrentContents, videos: storedVideos })
|
||||
.then(torrentContents => assignSubtitles(torrentContents))
|
||||
.catch(error => {
|
||||
logger.warn(`Failed getting contents for [${torrent.infoHash}] ${torrent.title}`, error.message);
|
||||
return {};
|
||||
});
|
||||
|
||||
if (!contents || !contents.length) {
|
||||
return;
|
||||
}
|
||||
if (notOpenedVideo && videos.length === 1) {
|
||||
// if both have a single video and stored one was not opened, update stored one to true metadata and use that
|
||||
storedVideos[0].fileIndex = videos[0].fileIndex;
|
||||
storedVideos[0].title = videos[0].title;
|
||||
storedVideos[0].size = videos[0].size;
|
||||
storedVideos[0].subtitles = videos[0].subtitles;
|
||||
videos[0] = storedVideos[0];
|
||||
}
|
||||
// no videos available or more than one new videos were in the torrent
|
||||
const shouldDeleteOld = notOpenedVideo && videos.every(video => !video.id);
|
||||
|
||||
return repository.createTorrent({ ...torrent, contents, subtitles })
|
||||
.then(() => {
|
||||
if (shouldDeleteOld) {
|
||||
logger.debug(`Deleting old video for [${torrent.infoHash}] ${torrent.title}`)
|
||||
return storedVideos[0].destroy();
|
||||
}
|
||||
return Promise.resolve();
|
||||
})
|
||||
.then(() => Promises.sequence(videos.map(video => () => repository.createFile(video))))
|
||||
.then(() => logger.info(`Created contents for ${torrent.provider} [${torrent.infoHash}] ${torrent.title}`))
|
||||
.catch(error => logger.error(`Failed saving contents for [${torrent.infoHash}] ${torrent.title}`, error));
|
||||
}
|
||||
|
||||
export async function updateTorrentSeeders(torrent) {
|
||||
if (!(torrent.infoHash || (torrent.provider && torrent.torrentId)) || !Number.isInteger(torrent.seeders)) {
|
||||
return torrent;
|
||||
}
|
||||
|
||||
return repository.setTorrentSeeders(torrent, torrent.seeders)
|
||||
.catch(error => {
|
||||
logger.warn('Failed updating seeders:', error);
|
||||
return undefined;
|
||||
});
|
||||
}
|
||||
@@ -1,513 +0,0 @@
|
||||
import Bottleneck from 'bottleneck';
|
||||
import distance from 'jaro-winkler';
|
||||
import moment from 'moment';
|
||||
import { parse } from 'parse-torrent-title';
|
||||
import { metadataConfig } from './config.js';
|
||||
import { isDisk } from './extension.js';
|
||||
import { getMetadata, getImdbId, getKitsuId } from './metadata.js';
|
||||
import { parseSeriesVideos, isPackTorrent } from './parseHelper.js';
|
||||
import * as Promises from './promises.js';
|
||||
import {torrentFiles} from "./torrent.js";
|
||||
import { TorrentType } from './types.js';
|
||||
import {logger} from "./logger.js";
|
||||
|
||||
const MIN_SIZE = 5 * 1024 * 1024; // 5 MB
|
||||
const imdb_limiter = new Bottleneck({ maxConcurrent: metadataConfig.IMDB_CONCURRENT, minTime: metadataConfig.IMDB_INTERVAL_MS });
|
||||
|
||||
export async function parseTorrentFiles(torrent) {
|
||||
const parsedTorrentName = parse(torrent.title);
|
||||
const metadata = await getMetadata(torrent.kitsuId || torrent.imdbId, torrent.type || TorrentType.MOVIE)
|
||||
.then(meta => Object.assign({}, meta))
|
||||
.catch(() => undefined);
|
||||
|
||||
// if (metadata && metadata.type !== torrent.type && torrent.type !== Type.ANIME) {
|
||||
// throw new Error(`Mismatching entry type for ${torrent.name}: ${torrent.type}!=${metadata.type}`);
|
||||
// }
|
||||
if (torrent.type !== TorrentType.ANIME && metadata && metadata.type && metadata.type !== torrent.type) {
|
||||
// it's actually a movie/series
|
||||
torrent.type = metadata.type;
|
||||
}
|
||||
|
||||
if (torrent.type === TorrentType.MOVIE && (!parsedTorrentName.seasons ||
|
||||
parsedTorrentName.season === 5 && [1, 5].includes(parsedTorrentName.episode))) {
|
||||
return parseMovieFiles(torrent, parsedTorrentName, metadata);
|
||||
}
|
||||
|
||||
return parseSeriesFiles(torrent, parsedTorrentName, metadata)
|
||||
}
|
||||
|
||||
async function parseMovieFiles(torrent, parsedName, metadata) {
|
||||
const { contents, videos, subtitles } = await getMoviesTorrentContent(torrent);
|
||||
const filteredVideos = videos
|
||||
.filter(video => video.size > MIN_SIZE)
|
||||
.filter(video => !isFeaturette(video));
|
||||
if (isSingleMovie(filteredVideos)) {
|
||||
const parsedVideos = filteredVideos.map(video => ({
|
||||
infoHash: torrent.infoHash,
|
||||
fileIndex: video.fileIndex,
|
||||
title: video.path || torrent.title,
|
||||
size: video.size || torrent.size,
|
||||
imdbId: torrent.imdbId || metadata && metadata.imdbId,
|
||||
kitsuId: torrent.kitsuId || metadata && metadata.kitsuId
|
||||
}));
|
||||
return { contents, videos: parsedVideos, subtitles };
|
||||
}
|
||||
|
||||
const parsedVideos = await Promises.sequence(filteredVideos.map(video => () => isFeaturette(video)
|
||||
? Promise.resolve(video)
|
||||
: findMovieImdbId(video.name).then(imdbId => ({ ...video, imdbId }))))
|
||||
.then(videos => videos.map(video => ({
|
||||
infoHash: torrent.infoHash,
|
||||
fileIndex: video.fileIndex,
|
||||
title: video.path || video.name,
|
||||
size: video.size,
|
||||
imdbId: video.imdbId,
|
||||
})));
|
||||
return { contents, videos: parsedVideos, subtitles };
|
||||
}
|
||||
|
||||
async function parseSeriesFiles(torrent, parsedName, metadata) {
|
||||
const { contents, videos, subtitles } = await getSeriesTorrentContent(torrent);
|
||||
const parsedVideos = await Promise.resolve(videos)
|
||||
.then(videos => videos.filter(video => videos.length === 1 || video.size > MIN_SIZE))
|
||||
.then(videos => parseSeriesVideos(torrent, videos))
|
||||
.then(videos => decomposeEpisodes(torrent, videos, metadata))
|
||||
.then(videos => assignKitsuOrImdbEpisodes(torrent, videos, metadata))
|
||||
.then(videos => Promise.all(videos.map(video => video.isMovie
|
||||
? mapSeriesMovie(video, torrent)
|
||||
: mapSeriesEpisode(video, torrent, videos))))
|
||||
.then(videos => videos
|
||||
.reduce((a, b) => a.concat(b), [])
|
||||
.map(video => isFeaturette(video) ? clearInfoFields(video) : video))
|
||||
return { contents, videos: parsedVideos, subtitles };
|
||||
}
|
||||
|
||||
async function getMoviesTorrentContent(torrent) {
|
||||
const files = await torrentFiles(torrent)
|
||||
.catch(error => {
|
||||
if (!isPackTorrent(torrent)) {
|
||||
return { videos: [{ name: torrent.title, path: torrent.title, size: torrent.size }] }
|
||||
}
|
||||
return Promise.reject(error);
|
||||
});
|
||||
if (files.contents && files.contents.length && !files.videos.length && isDiskTorrent(files.contents)) {
|
||||
files.videos = [{ name: torrent.title, path: torrent.title, size: torrent.size }];
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
async function getSeriesTorrentContent(torrent) {
|
||||
return torrentFiles(torrent)
|
||||
.catch(error => {
|
||||
if (!isPackTorrent(torrent)) {
|
||||
return { videos: [{ name: torrent.title, path: torrent.title, size: torrent.size }] }
|
||||
}
|
||||
return Promise.reject(error);
|
||||
});
|
||||
}
|
||||
|
||||
async function mapSeriesEpisode(file, torrent, files) {
|
||||
if (!file.episodes && !file.kitsuEpisodes) {
|
||||
if (files.length === 1 || files.some(f => f.episodes || f.kitsuEpisodes) || parse(torrent.title).seasons) {
|
||||
return Promise.resolve({
|
||||
infoHash: torrent.infoHash,
|
||||
fileIndex: file.fileIndex,
|
||||
title: file.path || file.name,
|
||||
size: file.size,
|
||||
imdbId: torrent.imdbId || file.imdbId,
|
||||
});
|
||||
}
|
||||
return Promise.resolve([]);
|
||||
}
|
||||
const episodeIndexes = [...(file.episodes || file.kitsuEpisodes).keys()];
|
||||
return Promise.resolve(episodeIndexes.map((index) => ({
|
||||
infoHash: torrent.infoHash,
|
||||
fileIndex: file.fileIndex,
|
||||
title: file.path || file.name,
|
||||
size: file.size,
|
||||
imdbId: file.imdbId || torrent.imdbId,
|
||||
imdbSeason: file.season,
|
||||
imdbEpisode: file.episodes && file.episodes[index],
|
||||
kitsuId: file.kitsuId || torrent.kitsuId,
|
||||
kitsuEpisode: file.kitsuEpisodes && file.kitsuEpisodes[index]
|
||||
})))
|
||||
}
|
||||
|
||||
async function mapSeriesMovie(file, torrent) {
|
||||
const kitsuId = torrent.type === TorrentType.ANIME ? await findMovieKitsuId(file) : undefined;
|
||||
const imdbId = !kitsuId ? await findMovieImdbId(file) : undefined;
|
||||
const metadata = await getMetadata(kitsuId || imdbId, TorrentType.MOVIE).catch(() => ({}));
|
||||
const hasEpisode = metadata.videos && metadata.videos.length && (file.episode || metadata.videos.length === 1);
|
||||
const episodeVideo = hasEpisode && metadata.videos[(file.episode || 1) - 1];
|
||||
return [{
|
||||
infoHash: torrent.infoHash,
|
||||
fileIndex: file.fileIndex,
|
||||
title: file.path || file.name,
|
||||
size: file.size,
|
||||
imdbId: metadata.imdbId || imdbId,
|
||||
kitsuId: metadata.kitsuId || kitsuId,
|
||||
imdbSeason: episodeVideo && metadata.imdbId ? episodeVideo.imdbSeason : undefined,
|
||||
imdbEpisode: episodeVideo && metadata.imdbId ? episodeVideo.imdbEpisode || episodeVideo.episode : undefined,
|
||||
kitsuEpisode: episodeVideo && metadata.kitsuId ? episodeVideo.kitsuEpisode || episodeVideo.episode : undefined
|
||||
}];
|
||||
}
|
||||
|
||||
async function decomposeEpisodes(torrent, files, metadata = { episodeCount: [] }) {
|
||||
if (files.every(file => !file.episodes && !file.date)) {
|
||||
return files;
|
||||
}
|
||||
|
||||
preprocessEpisodes(files);
|
||||
|
||||
if (torrent.type === TorrentType.ANIME && torrent.kitsuId) {
|
||||
if (needsCinemetaMetadataForAnime(files, metadata)) {
|
||||
// In some cases anime could be resolved to wrong kitsuId
|
||||
// because of imdb season naming/absolute per series naming/multiple seasons
|
||||
// So in these cases we need to fetch cinemeta based metadata and decompose episodes using that
|
||||
await updateToCinemetaMetadata(metadata);
|
||||
if (files.some(file => Number.isInteger(file.season))) {
|
||||
// sometimes multi season anime torrents don't include season 1 naming
|
||||
files
|
||||
.filter(file => !Number.isInteger(file.season) && file.episodes)
|
||||
.forEach(file => file.season = 1);
|
||||
}
|
||||
} else {
|
||||
// otherwise for anime type episodes are always absolute and for a single season
|
||||
files
|
||||
.filter(file => file.episodes && file.season !== 0)
|
||||
.forEach(file => file.season = 1);
|
||||
return files;
|
||||
}
|
||||
}
|
||||
|
||||
const sortedEpisodes = files
|
||||
.map(file => !file.isMovie && file.episodes || [])
|
||||
.reduce((a, b) => a.concat(b), [])
|
||||
.sort((a, b) => a - b);
|
||||
|
||||
if (isConcatSeasonAndEpisodeFiles(files, sortedEpisodes, metadata)) {
|
||||
decomposeConcatSeasonAndEpisodeFiles(torrent, files, metadata);
|
||||
} else if (isDateEpisodeFiles(files, metadata)) {
|
||||
decomposeDateEpisodeFiles(torrent, files, metadata);
|
||||
} else if (isAbsoluteEpisodeFiles(torrent, files, metadata)) {
|
||||
decomposeAbsoluteEpisodeFiles(torrent, files, metadata);
|
||||
}
|
||||
// decomposeEpisodeTitleFiles(torrent, files, metadata);
|
||||
|
||||
return files;
|
||||
}
|
||||
|
||||
function preprocessEpisodes(files) {
|
||||
// reverse special episode naming when they named with 0 episode, ie. S02E00
|
||||
files
|
||||
.filter(file => Number.isInteger(file.season) && file.episode === 0)
|
||||
.forEach(file => {
|
||||
file.episode = file.season
|
||||
file.episodes = [file.season]
|
||||
file.season = 0;
|
||||
})
|
||||
}
|
||||
|
||||
function isConcatSeasonAndEpisodeFiles(files, sortedEpisodes, metadata) {
|
||||
if (metadata.kitsuId !== undefined) {
|
||||
// anime does not use this naming scheme in 99% of cases;
|
||||
return false;
|
||||
}
|
||||
// decompose concat season and episode files (ex. 101=S01E01) in case:
|
||||
// 1. file has a season, but individual files are concatenated with that season (ex. path Season 5/511 - Prize
|
||||
// Fighters.avi)
|
||||
// 2. file does not have a season and the episode does not go out of range for the concat season
|
||||
// episode count
|
||||
const thresholdAbove = Math.max(Math.ceil(files.length * 0.05), 5);
|
||||
const thresholdSorted = Math.max(Math.ceil(files.length * 0.8), 8);
|
||||
const threshold = Math.max(Math.ceil(files.length * 0.8), 5);
|
||||
const sortedConcatEpisodes = sortedEpisodes
|
||||
.filter(ep => ep > 100)
|
||||
.filter(ep => metadata.episodeCount[div100(ep) - 1] < ep)
|
||||
.filter(ep => metadata.episodeCount[div100(ep) - 1] >= mod100(ep));
|
||||
const concatFileEpisodes = files
|
||||
.filter(file => !file.isMovie && file.episodes)
|
||||
.filter(file => !file.season || file.episodes.every(ep => div100(ep) === file.season));
|
||||
const concatAboveTotalEpisodeCount = files
|
||||
.filter(file => !file.isMovie && file.episodes && file.episodes.every(ep => ep > 100))
|
||||
.filter(file => file.episodes.every(ep => ep > metadata.totalCount));
|
||||
return sortedConcatEpisodes.length >= thresholdSorted && concatFileEpisodes.length >= threshold
|
||||
|| concatAboveTotalEpisodeCount.length >= thresholdAbove;
|
||||
}
|
||||
|
||||
function isDateEpisodeFiles(files, metadata) {
|
||||
return files.every(file => (!file.season || !metadata.episodeCount[file.season - 1]) && file.date);
|
||||
}
|
||||
|
||||
function isAbsoluteEpisodeFiles(torrent, files, metadata) {
|
||||
const threshold = Math.ceil(files.length / 5);
|
||||
const isAnime = torrent.type === TorrentType.ANIME && torrent.kitsuId;
|
||||
const nonMovieEpisodes = files
|
||||
.filter(file => !file.isMovie && file.episodes);
|
||||
const absoluteEpisodes = files
|
||||
.filter(file => file.season && file.episodes)
|
||||
.filter(file => file.episodes.every(ep => metadata.episodeCount[file.season - 1] < ep))
|
||||
return nonMovieEpisodes.every(file => !file.season)
|
||||
|| (isAnime && nonMovieEpisodes.every(file => file.season > metadata.episodeCount.length))
|
||||
|| absoluteEpisodes.length >= threshold;
|
||||
}
|
||||
|
||||
function isNewEpisodeNotInMetadata(torrent, file, metadata) {
|
||||
// new episode might not yet been indexed by cinemeta.
|
||||
// detect this if episode number is larger than the last episode or season is larger than the last one
|
||||
// only for non anime metas
|
||||
const isAnime = torrent.type === TorrentType.ANIME && torrent.kitsuId;
|
||||
return !isAnime && !file.isMovie && file.episodes && file.season !== 1
|
||||
&& /continuing|current/i.test(metadata.status)
|
||||
&& file.season >= metadata.episodeCount.length
|
||||
&& file.episodes.every(ep => ep > (metadata.episodeCount[file.season - 1] || 0));
|
||||
}
|
||||
|
||||
function decomposeConcatSeasonAndEpisodeFiles(torrent, files, metadata) {
|
||||
files
|
||||
.filter(file => file.episodes && file.season !== 0 && file.episodes.every(ep => ep > 100))
|
||||
.filter(file => metadata.episodeCount[(file.season || div100(file.episodes[0])) - 1] < 100)
|
||||
.filter(file => file.season && file.episodes.every(ep => div100(ep) === file.season) || !file.season)
|
||||
.forEach(file => {
|
||||
file.season = div100(file.episodes[0]);
|
||||
file.episodes = file.episodes.map(ep => mod100(ep))
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
function decomposeAbsoluteEpisodeFiles(torrent, files, metadata) {
|
||||
if (metadata.episodeCount.length === 0) {
|
||||
files
|
||||
.filter(file => !Number.isInteger(file.season) && file.episodes && !file.isMovie)
|
||||
.forEach(file => {
|
||||
file.season = 1;
|
||||
});
|
||||
return;
|
||||
}
|
||||
files
|
||||
.filter(file => file.episodes && !file.isMovie && file.season !== 0)
|
||||
.filter(file => !isNewEpisodeNotInMetadata(torrent, file, metadata))
|
||||
.filter(file => !file.season || (metadata.episodeCount[file.season - 1] || 0) < file.episodes[0])
|
||||
.forEach(file => {
|
||||
const seasonIdx = ([...metadata.episodeCount.keys()]
|
||||
.find((i) => metadata.episodeCount.slice(0, i + 1).reduce((a, b) => a + b) >= file.episodes[0])
|
||||
+ 1 || metadata.episodeCount.length) - 1;
|
||||
|
||||
file.season = seasonIdx + 1;
|
||||
file.episodes = file.episodes
|
||||
.map(ep => ep - metadata.episodeCount.slice(0, seasonIdx).reduce((a, b) => a + b, 0))
|
||||
});
|
||||
}
|
||||
|
||||
function decomposeDateEpisodeFiles(torrent, files, metadata) {
|
||||
if (!metadata || !metadata.videos || !metadata.videos.length) {
|
||||
return;
|
||||
}
|
||||
|
||||
const timeZoneOffset = getTimeZoneOffset(metadata.country);
|
||||
const offsetVideos = metadata.videos
|
||||
.reduce((map, video) => {
|
||||
const releaseDate = moment(video.released).utcOffset(timeZoneOffset).format('YYYY-MM-DD');
|
||||
map[releaseDate] = video;
|
||||
return map;
|
||||
}, {});
|
||||
|
||||
files
|
||||
.filter(file => file.date)
|
||||
.forEach(file => {
|
||||
const video = offsetVideos[file.date];
|
||||
if (video) {
|
||||
file.season = video.season;
|
||||
file.episodes = [video.episode];
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
/* eslint-disable no-unused-vars */
|
||||
function decomposeEpisodeTitleFiles(torrent, files, metadata) {
|
||||
files
|
||||
// .filter(file => !file.season)
|
||||
.map(file => {
|
||||
const episodeTitle = file.name.replace('_', ' ')
|
||||
.replace(/^.*(?:E\d+[abc]?|- )\s?(.+)\.\w{1,4}$/, '$1')
|
||||
.trim();
|
||||
const foundEpisode = metadata.videos
|
||||
.map(video => ({ ...video, distance: distance(episodeTitle, video.name) }))
|
||||
.sort((a, b) => b.distance - a.distance)[0];
|
||||
if (foundEpisode) {
|
||||
file.isMovie = false;
|
||||
file.season = foundEpisode.season;
|
||||
file.episodes = [foundEpisode.episode];
|
||||
}
|
||||
})
|
||||
}
|
||||
/* eslint-enable no-unused-vars */
|
||||
|
||||
function getTimeZoneOffset(country) {
|
||||
switch (country) {
|
||||
case 'United States':
|
||||
case 'USA':
|
||||
return '-08:00';
|
||||
default:
|
||||
return '00:00';
|
||||
}
|
||||
}
|
||||
|
||||
function assignKitsuOrImdbEpisodes(torrent, files, metadata) {
|
||||
if (!metadata || !metadata.videos || !metadata.videos.length) {
|
||||
if (torrent.type === TorrentType.ANIME) {
|
||||
// assign episodes as kitsu episodes for anime when no metadata available for imdb mapping
|
||||
files
|
||||
.filter(file => file.season && file.episodes)
|
||||
.forEach(file => {
|
||||
file.kitsuEpisodes = file.episodes;
|
||||
file.season = undefined;
|
||||
file.episodes = undefined;
|
||||
})
|
||||
if (metadata.type === TorrentType.MOVIE && files.every(file => !file.imdbId)) {
|
||||
// sometimes a movie has episode naming, thus not recognized as a movie and imdbId not assigned
|
||||
files.forEach(file => file.imdbId = metadata.imdbId);
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
const seriesMapping = metadata.videos
|
||||
.reduce((map, video) => {
|
||||
const episodeMap = map[video.season] || {};
|
||||
episodeMap[video.episode] = video;
|
||||
map[video.season] = episodeMap;
|
||||
return map;
|
||||
}, {});
|
||||
|
||||
if (metadata.videos.some(video => Number.isInteger(video.imdbSeason)) || !metadata.imdbId) {
|
||||
// kitsu episode info is the base
|
||||
files
|
||||
.filter(file => Number.isInteger(file.season) && file.episodes)
|
||||
.map(file => {
|
||||
const seasonMapping = seriesMapping[file.season];
|
||||
const episodeMapping = seasonMapping && seasonMapping[file.episodes[0]];
|
||||
file.kitsuEpisodes = file.episodes;
|
||||
if (episodeMapping && Number.isInteger(episodeMapping.imdbSeason)) {
|
||||
file.imdbId = metadata.imdbId;
|
||||
file.season = episodeMapping.imdbSeason;
|
||||
file.episodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].imdbEpisode);
|
||||
} else {
|
||||
// no imdb mapping available for episode
|
||||
file.season = undefined;
|
||||
file.episodes = undefined;
|
||||
}
|
||||
});
|
||||
} else if (metadata.videos.some(video => video.kitsuEpisode)) {
|
||||
// imdb episode info is base
|
||||
files
|
||||
.filter(file => Number.isInteger(file.season) && file.episodes)
|
||||
.forEach(file => {
|
||||
if (seriesMapping[file.season]) {
|
||||
const seasonMapping = seriesMapping[file.season];
|
||||
file.imdbId = metadata.imdbId;
|
||||
file.kitsuId = seasonMapping[file.episodes[0]] && seasonMapping[file.episodes[0]].kitsuId;
|
||||
file.kitsuEpisodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].kitsuEpisode);
|
||||
} else if (seriesMapping[file.season - 1]) {
|
||||
// sometimes a second season might be a continuation of the previous season
|
||||
const seasonMapping = seriesMapping[file.season - 1];
|
||||
const episodes = Object.values(seasonMapping);
|
||||
const firstKitsuId = episodes.length && episodes[0].kitsuId;
|
||||
const differentTitlesCount = new Set(episodes.map(ep => ep.kitsuId)).size
|
||||
const skippedCount = episodes.filter(ep => ep.kitsuId === firstKitsuId).length;
|
||||
const seasonEpisodes = files
|
||||
.filter(otherFile => otherFile.season === file.season)
|
||||
.reduce((a, b) => a.concat(b.episodes), []);
|
||||
const isAbsoluteOrder = seasonEpisodes.every(ep => ep > skippedCount && ep <= episodes.length)
|
||||
const isNormalOrder = seasonEpisodes.every(ep => ep + skippedCount <= episodes.length)
|
||||
if (differentTitlesCount >= 1 && (isAbsoluteOrder || isNormalOrder)) {
|
||||
file.imdbId = metadata.imdbId;
|
||||
file.season = file.season - 1;
|
||||
file.episodes = file.episodes.map(ep => isAbsoluteOrder ? ep : ep + skippedCount);
|
||||
file.kitsuId = seasonMapping[file.episodes[0]].kitsuId;
|
||||
file.kitsuEpisodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].kitsuEpisode);
|
||||
}
|
||||
} else if (Object.values(seriesMapping).length === 1 && seriesMapping[1]) {
|
||||
// sometimes series might be named with sequel season but it's not a season on imdb and a new title
|
||||
const seasonMapping = seriesMapping[1];
|
||||
file.imdbId = metadata.imdbId;
|
||||
file.season = 1;
|
||||
file.kitsuId = seasonMapping[file.episodes[0]].kitsuId;
|
||||
file.kitsuEpisodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].kitsuEpisode);
|
||||
}
|
||||
});
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
function needsCinemetaMetadataForAnime(files, metadata) {
|
||||
if (!metadata || !metadata.imdbId || !metadata.videos || !metadata.videos.length) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const minSeason = Math.min(...metadata.videos.map(video => video.imdbSeason)) || Number.MAX_VALUE;
|
||||
const maxSeason = Math.max(...metadata.videos.map(video => video.imdbSeason)) || Number.MAX_VALUE;
|
||||
const differentSeasons = new Set(metadata.videos
|
||||
.map(video => video.imdbSeason)
|
||||
.filter(season => Number.isInteger(season))).size;
|
||||
const total = metadata.totalCount || Number.MAX_VALUE;
|
||||
return differentSeasons > 1 || files
|
||||
.filter(file => !file.isMovie && file.episodes)
|
||||
.some(file => file.season < minSeason || file.season > maxSeason || file.episodes.every(ep => ep > total));
|
||||
}
|
||||
|
||||
async function updateToCinemetaMetadata(metadata) {
|
||||
return getMetadata(metadata.imdbId, metadata.type)
|
||||
.then(newMetadata => !newMetadata.videos || !newMetadata.videos.length ? metadata : newMetadata)
|
||||
.then(newMetadata => {
|
||||
metadata.videos = newMetadata.videos;
|
||||
metadata.episodeCount = newMetadata.episodeCount;
|
||||
metadata.totalCount = newMetadata.totalCount;
|
||||
return metadata;
|
||||
})
|
||||
.catch(error => logger.warn(`Failed ${metadata.imdbId} metadata cinemeta update due: ${error.message}`));
|
||||
}
|
||||
|
||||
function findMovieImdbId(title) {
|
||||
const parsedTitle = typeof title === 'string' ? parse(title) : title;
|
||||
logger.debug(`Finding movie imdbId for ${title}`);
|
||||
return imdb_limiter.schedule(() => getImdbId(parsedTitle, TorrentType.MOVIE).catch(() => undefined));
|
||||
}
|
||||
|
||||
function findMovieKitsuId(title) {
|
||||
const parsedTitle = typeof title === 'string' ? parse(title) : title;
|
||||
return getKitsuId(parsedTitle, TorrentType.MOVIE).catch(() => undefined);
|
||||
}
|
||||
|
||||
function isDiskTorrent(contents) {
|
||||
return contents.some(content => isDisk(content.path));
|
||||
}
|
||||
|
||||
function isSingleMovie(videos) {
|
||||
return videos.length === 1 ||
|
||||
(videos.length === 2 &&
|
||||
videos.find(v => /\b(?:part|disc|cd)[ ._-]?0?1\b|^0?1\.\w{2,4}$/i.test(v.path)) &&
|
||||
videos.find(v => /\b(?:part|disc|cd)[ ._-]?0?2\b|^0?2\.\w{2,4}$/i.test(v.path)));
|
||||
}
|
||||
|
||||
function isFeaturette(video) {
|
||||
return /featurettes?\/|extras-grym/i.test(video.path);
|
||||
}
|
||||
|
||||
function clearInfoFields(video) {
|
||||
video.imdbId = undefined;
|
||||
video.imdbSeason = undefined;
|
||||
video.imdbEpisode = undefined;
|
||||
video.kitsuId = undefined;
|
||||
video.kitsuEpisode = undefined;
|
||||
return video;
|
||||
}
|
||||
|
||||
function div100(episode) {
|
||||
return (episode / 100 >> 0); // floor to nearest int
|
||||
}
|
||||
|
||||
function mod100(episode) {
|
||||
return episode % 100;
|
||||
}
|
||||
@@ -1,89 +0,0 @@
|
||||
import { parse } from 'parse-torrent-title';
|
||||
|
||||
export function assignSubtitles({ contents, videos, subtitles }) {
|
||||
if (videos && videos.length && subtitles && subtitles.length) {
|
||||
if (videos.length === 1) {
|
||||
videos[0].subtitles = subtitles;
|
||||
return { contents, videos, subtitles: [] };
|
||||
}
|
||||
|
||||
const parsedVideos = videos
|
||||
.map(video => _parseVideo(video));
|
||||
const assignedSubs = subtitles
|
||||
.map(subtitle => ({ subtitle, videos: _mostProbableSubtitleVideos(subtitle, parsedVideos) }));
|
||||
const unassignedSubs = assignedSubs
|
||||
.filter(assignedSub => !assignedSub.videos)
|
||||
.map(assignedSub => assignedSub.subtitle);
|
||||
|
||||
assignedSubs
|
||||
.filter(assignedSub => assignedSub.videos)
|
||||
.forEach(assignedSub => assignedSub.videos
|
||||
.forEach(video => video.subtitles = (video.subtitles || []).concat(assignedSub.subtitle)));
|
||||
return { contents, videos, subtitles: unassignedSubs };
|
||||
}
|
||||
return { contents, videos, subtitles };
|
||||
}
|
||||
|
||||
function _parseVideo(video) {
|
||||
const fileName = video.title.split('/').pop().replace(/\.(\w{2,4})$/, '');
|
||||
const folderName = video.title.replace(/\/?[^/]+$/, '');
|
||||
return {
|
||||
videoFile: video,
|
||||
fileName: fileName,
|
||||
folderName: folderName,
|
||||
...parseFilename(video.title)
|
||||
};
|
||||
}
|
||||
|
||||
function _mostProbableSubtitleVideos(subtitle, parsedVideos) {
|
||||
const subTitle = (subtitle.title || subtitle.path).split('/').pop().replace(/\.(\w{2,4})$/, '');
|
||||
const parsedSub = parsePath(subtitle.title || subtitle.path);
|
||||
const byFileName = parsedVideos.filter(video => subTitle.includes(video.fileName));
|
||||
if (byFileName.length === 1) {
|
||||
return byFileName.map(v => v.videoFile);
|
||||
}
|
||||
const byTitleSeasonEpisode = parsedVideos.filter(video => video.title === parsedSub.title
|
||||
&& arrayEquals(video.seasons, parsedSub.seasons)
|
||||
&& arrayEquals(video.episodes, parsedSub.episodes));
|
||||
if (singleVideoFile(byTitleSeasonEpisode)) {
|
||||
return byTitleSeasonEpisode.map(v => v.videoFile);
|
||||
}
|
||||
const bySeasonEpisode = parsedVideos.filter(video => arrayEquals(video.seasons, parsedSub.seasons)
|
||||
&& arrayEquals(video.episodes, parsedSub.episodes));
|
||||
if (singleVideoFile(bySeasonEpisode)) {
|
||||
return bySeasonEpisode.map(v => v.videoFile);
|
||||
}
|
||||
const byTitle = parsedVideos.filter(video => video.title && video.title === parsedSub.title);
|
||||
if (singleVideoFile(byTitle)) {
|
||||
return byTitle.map(v => v.videoFile);
|
||||
}
|
||||
const byEpisode = parsedVideos.filter(video => arrayEquals(video.episodes, parsedSub.episodes));
|
||||
if (singleVideoFile(byEpisode)) {
|
||||
return byEpisode.map(v => v.videoFile);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function singleVideoFile(videos) {
|
||||
return new Set(videos.map(v => v.videoFile.fileIndex)).size === 1;
|
||||
}
|
||||
|
||||
function parsePath(path) {
|
||||
const pathParts = path.split('/').map(part => parseFilename(part));
|
||||
const parsedWithEpisode = pathParts.find(parsed => parsed.season && parsed.episodes);
|
||||
return parsedWithEpisode || pathParts[pathParts.length - 1];
|
||||
}
|
||||
|
||||
function parseFilename(filename) {
|
||||
const parsedInfo = parse(filename)
|
||||
const titleEpisode = parsedInfo.title.match(/(\d+)$/);
|
||||
if (!parsedInfo.episodes && titleEpisode) {
|
||||
parsedInfo.episodes = [parseInt(titleEpisode[1], 10)];
|
||||
}
|
||||
return parsedInfo;
|
||||
}
|
||||
|
||||
function arrayEquals(array1, array2) {
|
||||
if (!array1 || !array2) return array1 === array2;
|
||||
return array1.length === array2.length && array1.every((value, index) => value === array2[index])
|
||||
}
|
||||
@@ -1,26 +0,0 @@
|
||||
import axios from 'axios';
|
||||
import {cacheTrackers} from "./cache.js";
|
||||
import { trackerConfig } from './config.js';
|
||||
import {logger} from "./logger.js";
|
||||
|
||||
const downloadTrackers = async () => {
|
||||
const response = await axios.get(trackerConfig.TRACKERS_URL);
|
||||
const trackersListText = response.data;
|
||||
// Trackers are separated by a newline character
|
||||
let urlTrackers = trackersListText.split("\n");
|
||||
// remove blank lines
|
||||
urlTrackers = urlTrackers.filter(line => line.trim() !== '');
|
||||
|
||||
if (!trackerConfig.UDP_ENABLED) {
|
||||
// remove any udp trackers
|
||||
urlTrackers = urlTrackers.filter(line => !line.startsWith('udp://'));
|
||||
}
|
||||
|
||||
logger.info(`Trackers updated at ${Date.now()}: ${urlTrackers.length} trackers`);
|
||||
|
||||
return urlTrackers;
|
||||
};
|
||||
|
||||
export const getTrackers = async () => {
|
||||
return cacheTrackers(downloadTrackers);
|
||||
};
|
||||
@@ -1,11 +0,0 @@
|
||||
export const TorrentType = {
|
||||
MOVIE: 'movie',
|
||||
SERIES: 'series',
|
||||
ANIME: 'anime',
|
||||
PORN: 'xxx',
|
||||
};
|
||||
|
||||
export const CacheType = {
|
||||
MEMORY: 'memory',
|
||||
MONGODB: 'mongodb',
|
||||
};
|
||||
9
src/node/consumer/src/main.ts
Normal file
9
src/node/consumer/src/main.ts
Normal file
@@ -0,0 +1,9 @@
|
||||
import "reflect-metadata"; // required
|
||||
import {ICompositionalRoot} from "@setup/composition_root";
|
||||
import {serviceContainer} from "@setup/inversify_config";
|
||||
import {IocTypes} from "@setup/ioc_types";
|
||||
|
||||
(async (): Promise<void> => {
|
||||
const compositionalRoot = serviceContainer.get<ICompositionalRoot>(IocTypes.ICompositionalRoot);
|
||||
await compositionalRoot.start();
|
||||
})();
|
||||
22
src/node/consumer/src/setup/composition_root.ts
Normal file
22
src/node/consumer/src/setup/composition_root.ts
Normal file
@@ -0,0 +1,22 @@
|
||||
import {IProcessTorrentsJob} from "@interfaces/process_torrents_job";
|
||||
import {ITrackerService} from "@interfaces/tracker_service";
|
||||
import {IDatabaseRepository} from "@repository/interfaces/database_repository";
|
||||
import {IocTypes} from "@setup/ioc_types";
|
||||
import {inject, injectable} from "inversify";
|
||||
|
||||
export interface ICompositionalRoot {
|
||||
start(): Promise<void>;
|
||||
}
|
||||
|
||||
@injectable()
|
||||
export class CompositionalRoot implements ICompositionalRoot {
|
||||
@inject(IocTypes.ITrackerService) trackerService: ITrackerService;
|
||||
@inject(IocTypes.IDatabaseRepository) databaseRepository: IDatabaseRepository;
|
||||
@inject(IocTypes.IProcessTorrentsJob) processTorrentsJob: IProcessTorrentsJob;
|
||||
|
||||
async start(): Promise<void> {
|
||||
await this.trackerService.getTrackers();
|
||||
await this.databaseRepository.connect();
|
||||
await this.processTorrentsJob.listenToQueue();
|
||||
}
|
||||
}
|
||||
42
src/node/consumer/src/setup/inversify_config.ts
Normal file
42
src/node/consumer/src/setup/inversify_config.ts
Normal file
@@ -0,0 +1,42 @@
|
||||
import {ICacheService} from "@interfaces/cache_service";
|
||||
import {ILoggingService} from "@interfaces/logging_service";
|
||||
import {IMetadataService} from "@interfaces/metadata_service";
|
||||
import {IProcessTorrentsJob} from "@interfaces/process_torrents_job";
|
||||
import {ITorrentDownloadService} from "@interfaces/torrent_download_service";
|
||||
import {ITorrentEntriesService} from "@interfaces/torrent_entries_service";
|
||||
import {ITorrentFileService} from "@interfaces/torrent_file_service";
|
||||
import {ITorrentProcessingService} from "@interfaces/torrent_processing_service";
|
||||
import {ITorrentSubtitleService} from "@interfaces/torrent_subtitle_service";
|
||||
import {ITrackerService} from "@interfaces/tracker_service";
|
||||
import {ProcessTorrentsJob} from "@jobs/process_torrents_job";
|
||||
import {DatabaseRepository} from "@repository/database_repository";
|
||||
import {IDatabaseRepository} from "@repository/interfaces/database_repository";
|
||||
import {CacheService} from "@services/cache_service";
|
||||
import {LoggingService} from "@services/logging_service";
|
||||
import {MetadataService} from "@services/metadata_service";
|
||||
import {TorrentDownloadService} from "@services/torrent_download_service";
|
||||
import {TorrentEntriesService} from "@services/torrent_entries_service";
|
||||
import {TorrentFileService} from "@services/torrent_file_service";
|
||||
import {TorrentProcessingService} from "@services/torrent_processing_service";
|
||||
import {TorrentSubtitleService} from "@services/torrent_subtitle_service";
|
||||
import {TrackerService} from "@services/tracker_service";
|
||||
import {ICompositionalRoot, CompositionalRoot} from "@setup/composition_root";
|
||||
import {IocTypes} from "@setup/ioc_types";
|
||||
import {Container} from "inversify";
|
||||
|
||||
const serviceContainer = new Container();
|
||||
|
||||
serviceContainer.bind<ICompositionalRoot>(IocTypes.ICompositionalRoot).to(CompositionalRoot).inSingletonScope();
|
||||
serviceContainer.bind<ICacheService>(IocTypes.ICacheService).to(CacheService).inSingletonScope();
|
||||
serviceContainer.bind<ILoggingService>(IocTypes.ILoggingService).to(LoggingService).inSingletonScope();
|
||||
serviceContainer.bind<ITrackerService>(IocTypes.ITrackerService).to(TrackerService).inSingletonScope();
|
||||
serviceContainer.bind<ITorrentDownloadService>(IocTypes.ITorrentDownloadService).to(TorrentDownloadService).inSingletonScope();
|
||||
serviceContainer.bind<ITorrentFileService>(IocTypes.ITorrentFileService).to(TorrentFileService);
|
||||
serviceContainer.bind<ITorrentProcessingService>(IocTypes.ITorrentProcessingService).to(TorrentProcessingService);
|
||||
serviceContainer.bind<ITorrentSubtitleService>(IocTypes.ITorrentSubtitleService).to(TorrentSubtitleService);
|
||||
serviceContainer.bind<ITorrentEntriesService>(IocTypes.ITorrentEntriesService).to(TorrentEntriesService);
|
||||
serviceContainer.bind<IMetadataService>(IocTypes.IMetadataService).to(MetadataService);
|
||||
serviceContainer.bind<IDatabaseRepository>(IocTypes.IDatabaseRepository).to(DatabaseRepository);
|
||||
serviceContainer.bind<IProcessTorrentsJob>(IocTypes.IProcessTorrentsJob).to(ProcessTorrentsJob);
|
||||
|
||||
export {serviceContainer};
|
||||
18
src/node/consumer/src/setup/ioc_types.ts
Normal file
18
src/node/consumer/src/setup/ioc_types.ts
Normal file
@@ -0,0 +1,18 @@
|
||||
export const IocTypes = {
|
||||
// Composition root
|
||||
ICompositionalRoot: Symbol.for("ICompositionalRoot"),
|
||||
// Services
|
||||
ICacheService: Symbol.for("ICacheService"),
|
||||
ILoggingService: Symbol.for("ILoggingService"),
|
||||
IMetadataService: Symbol.for("IMetadataService"),
|
||||
ITorrentDownloadService: Symbol.for("ITorrentDownloadService"),
|
||||
ITorrentEntriesService: Symbol.for("ITorrentEntriesService"),
|
||||
ITorrentFileService: Symbol.for("ITorrentFileService"),
|
||||
ITorrentProcessingService: Symbol.for("ITorrentProcessingService"),
|
||||
ITorrentSubtitleService: Symbol.for("ITorrentSubtitleService"),
|
||||
ITrackerService: Symbol.for("ITrackerService"),
|
||||
// DAL
|
||||
IDatabaseRepository: Symbol.for("IDatabaseRepository"),
|
||||
// Jobs
|
||||
IProcessTorrentsJob: Symbol.for("IProcessTorrentsJob"),
|
||||
};
|
||||
35
src/node/consumer/test/helpers/boolean_helpers.test.ts
Normal file
35
src/node/consumer/test/helpers/boolean_helpers.test.ts
Normal file
@@ -0,0 +1,35 @@
|
||||
import { BooleanHelpers } from '@helpers/boolean_helpers';
|
||||
|
||||
describe('BooleanHelpers.parseBool', () => {
|
||||
it('should return true when value is "true"', () => {
|
||||
expect(BooleanHelpers.parseBool('true', false)).toBe(true);
|
||||
});
|
||||
|
||||
it('should return true when value is "1"', () => {
|
||||
expect(BooleanHelpers.parseBool('1', false)).toBe(true);
|
||||
});
|
||||
|
||||
it('should return true when value is "yes"', () => {
|
||||
expect(BooleanHelpers.parseBool('yes', false)).toBe(true);
|
||||
});
|
||||
|
||||
it('should return false when value is "false"', () => {
|
||||
expect(BooleanHelpers.parseBool('false', true)).toBe(false);
|
||||
});
|
||||
|
||||
it('should return false when value is "0"', () => {
|
||||
expect(BooleanHelpers.parseBool('0', true)).toBe(false);
|
||||
});
|
||||
|
||||
it('should return false when value is "no"', () => {
|
||||
expect(BooleanHelpers.parseBool('no', true)).toBe(false);
|
||||
});
|
||||
|
||||
it('should return default value when value is undefined', () => {
|
||||
expect(BooleanHelpers.parseBool(undefined, true)).toBe(true);
|
||||
});
|
||||
|
||||
it('should return default value when value is not "true", "1", "yes", "false", "0", or "no"', () => {
|
||||
expect(BooleanHelpers.parseBool('random', true)).toBe(true);
|
||||
});
|
||||
});
|
||||
33
src/node/consumer/test/helpers/extension_helpers.test.ts
Normal file
33
src/node/consumer/test/helpers/extension_helpers.test.ts
Normal file
@@ -0,0 +1,33 @@
|
||||
import { ExtensionHelpers } from '@helpers/extension_helpers';
|
||||
|
||||
describe('ExtensionHelpers', () => {
|
||||
describe('isVideo', () => {
|
||||
it('should return true when file extension is a video extension', () => {
|
||||
expect(ExtensionHelpers.isVideo('file.mp4')).toBe(true);
|
||||
});
|
||||
|
||||
it('should return false when file extension is not a video extension', () => {
|
||||
expect(ExtensionHelpers.isVideo('file.txt')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('isSubtitle', () => {
|
||||
it('should return true when file extension is a subtitle extension', () => {
|
||||
expect(ExtensionHelpers.isSubtitle('file.srt')).toBe(true);
|
||||
});
|
||||
|
||||
it('should return false when file extension is not a subtitle extension', () => {
|
||||
expect(ExtensionHelpers.isSubtitle('file.txt')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('isDisk', () => {
|
||||
it('should return true when file extension is a disk extension', () => {
|
||||
expect(ExtensionHelpers.isDisk('file.iso')).toBe(true);
|
||||
});
|
||||
|
||||
it('should return false when file extension is not a disk extension', () => {
|
||||
expect(ExtensionHelpers.isDisk('file.txt')).toBe(false);
|
||||
});
|
||||
});
|
||||
});
|
||||
55
src/node/consumer/test/helpers/promise_helpers.test.ts
Normal file
55
src/node/consumer/test/helpers/promise_helpers.test.ts
Normal file
@@ -0,0 +1,55 @@
|
||||
import { PromiseHelpers } from '@helpers/promises_helpers';
|
||||
|
||||
describe('PromiseHelpers', () => {
|
||||
beforeAll(() => {
|
||||
jest.useFakeTimers({timerLimit: 5000});
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
describe('sequence', () => {
|
||||
it('should resolve promises in sequence', async () => {
|
||||
const promises = [() => Promise.resolve(1), () => Promise.resolve(2), () => Promise.resolve(3)];
|
||||
const result = await PromiseHelpers.sequence(promises);
|
||||
expect(result).toEqual([1, 2, 3]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('first', () => {
|
||||
it('should resolve the first fulfilled promise', async () => {
|
||||
const promises = [Promise.reject('error'), Promise.resolve('success'), Promise.resolve('success2')];
|
||||
const result = await PromiseHelpers.first(promises);
|
||||
expect(result).toBe('success');
|
||||
});
|
||||
});
|
||||
|
||||
describe('delay', () => {
|
||||
it('should delay execution', async () => {
|
||||
const startTime = Date.now();
|
||||
const delayPromise = PromiseHelpers.delay(1000);
|
||||
jest.runAllTimers();
|
||||
await delayPromise;
|
||||
const endTime = Date.now();
|
||||
expect(endTime - startTime).toBeGreaterThanOrEqual(1000);
|
||||
}, 30000);
|
||||
});
|
||||
|
||||
describe('timeout', () => {
|
||||
it('should reject promise after timeout', async () => {
|
||||
const promise = new Promise((resolve) => setTimeout(resolve, 2000));
|
||||
const timeoutPromise = PromiseHelpers.timeout(1000, promise);
|
||||
jest.advanceTimersByTime(1000);
|
||||
await expect(timeoutPromise).rejects.toBe('Timed out');
|
||||
}, 20000);
|
||||
});
|
||||
|
||||
describe('mostCommonValue', () => {
|
||||
it('should return the most common value in an array', () => {
|
||||
const array = [1, 2, 2, 3, 3, 3];
|
||||
const result = PromiseHelpers.mostCommonValue(array);
|
||||
expect(result).toBe(3);
|
||||
});
|
||||
});
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,232 @@
|
||||
{
|
||||
"meta": {
|
||||
"status": "Ended",
|
||||
"videos": [
|
||||
{
|
||||
"name": "Pilot",
|
||||
"season": 1,
|
||||
"number": 0,
|
||||
"firstAired": "1990-09-20T00:00:00.000Z",
|
||||
"rating": "6.6",
|
||||
"id": "tt0098798:1:0",
|
||||
"overview": "",
|
||||
"imdb_id": "tt0099580"
|
||||
},
|
||||
{
|
||||
"name": "Out of Control",
|
||||
"season": 1,
|
||||
"number": 1,
|
||||
"firstAired": "1990-09-26T00:00:00.000Z",
|
||||
"rating": "6.8",
|
||||
"id": "tt0098798:1:1",
|
||||
"overview": "",
|
||||
"imdb_id": "tt0579962"
|
||||
},
|
||||
{
|
||||
"name": "Watching the Detectives",
|
||||
"season": 1,
|
||||
"number": 2,
|
||||
"firstAired": "1990-10-17T00:00:00.000Z",
|
||||
"rating": "6.9",
|
||||
"id": "tt0098798:1:2",
|
||||
"overview": "",
|
||||
"imdb_id": "tt0579971"
|
||||
},
|
||||
{
|
||||
"name": "Honor Among Thieves",
|
||||
"season": 1,
|
||||
"number": 3,
|
||||
"firstAired": "1990-10-25T00:00:00.000Z",
|
||||
"rating": "6.8",
|
||||
"id": "tt0098798:1:3",
|
||||
"overview": "",
|
||||
"imdb_id": "tt0579961"
|
||||
},
|
||||
{
|
||||
"name": "Double Vision",
|
||||
"season": 1,
|
||||
"number": 4,
|
||||
"firstAired": "1990-11-01T00:00:00.000Z",
|
||||
"rating": "6.6",
|
||||
"id": "tt0098798:1:4",
|
||||
"overview": "",
|
||||
"imdb_id": "tt0579957"
|
||||
},
|
||||
{
|
||||
"name": "Sins of the Father",
|
||||
"season": 1,
|
||||
"number": 5,
|
||||
"firstAired": "1990-11-08T00:00:00.000Z",
|
||||
"rating": "6.9",
|
||||
"id": "tt0098798:1:5",
|
||||
"overview": "",
|
||||
"imdb_id": "tt0579965"
|
||||
},
|
||||
{
|
||||
"name": "Child's Play",
|
||||
"season": 1,
|
||||
"number": 6,
|
||||
"firstAired": "1990-11-15T00:00:00.000Z",
|
||||
"rating": "6.8",
|
||||
"id": "tt0098798:1:6",
|
||||
"overview": "",
|
||||
"imdb_id": "tt0579955"
|
||||
},
|
||||
{
|
||||
"name": "Shroud of Death",
|
||||
"season": 1,
|
||||
"number": 7,
|
||||
"firstAired": "1990-11-29T00:00:00.000Z",
|
||||
"rating": "7.1",
|
||||
"id": "tt0098798:1:7",
|
||||
"overview": "",
|
||||
"imdb_id": "tt0579963"
|
||||
},
|
||||
{
|
||||
"name": "Ghost in the Machine",
|
||||
"season": 1,
|
||||
"number": 8,
|
||||
"firstAired": "1990-12-13T00:00:00.000Z",
|
||||
"rating": "7.7",
|
||||
"id": "tt0098798:1:8",
|
||||
"overview": "",
|
||||
"imdb_id": "tt0579959"
|
||||
},
|
||||
{
|
||||
"name": "Sight Unseen",
|
||||
"season": 1,
|
||||
"number": 9,
|
||||
"firstAired": "1991-01-10T00:00:00.000Z",
|
||||
"rating": "7",
|
||||
"id": "tt0098798:1:9",
|
||||
"overview": "",
|
||||
"imdb_id": "tt0579964"
|
||||
},
|
||||
{
|
||||
"name": "Beat the Clock",
|
||||
"season": 1,
|
||||
"number": 10,
|
||||
"firstAired": "1991-01-31T00:00:00.000Z",
|
||||
"rating": "7",
|
||||
"id": "tt0098798:1:10",
|
||||
"overview": "",
|
||||
"imdb_id": "tt0579953"
|
||||
},
|
||||
{
|
||||
"name": "The Trickster",
|
||||
"season": 1,
|
||||
"number": 11,
|
||||
"firstAired": "1991-02-07T00:00:00.000Z",
|
||||
"rating": "7.7",
|
||||
"id": "tt0098798:1:11",
|
||||
"overview": "",
|
||||
"imdb_id": "tt0579969"
|
||||
},
|
||||
{
|
||||
"name": "Tina, Is That You?",
|
||||
"season": 1,
|
||||
"number": 12,
|
||||
"firstAired": "1991-02-14T00:00:00.000Z",
|
||||
"rating": "6.8",
|
||||
"id": "tt0098798:1:12",
|
||||
"overview": "",
|
||||
"imdb_id": "tt0579967"
|
||||
},
|
||||
{
|
||||
"name": "Be My Baby",
|
||||
"season": 1,
|
||||
"number": 13,
|
||||
"firstAired": "1991-02-20T00:00:00.000Z",
|
||||
"rating": "6.5",
|
||||
"id": "tt0098798:1:13",
|
||||
"overview": "",
|
||||
"imdb_id": "tt0579952"
|
||||
},
|
||||
{
|
||||
"name": "Fast Forward",
|
||||
"season": 1,
|
||||
"number": 14,
|
||||
"firstAired": "1991-02-27T00:00:00.000Z",
|
||||
"rating": "7.9",
|
||||
"id": "tt0098798:1:14",
|
||||
"overview": "",
|
||||
"imdb_id": "tt0579958"
|
||||
},
|
||||
{
|
||||
"name": "Deadly Nightshade",
|
||||
"season": 1,
|
||||
"number": 15,
|
||||
"firstAired": "1991-03-28T00:00:00.000Z",
|
||||
"rating": "7.7",
|
||||
"id": "tt0098798:1:15",
|
||||
"overview": "",
|
||||
"imdb_id": "tt0579966"
|
||||
},
|
||||
{
|
||||
"name": "Captain Cold",
|
||||
"season": 1,
|
||||
"number": 16,
|
||||
"firstAired": "1991-04-05T00:00:00.000Z",
|
||||
"rating": "7.7",
|
||||
"id": "tt0098798:1:16",
|
||||
"overview": "",
|
||||
"imdb_id": "tt0579954"
|
||||
},
|
||||
{
|
||||
"name": "Twin Streaks",
|
||||
"season": 1,
|
||||
"number": 17,
|
||||
"firstAired": "1991-04-12T00:00:00.000Z",
|
||||
"rating": "7.1",
|
||||
"id": "tt0098798:1:17",
|
||||
"overview": "",
|
||||
"imdb_id": "tt0579970"
|
||||
},
|
||||
{
|
||||
"name": "Done with Mirrors",
|
||||
"season": 1,
|
||||
"number": 18,
|
||||
"firstAired": "1991-04-27T00:00:00.000Z",
|
||||
"rating": "7.3",
|
||||
"id": "tt0098798:1:18",
|
||||
"overview": "",
|
||||
"imdb_id": "tt0579956"
|
||||
},
|
||||
{
|
||||
"name": "Good Night, Central City",
|
||||
"season": 1,
|
||||
"number": 19,
|
||||
"firstAired": "1991-05-04T00:00:00.000Z",
|
||||
"rating": "7.1",
|
||||
"id": "tt0098798:1:19",
|
||||
"overview": "",
|
||||
"imdb_id": "tt0579960"
|
||||
},
|
||||
{
|
||||
"name": "Alpha",
|
||||
"season": 1,
|
||||
"number": 20,
|
||||
"firstAired": "1991-05-11T00:00:00.000Z",
|
||||
"rating": "7.5",
|
||||
"id": "tt0098798:1:20",
|
||||
"overview": "",
|
||||
"imdb_id": "tt0579951"
|
||||
},
|
||||
{
|
||||
"name": "Trial of the Trickster",
|
||||
"season": 1,
|
||||
"number": 21,
|
||||
"firstAired": "1991-05-18T00:00:00.000Z",
|
||||
"rating": "7.9",
|
||||
"id": "tt0098798:1:21",
|
||||
"overview": "",
|
||||
"imdb_id": "tt0579968"
|
||||
}
|
||||
],
|
||||
"id": "tt0098798",
|
||||
"behaviorHints": {
|
||||
"defaultVideoId": null,
|
||||
"hasScheduledVideos": false
|
||||
}
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user