Merge pull request #69 from iPromKnight/ts-repo

deployment/docker/.env.example

@@ -23,7 +23,9 @@ RABBIT_URI=amqp://guest:guest@rabbitmq:5672/?heartbeat=30
 QUEUE_NAME=ingested
 JOB_CONCURRENCY=5
 JOBS_ENABLED=true
-MAX_SINGLE_TORRENT_CONNECTIONS=10
+LOG_LEVEL=info # can be debug for extra verbosity (a lot more verbosity - useful for development)
+MAX_CONNECTIONS_PER_TORRENT=10
+MAX_CONNECTIONS_OVERALL=100
 TORRENT_TIMEOUT=30000
 UDP_TRACKERS_ENABLED=true
 CONSUMER_REPLICAS=3

src/node/consumer/.eslintignore

@@ -1 +1,3 @@
-*.ts
+dist/
+esbuild.ts
+jest.config.ts

src/node/consumer/.eslintrc

@@ -0,0 +1,84 @@
+{
+  "root": true,
+  "extends": [
+    "eslint:recommended",
+    "plugin:@typescript-eslint/recommended"
+  ],
+  "parser": "@typescript-eslint/parser",
+  "parserOptions": {
+    "project": [
+      "./tsconfig.json"
+    ]
+  },
+  "plugins": [
+    "@typescript-eslint",
+    "import",
+    "import-helpers"
+  ],
+  "rules": {
+    "default-case": "off",
+    "import/no-duplicates": "error",
+    "import/no-extraneous-dependencies": "error",
+    "import/order": "off",
+    "import-helpers/order-imports": [
+      "warn",
+      {
+        "alphabetize": {
+          "order": "asc"
+        }
+      }
+    ],
+    "lines-between-class-members": [
+      "error",
+      "always",
+      {
+        "exceptAfterSingleLine": true
+      }
+    ],
+    "no-continue": "off",
+    "no-param-reassign": "off",
+    "no-plusplus": [
+      "error",
+      {
+        "allowForLoopAfterthoughts": true
+      }
+    ],
+    "no-restricted-syntax": "off",
+    "no-unused-expressions": [
+      "off",
+      {
+        "allowShortCircuit": true
+      }
+    ],
+    "no-unused-vars": "off",
+    "no-use-before-define": "off",
+    "one-var": [
+      "error",
+      {
+        "uninitialized": "consecutive"
+      }
+    ],
+    "prefer-destructuring": "error",
+    "@typescript-eslint/explicit-function-return-type": "error",
+    "@typescript-eslint/consistent-type-assertions": [
+      "error",
+      {
+        "assertionStyle": "as",
+        "objectLiteralTypeAssertions": "never"
+      }
+    ]
+  },
+  "overrides": [
+    {
+      "files": [
+        "*.test.ts"
+      ],
+      "rules": {
+        "@typescript-eslint/consistent-type-assertions": "off",
+        "@typescript-eslint/explicit-function-return-type": "off",
+        "@typescript-eslint/no-explicit-any": "off",
+        "@typescript-eslint/no-unused-vars": "off"
+      }
+    }
+  ]
+}

src/node/consumer/.eslintrc.cjs

@@ -1,39 +0,0 @@
-/** @type {import("eslint").ESLint.Options} */
-module.exports = {
-    env: {
-        es2024: true,
-        node: true,
-    },
-    globals: {
-        Atomics: "readonly",
-        SharedArrayBuffer: "readonly",
-    },
-    parserOptions: {
-        sourceType: "module",
-    },
-    plugins: ["import", "import-helpers"],
-    rules: {
-        "default-case": "off",
-        "import/no-duplicates": "off",
-        "import/no-extraneous-dependencies": ["off", { devDependencies: ["backend", "frontend", "mobile"] }],
-        "import/order": "off",
-        "import-helpers/order-imports": [
-            "warn",
-            {
-                alphabetize: {
-                    order: "asc",
-                },
-            },
-        ],
-        "lines-between-class-members": ["error", "always", { exceptAfterSingleLine: true }],
-        "no-continue": "off",
-        "no-param-reassign": "off",
-        "no-plusplus": ["error", { allowForLoopAfterthoughts: true }],
-        "no-restricted-syntax": "off",
-        "no-unused-expressions": ["off", { allowShortCircuit: true }],
-        "no-unused-vars": "off",
-        "no-use-before-define": "off",
-        "one-var": ["error", { uninitialized: "consecutive" }],
-        "prefer-destructuring": "off",
-    },
-};

src/node/consumer/.nvmrc

@@ -0,0 +1 @@
+v20.10.0

src/node/consumer/Dockerfile

@@ -1,8 +1,6 @@
 FROM node:lts-buster-slim as builder
 
-RUN apt-get update && \
+RUN apt update && apt install -y git && rm -rf /var/lib/apt/lists/*
-    apt-get install -y git && \
-    rm -rf /var/lib/apt/lists/*
 
 WORKDIR /app
 
@@ -10,16 +8,16 @@ COPY package*.json ./
 RUN npm install
 COPY . .
 RUN npm run build
+RUN npm prune --omit=dev
 
-# --- Runtime Stage ---
 FROM node:lts-buster-slim
 
 WORKDIR /app
 
-ENV NODE_ENV production
-
 COPY --from=builder /app ./
-RUN npm prune --omit=dev
+
+ENV NODE_ENV production
+ENV NODE_OPTIONS "--no-deprecation"
 
 # CIS-DI-0001
 RUN useradd -d /home/consumer -m -s /bin/bash consumer

src/node/consumer/esbuild.ts

@@ -1,8 +1,6 @@
 import { build } from "esbuild";
 import { readFileSync, rmSync } from "fs";
 
-const { devDependencies } = JSON.parse(readFileSync("./package.json", "utf8"));
-
 const start = Date.now();
 
 try {
@@ -13,9 +11,8 @@ try {
     build({
         bundle: true,
         entryPoints: [
-            "./src/index.js",
+            "./src/main.ts",
         ],
-        external: [...(devDependencies && Object.keys(devDependencies))],
         keepNames: true,
         minify: true,
         outbase: "./src",
@@ -42,7 +39,6 @@ try {
             }
         ],
     }).then(() => {
-        // biome-ignore lint/style/useTemplate: <explanation>
         console.log("⚡ " + "\x1b[32m" + `Done in ${Date.now() - start}ms`);
     });
 } catch (e) {

src/node/consumer/jest.config.ts

@@ -0,0 +1,14 @@
+import { pathsToModuleNameMapper } from 'ts-jest';
+import { compilerOptions } from './tsconfig.json';
+
+export default {
+    preset: 'ts-jest',
+    testEnvironment: 'node',
+    moduleNameMapper: pathsToModuleNameMapper(compilerOptions.paths, { prefix: '<rootDir>/src/' }),
+    modulePaths: [
+        '<rootDir>'
+    ],
+    transform: {
+        '^.+\\.tsx?$': 'ts-jest',
+    },
+};

src/node/consumer/jsconfig.json

@@ -1,21 +0,0 @@
-{
-  "compilerOptions": {
-    "baseUrl": "./src",
-    "checkJs": true,
-    "isolatedModules": true,
-    "lib": ["es6"],
-    "module": "ESNext",
-    "moduleResolution": "node",
-    "outDir": "./dist",
-    "pretty": true,
-    "removeComments": true,
-    "resolveJsonModule": true,
-    "rootDir": "./src",
-    "skipLibCheck": true,
-    "sourceMap": true,
-    "target": "ES6",
-    "types": ["node"],
-    "typeRoots": ["node_modules/@types", "src/@types"]
-  },
-  "exclude": ["node_modules"]
-}

src/node/consumer/package.json

@@ -3,10 +3,14 @@
   "version": "0.0.1",
   "type": "module",
   "scripts": {
-    "build": "node esbuild.js",
+    "clean": "rm -rf dist",
-    "dev": "tsx watch --ignore node_modules src/index.js | pino-pretty",
+    "build": "tsx esbuild.ts",
-    "start": "node dist/index.cjs",
+    "dev": "tsx watch --ignore node_modules src/main.ts | pino-pretty",
-    "lint": "eslint . --ext .ts,.js"
+    "start": "node dist/main.cjs",
+    "lint": "eslint ./src --ext .ts,.js",
+    "lint-fix": "npm run lint -- --fix",
+    "test": "jest",
+    "test:watch": "jest --watch"
   },
   "license": "MIT",
   "dependencies": {
@@ -16,7 +20,7 @@
     "bottleneck": "^2.19.5",
     "cache-manager": "^5.4.0",
     "google-sr": "^3.2.1",
-    "jaro-winkler": "^0.2.8",
+    "inversify": "^6.0.2",
     "magnet-uri": "^6.2.0",
     "moment": "^2.30.1",
     "name-to-imdb": "^3.0.4",
@@ -24,18 +28,32 @@
     "pg": "^8.11.3",
     "pg-hstore": "^2.3.4",
     "pino": "^8.18.0",
-    "sequelize": "^6.31.1",
+    "reflect-metadata": "^0.2.1",
-    "torrent-stream": "^1.2.1",
+    "sequelize": "^6.36.0",
-    "user-agents": "^1.0.1444"
+    "sequelize-typescript": "^2.1.6",
+    "torrent-stream": "^1.2.1"
   },
   "devDependencies": {
-    "@types/node": "^20.11.6",
+    "@types/amqplib": "^0.10.4",
-    "@types/stremio-addon-sdk": "^1.6.10",
+    "@types/jest": "^29.5.12",
+    "@types/magnet-uri": "^5.1.5",
+    "@types/node": "^20.11.16",
+    "@types/pg": "^8.11.0",
+    "@types/torrent-stream": "^0.0.9",
+    "@types/validator": "^13.11.8",
+    "@typescript-eslint/eslint-plugin": "^6.21.0",
+    "@typescript-eslint/parser": "^6.21.0",
     "esbuild": "^0.20.0",
     "eslint": "^8.56.0",
     "eslint-plugin-import": "^2.29.1",
     "eslint-plugin-import-helpers": "^1.3.1",
+    "jest": "^29.7.0",
+    "msw": "^2.1.7",
+    "pino-pretty": "^10.3.1",
+    "ts-jest": "^29.1.2",
+    "ts-node": "^10.9.2",
+    "tsconfig-paths": "^4.2.0",
     "tsx": "^4.7.0",
-    "pino-pretty": "^10.3.1"
+    "typescript": "^5.3.3"
   }
 }

src/node/consumer/src/index.js

@@ -1,9 +0,0 @@
-import { listenToQueue } from './jobs/processTorrents.js';
-import { connect } from './lib/repository.js';
-import { getTrackers } from "./lib/trackerService.js";
-
-(async () => {
-    await getTrackers();
-    await connect();
-    await listenToQueue();
-})();

src/node/consumer/src/jobs/processTorrents.js

@@ -1,37 +0,0 @@
-import amqp from 'amqplib'
-import { rabbitConfig, jobConfig } from '../lib/config.js'
-import { processTorrentRecord } from "../lib/ingestedTorrent.js";
-import {logger} from "../lib/logger.js";
-
-const assertQueueOptions = { durable: true }
-const consumeQueueOptions = { noAck: false }
-
-const processMessage = msg => processTorrentRecord(getMessageAsJson(msg));
-
-const getMessageAsJson = msg => 
-    JSON.parse(msg.content.toString()).message;
-
-const assertAndConsumeQueue = async channel => {
-    logger.info('Worker is running! Waiting for new torrents...')
-
-    const ackMsg = msg => 
-    processMessage(msg)
-        .then(() => channel.ack(msg))
-        .catch(error => logger.error('Failed processing torrent', error));
-
-    channel.assertQueue(rabbitConfig.QUEUE_NAME, assertQueueOptions)
-        .then(() => channel.prefetch(jobConfig.JOB_CONCURRENCY))
-        .then(() => channel.consume(rabbitConfig.QUEUE_NAME, ackMsg, consumeQueueOptions))
-        .catch(error => logger.error('Failed to setup channel', error));
-}
-
-export const listenToQueue = async () => {
-        if (!jobConfig.JOBS_ENABLED) {
-            return;
-        }
-
-        return amqp.connect(rabbitConfig.URI)
-            .then(connection => connection.createChannel())
-            .then(channel => assertAndConsumeQueue(channel))
-            .catch(error => logger.error('Failed to connect and setup channel', error));
-    };

src/node/consumer/src/lib/cache.js

@@ -1,84 +0,0 @@
-import { createCache, memoryStore} from 'cache-manager';
-import { mongoDbStore } from '@tirke/node-cache-manager-mongodb'
-import { cacheConfig } from './config.js';
-import { logger } from './logger.js';
-import { CacheType } from "./types.js";
-
-const GLOBAL_KEY_PREFIX = 'knightcrawler-consumer';
-const IMDB_ID_PREFIX = `${GLOBAL_KEY_PREFIX}|imdb_id`;
-const KITSU_ID_PREFIX = `${GLOBAL_KEY_PREFIX}|kitsu_id`;
-const METADATA_PREFIX = `${GLOBAL_KEY_PREFIX}|metadata`;
-const TRACKERS_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|trackers`;
-
-const GLOBAL_TTL = process.env.METADATA_TTL || 7 * 24 * 60 * 60; // 7 days
-const MEMORY_TTL = process.env.METADATA_TTL || 2 * 60 * 60; // 2 hours
-const TRACKERS_TTL = 2 * 24 * 60 * 60; // 2 days
-
-const initiateMemoryCache = () =>
-    createCache(memoryStore(), {
-        ttl: parseInt(MEMORY_TTL)
-    }); 
-
-const initiateMongoCache = () => {
-    const store = mongoDbStore({
-        collectionName: cacheConfig.COLLECTION_NAME,
-        ttl: parseInt(GLOBAL_TTL),
-        url: cacheConfig.MONGO_URI,
-        mongoConfig:{
-            socketTimeoutMS: 120000,
-            appName: 'knightcrawler-consumer',
-        }
-    });    
-    
-    return createCache(store, {
-        ttl: parseInt(GLOBAL_TTL),
-    });
-}
-
-const initiateRemoteCache = ()=> {
-    if (cacheConfig.NO_CACHE) {
-        logger.debug('Cache is disabled');
-        return null;
-    }
-    return cacheConfig.MONGO_URI ? initiateMongoCache() : initiateMemoryCache();
-}
-
-const getCacheType = (cacheType) => {
-    switch (cacheType) {
-        case CacheType.MEMORY:
-            return memoryCache;
-        case CacheType.MONGODB:
-            return remoteCache;
-        default:
-            return null;
-    }
-}
-
-const memoryCache = initiateMemoryCache()
-const remoteCache = initiateRemoteCache()
-
-const cacheWrap = async (cacheType, key, method, options) => {
-    const cache = getCacheType(cacheType);
-    
-    if (cacheConfig.NO_CACHE || !cache) {
-        return method();
-    }
-
-    logger.debug(`Cache type: ${cacheType}`);
-    logger.debug(`Cache key: ${key}`);
-    logger.debug(`Cache options: ${JSON.stringify(options)}`);
-        
-    return cache.wrap(key, method, options.ttl);
-}
-
-export const cacheWrapImdbId = (key, method) => 
-    cacheWrap(CacheType.MONGODB, `${IMDB_ID_PREFIX}:${key}`, method, { ttl: parseInt(GLOBAL_TTL) });
-
-export const cacheWrapKitsuId = (key, method) => 
-    cacheWrap(CacheType.MONGODB, `${KITSU_ID_PREFIX}:${key}`, method, { ttl: parseInt(GLOBAL_TTL) });
-
-export const cacheWrapMetadata = (id, method) => 
-    cacheWrap(CacheType.MEMORY, `${METADATA_PREFIX}:${id}`, method, { ttl: parseInt(MEMORY_TTL) });
-
-export const cacheTrackers = (method) => 
-    cacheWrap(CacheType.MEMORY, `${TRACKERS_KEY_PREFIX}`, method, { ttl: parseInt(TRACKERS_TTL) });

src/node/consumer/src/lib/config.js

@@ -1,63 +0,0 @@
-export const rabbitConfig = {
-    URI: process.env.RABBIT_URI || 'amqp://localhost',
-    QUEUE_NAME: process.env.QUEUE_NAME || 'test-queue'
-}
-
-export const cacheConfig = {
-    MONGODB_HOST: process.env.MONGODB_HOST || 'mongodb',
-    MONGODB_PORT: process.env.MONGODB_PORT || '27017',
-    MONGODB_DB: process.env.MONGODB_DB || 'knightcrawler',
-    MONGO_INITDB_ROOT_USERNAME: process.env.MONGO_INITDB_ROOT_USERNAME || 'mongo',
-    MONGO_INITDB_ROOT_PASSWORD: process.env.MONGO_INITDB_ROOT_PASSWORD || 'mongo',
-    NO_CACHE: parseBool(process.env.NO_CACHE, false),
-    COLLECTION_NAME: process.env.MONGODB_COLLECTION || 'knightcrawler_consumer_collection'
-}
-
-// Combine the environment variables into a connection string
-// The combined string will look something like:
-// 'mongodb://mongo:mongo@localhost:27017/knightcrawler?authSource=admin'
-cacheConfig.MONGO_URI = 'mongodb://' + cacheConfig.MONGO_INITDB_ROOT_USERNAME + ':' + cacheConfig.MONGO_INITDB_ROOT_PASSWORD + '@' + cacheConfig.MONGODB_HOST + ':' + cacheConfig.MONGODB_PORT + '/' + cacheConfig.MONGODB_DB + '?authSource=admin';
-
-export const databaseConfig = {
-    POSTGRES_HOST: process.env.POSTGRES_HOST || 'postgres',
-    POSTGRES_PORT: process.env.POSTGRES_PORT || '5432',
-    POSTGRES_DB: process.env.POSTGRES_DB || 'knightcrawler',
-    POSTGRES_USER: process.env.POSTGRES_USER || 'postgres',
-    POSTGRES_PASSWORD: process.env.POSTGRES_PASSWORD || 'postgres',
-    AUTO_CREATE_AND_APPLY_MIGRATIONS: parseBool(process.env.AUTO_CREATE_AND_APPLY_MIGRATIONS, false)
-}
-
-// Combine the environment variables into a connection string
-// The combined string will look something like:
-// 'postgres://postgres:postgres@localhost:5432/knightcrawler'
-databaseConfig.POSTGRES_URI = 'postgres://' + databaseConfig.POSTGRES_USER + ':' + databaseConfig.POSTGRES_PASSWORD + '@' + databaseConfig.POSTGRES_HOST + ':' + databaseConfig.POSTGRES_PORT + '/' + databaseConfig.POSTGRES_DB;
-
-export const jobConfig = {
-    JOB_CONCURRENCY: parseInt(process.env.JOB_CONCURRENCY || 1),
-    JOBS_ENABLED: parseBool(process.env.JOBS_ENABLED || true)
-}
-
-export const metadataConfig = {
-    IMDB_CONCURRENT: parseInt(process.env.IMDB_CONCURRENT || 1),
-    IMDB_INTERVAL_MS: parseInt(process.env.IMDB_INTERVAL_MS || 1000),
-}
-
-export const trackerConfig = {
-    TRACKERS_URL: process.env.TRACKERS_URL || 'https://ngosang.github.io/trackerslist/trackers_all.txt',
-    UDP_ENABLED: parseBool(process.env.UDP_TRACKERS_ENABLED || false),
-}
-
-export const torrentConfig = {
-    MAX_CONNECTIONS_PER_TORRENT: parseInt(process.env.MAX_SINGLE_TORRENT_CONNECTIONS || 20),
-    TIMEOUT: parseInt(process.env.TORRENT_TIMEOUT || 30000),
-}
-
-function parseBool(boolString, defaultValue) {
-    const isString = typeof boolString === 'string' || boolString instanceof String;
-
-    if (!isString) {
-        return defaultValue;
-    }
-
-    return boolString.toLowerCase() === 'true' ? true : defaultValue;
-}

src/node/consumer/src/lib/enums/cache_types.ts

@@ -0,0 +1,4 @@
+export enum CacheType {
+    Memory = 'memory',
+    MongoDb = 'mongodb'
+}

src/node/consumer/src/lib/enums/torrent_types.ts

@@ -0,0 +1,5 @@
+export enum TorrentType {
+    Series = 'Series',
+    Movie = 'Movie',
+    Anime = 'anime',
+}

src/node/consumer/src/lib/extension.js

@@ -1,62 +0,0 @@
-const VIDEO_EXTENSIONS = [
-  "3g2",
-  "3gp",
-  "avi",
-  "flv",
-  "mkv",
-  "mk3d",
-  "mov",
-  "mp2",
-  "mp4",
-  "m4v",
-  "mpe",
-  "mpeg",
-  "mpg",
-  "mpv",
-  "webm",
-  "wmv",
-  "ogm",
-  "divx"
-];
-const SUBTITLE_EXTENSIONS = [
-  "aqt",
-  "gsub",
-  "jss",
-  "sub",
-  "ttxt",
-  "pjs",
-  "psb",
-  "rt",
-  "smi",
-  "slt",
-  "ssf",
-  "srt",
-  "ssa",
-  "ass",
-  "usf",
-  "idx",
-  "vtt"
-];
-const DISK_EXTENSIONS = [
-  "iso",
-  "m2ts",
-  "ts",
-  "vob"
-]
-
-export function isVideo(filename) {
-  return isExtension(filename, VIDEO_EXTENSIONS);
-}
-
-export function isSubtitle(filename) {
-  return isExtension(filename, SUBTITLE_EXTENSIONS);
-}
-
-export function isDisk(filename) {
-  return isExtension(filename, DISK_EXTENSIONS);
-}
-
-export function isExtension(filename, extensions) {
-  const extensionMatch = filename.match(/\.(\w{2,4})$/);
-  return extensionMatch && extensions.includes(extensionMatch[1].toLowerCase());
-}

src/node/consumer/src/lib/helpers/boolean_helpers.ts

@@ -0,0 +1,18 @@
+export const BooleanHelpers = {
+    parseBool: (value: string | undefined, defaultValue: boolean): boolean => {
+        switch (value?.trim().toLowerCase()) {
+            case undefined:
+                return defaultValue;
+            case 'true':
+            case 'yes':
+            case '1':
+                return true;
+            case 'false':
+            case 'no':
+            case '0':
+                return false;
+            default:
+                throw new Error(`Invalid boolean value: '${value}'. Allowed values are 'true', 'false', 'yes', 'no', '1', or '0'.`);
+        }
+    }
+}

src/node/consumer/src/lib/helpers/extension_helpers.ts

@@ -0,0 +1,66 @@
+const VIDEO_EXTENSIONS = [
+    "3g2",
+    "3gp",
+    "avi",
+    "flv",
+    "mkv",
+    "mk3d",
+    "mov",
+    "mp2",
+    "mp4",
+    "m4v",
+    "mpe",
+    "mpeg",
+    "mpg",
+    "mpv",
+    "webm",
+    "wmv",
+    "ogm",
+    "divx"
+];
+
+const SUBTITLE_EXTENSIONS = [
+    "aqt",
+    "gsub",
+    "jss",
+    "sub",
+    "ttxt",
+    "pjs",
+    "psb",
+    "rt",
+    "smi",
+    "slt",
+    "ssf",
+    "srt",
+    "ssa",
+    "ass",
+    "usf",
+    "idx",
+    "vtt"
+];
+
+const DISK_EXTENSIONS = [
+    "iso",
+    "m2ts",
+    "ts",
+    "vob"
+];
+
+export const ExtensionHelpers = {
+    isVideo(filename: string): boolean {
+        return this.isExtension(filename, VIDEO_EXTENSIONS);
+    },
+
+    isSubtitle(filename: string): boolean {
+        return this.isExtension(filename, SUBTITLE_EXTENSIONS);
+    },
+
+    isDisk(filename: string): boolean {
+        return this.isExtension(filename, DISK_EXTENSIONS);
+    },
+
+    isExtension(filename: string, extensions: string[]): boolean {
+        const extensionMatch = filename.match(/\.(\w{2,4})$/);
+        return extensionMatch !== null && extensions.includes(extensionMatch[1].toLowerCase());
+    }
+}

src/node/consumer/src/lib/helpers/promises_helpers.ts

@@ -0,0 +1,37 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+export const PromiseHelpers = {
+
+    sequence: async function (promises: (() => Promise<any>)[]): Promise<any> {
+        return promises.reduce((promise, func) =>
+            promise.then(result => func().then(res => result.concat(res))), Promise.resolve([]));
+    },
+
+    first: async function (promises: any): Promise<any> {
+        return Promise.all(promises.map((p: any) => {
+            return p.then((val: any) => Promise.reject(val), (err: any) => Promise.resolve(err));
+        })).then(
+            (errors) => Promise.reject(errors),
+            (val) => Promise.resolve(val)
+        );
+    },
+
+    delay: async function (duration: number): Promise<void> {
+        return new Promise<void>(resolve => setTimeout(() => resolve(), duration));
+    },
+
+    timeout: async function (timeoutMs: number, promise: any, message = 'Timed out'): Promise<any> {
+        return Promise.race([
+            promise,
+            new Promise(function (resolve, reject) {
+                setTimeout(function () {
+                    reject(message);
+                }, timeoutMs);
+            })
+        ]);
+    },
+
+    mostCommonValue: function (array: any[]): any {
+        return array.sort((a, b) => array.filter(v => v === a).length - array.filter(v => v === b).length).pop();
+    }
+};
+/* eslint-enable @typescript-eslint/no-explicit-any */

src/node/consumer/src/lib/ingestedTorrent.js

@@ -1,46 +0,0 @@
-import { createTorrentEntry, checkAndUpdateTorrent } from './torrentEntries.js';
-import {getTrackers} from "./trackerService.js";
-import { TorrentType } from './types.js';
-import {logger} from "./logger.js";
-
-export async function processTorrentRecord(torrent) {
-  const {category} = torrent;
-  const type = category === 'tv' ? TorrentType.SERIES : TorrentType.MOVIE;
-  const torrentInfo = await parseTorrent(torrent, type);
-  logger.info(`Processing torrent ${torrentInfo.title} with infoHash ${torrentInfo.infoHash}`)
-
-  if (await checkAndUpdateTorrent(torrentInfo)) {
-    return torrentInfo;
-  }
-
-  return createTorrentEntry(torrentInfo);
-}
-
-async function assignTorrentTrackers() {
-    const trackers = await getTrackers();
-    return trackers.join(',');
-}
-
-async function parseTorrent(torrent, category) {
-  const infoHash = torrent.infoHash?.trim().toLowerCase()
-  return {
-    title: torrent.name,
-    torrentId: `${torrent.name}_${infoHash}`,
-    infoHash: infoHash,
-    seeders: 100,
-    size: torrent.size,
-    uploadDate: torrent.createdAt,
-    imdbId: parseImdbId(torrent),
-    type: category,
-    provider: torrent.source,
-    trackers: await assignTorrentTrackers(),
-  }
-}
-
-function parseImdbId(torrent) {
-  if (torrent.imdb === undefined || torrent.imdb === null) {
-    return undefined;
-  }
-  
-  return torrent.imdb;
-}

src/node/consumer/src/lib/interfaces/cache_options.ts

@@ -0,0 +1,3 @@
+export interface ICacheOptions {
+    ttl: number;
+}

src/node/consumer/src/lib/interfaces/cache_service.ts

@@ -0,0 +1,11 @@
+import {CacheMethod} from "@services/cache_service";
+
+/* eslint-disable @typescript-eslint/no-explicit-any */
+export interface ICacheService {
+    cacheWrapImdbId: (key: string, method: CacheMethod) => Promise<any>;
+    cacheWrapKitsuId: (key: string, method: CacheMethod) => Promise<any>;
+    cacheWrapMetadata: (id: string, method: CacheMethod) => Promise<any>;
+    cacheTrackers: (method: CacheMethod) => Promise<any>;
+}
+
+/* eslint-enable @typescript-eslint/no-explicit-any */

src/node/consumer/src/lib/interfaces/cinemeta_metadata.ts

@@ -0,0 +1,84 @@
+import {ICommonVideoMetadata} from "@interfaces/common_video_metadata";
+
+export interface ICinemetaJsonResponse {
+    meta?: ICinemetaMetaData;
+    trailerStreams?: ICinemetaTrailerStream[];
+    links?: ICinemetaLink[];
+    behaviorHints?: ICinemetaBehaviorHints;
+}
+
+export interface ICinemetaMetaData {
+    awards?: string;
+    cast?: string[];
+    country?: string;
+    description?: string;
+    director?: null;
+    dvdRelease?: null;
+    genre?: string[];
+    imdbRating?: string;
+    name?: string;
+    popularity?: number;
+    poster?: string;
+    released?: string;
+    runtime?: string;
+    status?: string;
+    tvdb_id?: number;
+    type?: string;
+    writer?: string[];
+    year?: string;
+    background?: string;
+    logo?: string;
+    popularities?: ICinemetaPopularities;
+    moviedb_id?: number;
+    slug?: string;
+    trailers?: ICinemetaTrailer[];
+    id?: string;
+    genres?: string[];
+    releaseInfo?: string;
+    videos?: ICinemetaVideo[];
+}
+
+export interface ICinemetaPopularities {
+    PXS_TEST?: number;
+    PXS?: number;
+    SCM?: number;
+    EXMD?: number;
+    ALLIANCE?: number;
+    EJD?: number;
+    moviedb?: number;
+    trakt?: number;
+    stremio?: number;
+    stremio_lib?: number;
+}
+
+export interface ICinemetaTrailer {
+    source?: string;
+    type?: string;
+}
+
+export interface ICinemetaVideo extends ICommonVideoMetadata {
+    name?: string;
+    number?: number;
+    firstAired?: string;
+    tvdb_id?: number;
+    rating?: string;
+    overview?: string;
+    thumbnail?: string;
+    description?: string;
+}
+
+export interface ICinemetaTrailerStream {
+    title?: string;
+    ytId?: string;
+}
+
+export interface ICinemetaLink {
+    name?: string;
+    category?: string;
+    url?: string;
+}
+
+export interface ICinemetaBehaviorHints {
+    defaultVideoId?: null;
+    hasScheduledVideos?: boolean;
+}

src/node/consumer/src/lib/interfaces/common_video_metadata.ts

@@ -0,0 +1,9 @@
+export interface ICommonVideoMetadata {
+    season?: number;
+    episode?: number;
+    released?: string;
+    title?: string;
+    name?: string;
+    id?: string;
+    imdb_id?: string;
+}

src/node/consumer/src/lib/interfaces/ingested_rabbit_message.ts

@@ -0,0 +1,15 @@
+export interface IIngestedRabbitTorrent {
+    name: string;
+    source: string;
+    category: string;
+    infoHash: string;
+    size: string;
+    seeders: number;
+    leechers: number;
+    imdb: string;
+    processed: boolean;
+}
+
+export interface IIngestedRabbitMessage {
+    message: IIngestedRabbitTorrent;
+}

src/node/consumer/src/lib/interfaces/kitsu_catalog_metadata.ts

@@ -0,0 +1,23 @@
+import {IKitsuLink, IKitsuTrailer} from "@interfaces/kitsu_metadata";
+
+export interface IKitsuCatalogJsonResponse {
+    metas: IKitsuCatalogMetaData[];
+}
+
+export interface IKitsuCatalogMetaData {
+    id: string;
+    type: string;
+    animeType: string;
+    name: string;
+    aliases: string[];
+    description: string;
+    releaseInfo: string;
+    runtime: string;
+    imdbRating: string;
+    genres: string[];
+    logo?: string;
+    poster: string;
+    background: string;
+    trailers: IKitsuTrailer[];
+    links: IKitsuLink[];
+}

src/node/consumer/src/lib/interfaces/kitsu_metadata.ts

@@ -0,0 +1,49 @@
+import {ICommonVideoMetadata} from "@interfaces/common_video_metadata";
+
+export interface IKitsuJsonResponse {
+    cacheMaxAge?: number;
+    meta?: IKitsuMeta;
+}
+
+export interface IKitsuMeta {
+    aliases?: string[];
+    animeType?: string;
+    background?: string;
+    description?: string;
+    country?: string;
+    genres?: string[];
+    id?: string;
+    imdbRating?: string;
+    imdb_id?: string;
+    kitsu_id?: string;
+    links?: IKitsuLink[];
+    logo?: string;
+    name?: string;
+    poster?: string;
+    releaseInfo?: string;
+    runtime?: string;
+    slug?: string;
+    status?: string;
+    trailers?: IKitsuTrailer[];
+    type?: string;
+    userCount?: number;
+    videos?: IKitsuVideo[];
+    year?: string;
+}
+
+export interface IKitsuVideo extends ICommonVideoMetadata {
+    imdbEpisode?: number;
+    imdbSeason?: number;
+    thumbnail?: string;
+}
+
+export interface IKitsuTrailer {
+    source?: string;
+    type?: string;
+}
+
+export interface IKitsuLink {
+    name?: string;
+    category?: string;
+    url?: string;
+}

src/node/consumer/src/lib/interfaces/logging_service.ts

@@ -0,0 +1,12 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+export interface ILoggingService {
+    info(message: string, ...args: any[]): void;
+
+    error(message: string, ...args: any[]): void;
+
+    debug(message: string, ...args: any[]): void;
+
+    warn(message: string, ...args: any[]): void;
+}
+
+/* eslint-enable @typescript-eslint/no-explicit-any */

src/node/consumer/src/lib/interfaces/metadata_query.ts

@@ -0,0 +1,9 @@
+export interface IMetaDataQuery {
+    title?: string
+    type?: string
+    year?: number | string
+    date?: string
+    season?: number
+    episode?: number
+    id?: string | number
+}

src/node/consumer/src/lib/interfaces/metadata_response.ts

@@ -0,0 +1,15 @@
+import {ICommonVideoMetadata} from "@interfaces/common_video_metadata";
+
+export interface IMetadataResponse {
+    kitsuId?: number;
+    imdbId?: number;
+    type?: string;
+    title?: string;
+    year?: number;
+    country?: string;
+    genres?: string[];
+    status?: string;
+    videos?: ICommonVideoMetadata[];
+    episodeCount?: number[];
+    totalCount?: number;
+}

src/node/consumer/src/lib/interfaces/metadata_service.ts

@@ -0,0 +1,14 @@
+import {IMetaDataQuery} from "@interfaces/metadata_query";
+import {IMetadataResponse} from "@interfaces/metadata_response";
+
+export interface IMetadataService {
+    getKitsuId(info: IMetaDataQuery): Promise<number | Error>;
+
+    getImdbId(info: IMetaDataQuery): Promise<string | undefined>;
+
+    getMetadata(query: IMetaDataQuery): Promise<IMetadataResponse | Error>;
+
+    isEpisodeImdbId(imdbId: string | undefined): Promise<boolean>;
+
+    escapeTitle(title: string): string;
+}

src/node/consumer/src/lib/interfaces/parse_torrent_title_result.ts

@@ -0,0 +1,36 @@
+import {IFileAttributes} from "@repository/interfaces/file_attributes";
+
+export interface IParseTorrentTitleResult {
+    title?: string;
+    date?: string;
+    year?: number | string;
+    resolution?: string;
+    extended?: boolean;
+    unrated?: boolean;
+    proper?: boolean;
+    repack?: boolean;
+    convert?: boolean;
+    hardcoded?: boolean;
+    retail?: boolean;
+    remastered?: boolean;
+    complete?: boolean;
+    region?: string;
+    container?: string;
+    extension?: string;
+    source?: string;
+    codec?: string;
+    bitDepth?: string;
+    hdr?: Array<string>;
+    audio?: string;
+    group?: string;
+    volumes?: Array<number>;
+    seasons?: Array<number>;
+    season?: number;
+    episodes?: Array<number>;
+    episode?: number;
+    languages?: string;
+    dubbed?: boolean;
+    videoFile?: IFileAttributes;
+    folderName?: string;
+    fileName?: string;
+}

src/node/consumer/src/lib/interfaces/parsed_torrent.ts

@@ -0,0 +1,18 @@
+import {TorrentType} from "@enums/torrent_types";
+import {IParseTorrentTitleResult} from "@interfaces/parse_torrent_title_result";
+import {ITorrentFileCollection} from "@interfaces/torrent_file_collection";
+
+export interface IParsedTorrent extends IParseTorrentTitleResult {
+    size?: number;
+    isPack?: boolean;
+    imdbId?: string | number;
+    kitsuId?: number;
+    trackers?: string;
+    provider?: string | null;
+    infoHash: string;
+    type: string | TorrentType;
+    uploadDate?: Date;
+    seeders?: number;
+    torrentId?: string;
+    fileCollection?: ITorrentFileCollection;
+}

src/node/consumer/src/lib/interfaces/process_torrents_job.ts

@@ -0,0 +1,3 @@
+export interface IProcessTorrentsJob {
+    listenToQueue: () => Promise<void>;
+}

src/node/consumer/src/lib/interfaces/torrent_download_service.ts

@@ -0,0 +1,6 @@
+import {IParsedTorrent} from "@interfaces/parsed_torrent";
+import {ITorrentFileCollection} from "@interfaces/torrent_file_collection";
+
+export interface ITorrentDownloadService {
+    getTorrentFiles(torrent: IParsedTorrent, timeout: number): Promise<ITorrentFileCollection>;
+}

src/node/consumer/src/lib/interfaces/torrent_entries_service.ts

@@ -0,0 +1,18 @@
+import {IParsedTorrent} from "@interfaces/parsed_torrent";
+import {ITorrentAttributes, ITorrentCreationAttributes} from "@repository/interfaces/torrent_attributes";
+import {SkipTorrent} from "@repository/models/skipTorrent";
+import {Torrent} from "@repository/models/torrent";
+
+export interface ITorrentEntriesService {
+    createTorrentEntry(torrent: IParsedTorrent, overwrite: boolean): Promise<void>;
+
+    createSkipTorrentEntry(torrent: ITorrentCreationAttributes): Promise<[SkipTorrent, boolean | null]>;
+
+    getStoredTorrentEntry(torrent: Torrent): Promise<Torrent | SkipTorrent | null | undefined>;
+
+    checkAndUpdateTorrent(torrent: IParsedTorrent): Promise<boolean>;
+
+    createTorrentContents(torrent: Torrent): Promise<void>;
+
+    updateTorrentSeeders(torrent: ITorrentAttributes): Promise<[number] | undefined>;
+}

src/node/consumer/src/lib/interfaces/torrent_file_collection.ts

@@ -0,0 +1,9 @@
+import {IContentAttributes} from "@repository/interfaces/content_attributes";
+import {IFileAttributes} from "@repository/interfaces/file_attributes";
+import {ISubtitleAttributes} from "@repository/interfaces/subtitle_attributes";
+
+export interface ITorrentFileCollection {
+    contents?: IContentAttributes[];
+    videos?: IFileAttributes[];
+    subtitles?: ISubtitleAttributes[];
+}

src/node/consumer/src/lib/interfaces/torrent_file_service.ts

@@ -0,0 +1,8 @@
+import {IParsedTorrent} from "@interfaces/parsed_torrent";
+import {ITorrentFileCollection} from "@interfaces/torrent_file_collection";
+
+export interface ITorrentFileService {
+    parseTorrentFiles(torrent: IParsedTorrent): Promise<ITorrentFileCollection>;
+
+    isPackTorrent(torrent: IParsedTorrent): boolean;
+}

src/node/consumer/src/lib/interfaces/torrent_processing_service.ts

@@ -0,0 +1,5 @@
+import {IIngestedTorrentAttributes} from "@repository/interfaces/ingested_torrent_attributes";
+
+export interface ITorrentProcessingService {
+    processTorrentRecord(torrent: IIngestedTorrentAttributes): Promise<void>;
+}

src/node/consumer/src/lib/interfaces/torrent_subtitle_service.ts

@@ -0,0 +1,5 @@
+import {ITorrentFileCollection} from "@interfaces/torrent_file_collection";
+
+export interface ITorrentSubtitleService {
+    assignSubtitles(fileCollection: ITorrentFileCollection): ITorrentFileCollection;
+}

src/node/consumer/src/lib/interfaces/tracker_service.ts

@@ -0,0 +1,3 @@
+export interface ITrackerService {
+    getTrackers(): Promise<string[]>;
+}

src/node/consumer/src/lib/jobs/process_torrents_job.ts

@@ -0,0 +1,63 @@
+import {IIngestedRabbitMessage, IIngestedRabbitTorrent} from "@interfaces/ingested_rabbit_message";
+import {ILoggingService} from "@interfaces/logging_service";
+import {IProcessTorrentsJob} from "@interfaces/process_torrents_job";
+import {ITorrentProcessingService} from "@interfaces/torrent_processing_service";
+import {IIngestedTorrentAttributes} from "@repository/interfaces/ingested_torrent_attributes";
+import {configurationService} from '@services/configuration_service';
+import {IocTypes} from "@setup/ioc_types";
+import client, {Channel, Connection, ConsumeMessage, Options} from 'amqplib'
+import {inject, injectable} from "inversify";
+
+@injectable()
+export class ProcessTorrentsJob implements IProcessTorrentsJob {
+    @inject(IocTypes.ITorrentProcessingService) torrentProcessingService: ITorrentProcessingService;
+    @inject(IocTypes.ILoggingService) logger: ILoggingService;
+
+    private readonly assertQueueOptions: Options.AssertQueue = {durable: true};
+    private readonly consumeQueueOptions: Options.Consume = {noAck: false};
+
+    async listenToQueue(): Promise<void> {
+        if (!configurationService.jobConfig.JOBS_ENABLED) {
+            return;
+        }
+
+        try {
+            const connection: Connection = await client.connect(configurationService.rabbitConfig.RABBIT_URI);
+            const channel: Channel = await connection.createChannel();
+            await this.assertAndConsumeQueue(channel);
+        } catch (error) {
+            this.logger.error('Failed to connect and setup channel', error);
+        }
+    }
+
+    private processMessage = (msg: ConsumeMessage | null): Promise<void> => {
+        const ingestedTorrent: IIngestedTorrentAttributes = this.getMessageAsJson(msg);
+        return this.torrentProcessingService.processTorrentRecord(ingestedTorrent);
+    };
+
+    private getMessageAsJson = (msg: ConsumeMessage | null): IIngestedTorrentAttributes => {
+        const content = msg?.content.toString('utf8') ?? "{}";
+        const receivedObject: IIngestedRabbitMessage = JSON.parse(content);
+        const receivedTorrent: IIngestedRabbitTorrent = receivedObject.message;
+        return {...receivedTorrent, info_hash: receivedTorrent.infoHash};
+    };
+
+    private assertAndConsumeQueue = async (channel: Channel): Promise<void> => {
+        this.logger.info('Worker is running! Waiting for new torrents...');
+
+        const ackMsg = async (msg: ConsumeMessage | null): Promise<void> => {
+            await this.processMessage(msg)
+                .then(() => this.logger.info('Processed torrent'))
+                .then(() => msg && channel.ack(msg))
+                .catch((error) => this.logger.error('Failed to process torrent', error));
+        }
+
+        try {
+            await channel.assertQueue(configurationService.rabbitConfig.QUEUE_NAME, this.assertQueueOptions);
+            await channel.prefetch(configurationService.jobConfig.JOB_CONCURRENCY);
+            await channel.consume(configurationService.rabbitConfig.QUEUE_NAME, ackMsg, this.consumeQueueOptions);
+        } catch (error) {
+            this.logger.error('Failed to setup channel', error);
+        }
+    };
+}

src/node/consumer/src/lib/logger.js

@@ -1,5 +0,0 @@
-import pino from 'pino';
-
-export const logger = pino({
-    level: process.env.LOG_LEVEL || 'info'
-});

src/node/consumer/src/lib/metadata.js

@@ -1,165 +0,0 @@
-import axios from 'axios';
-import { search } from 'google-sr';
-import nameToImdb from 'name-to-imdb';
-import { cacheWrapImdbId, cacheWrapKitsuId, cacheWrapMetadata } from './cache.js';
-import { TorrentType } from './types.js';
-
-const CINEMETA_URL = 'https://v3-cinemeta.strem.io';
-const KITSU_URL = 'https://anime-kitsu.strem.fun';
-const TIMEOUT = 20000;
-
-export function getMetadata(id, type = TorrentType.SERIES) {
-  if (!id) {
-    return Promise.reject("no valid id provided");
-  }
-
-  const key = Number.isInteger(id) || id.match(/^\d+$/) ? `kitsu:${id}` : id;
-  const metaType = type === TorrentType.MOVIE ? TorrentType.MOVIE : TorrentType.SERIES;
-  return cacheWrapMetadata(key, () => _requestMetadata(`${KITSU_URL}/meta/${metaType}/${key}.json`)
-      .catch(() => _requestMetadata(`${CINEMETA_URL}/meta/${metaType}/${key}.json`))
-      .catch(() => {
-        // try different type in case there was a mismatch
-        const otherType = metaType === TorrentType.MOVIE ? TorrentType.SERIES : TorrentType.MOVIE;
-        return _requestMetadata(`${CINEMETA_URL}/meta/${otherType}/${key}.json`)
-      })
-      .catch((error) => {
-        throw new Error(`failed metadata query ${key} due: ${error.message}`);
-      }));
-}
-
-function _requestMetadata(url) {
-  return axios.get(url, { timeout: TIMEOUT })
-      .then((response) => {
-        const body = response.data;
-        if (body && body.meta && (body.meta.imdb_id || body.meta.kitsu_id)) {
-          return {
-            kitsuId: body.meta.kitsu_id,
-            imdbId: body.meta.imdb_id,
-            type: body.meta.type,
-            title: body.meta.name,
-            year: body.meta.year,
-            country: body.meta.country,
-            genres: body.meta.genres,
-            status: body.meta.status,
-            videos: (body.meta.videos || [])
-                .map((video) => Number.isInteger(video.imdbSeason)
-                    ? {
-                      name: video.name || video.title,
-                      season: video.season,
-                      episode: video.episode,
-                      imdbSeason: video.imdbSeason,
-                      imdbEpisode: video.imdbEpisode
-                    }
-                    : {
-                      name: video.name || video.title,
-                      season: video.season,
-                      episode: video.episode,
-                      kitsuId: video.kitsu_id,
-                      kitsuEpisode: video.kitsuEpisode,
-                      released: video.released
-                    }
-                ),
-            episodeCount: Object.values((body.meta.videos || [])
-                .filter((entry) => entry.season !== 0 && entry.episode !== 0)
-                .sort((a, b) => a.season - b.season)
-                .reduce((map, next) => {
-                  map[next.season] = map[next.season] + 1 || 1;
-                  return map;
-                }, {})),
-            totalCount: body.meta.videos && body.meta.videos
-                .filter((entry) => entry.season !== 0 && entry.episode !== 0).length
-          };
-        } else {
-          throw new Error('No search results');
-        }
-      });
-}
-
-export function escapeTitle(title) {
-  return title.toLowerCase()
-      .normalize('NFKD') // normalize non-ASCII characters
-      .replace(/[\u0300-\u036F]/g, '')
-      .replace(/&/g, 'and')
-      .replace(/[;, ~./]+/g, ' ') // replace dots, commas or underscores with spaces
-      .replace(/[^\w \-()×+#@!'\u0400-\u04ff]+/g, '') // remove all non-alphanumeric chars
-      .replace(/^\d{1,2}[.#\s]+(?=(?:\d+[.\s]*)?[\u0400-\u04ff])/i, '') // remove russian movie numbering
-      .replace(/\s{2,}/, ' ') // replace multiple spaces
-      .trim();
-}
-
-export async function getImdbId(info, type) {
-    const name = escapeTitle(info.title);
-    const year = info.year || (info.date && info.date.slice(0, 4));
-    const key = `${name}_${year || 'NA'}_${type}`;
-    const query = `${name} ${year || ''} ${type} imdb`;
-    const fallbackQuery = `${name} ${type} imdb`;
-    const googleQuery = year ? query : fallbackQuery;
-
-    try {
-        const imdbId = await cacheWrapImdbId(key,
-            () => getIMDbIdFromNameToImdb(name, info.year, type)
-        );
-        return imdbId && 'tt' + imdbId.replace(/tt0*([1-9][0-9]*)$/, '$1').padStart(7, '0');
-    } catch (error) {
-        const imdbIdFallback = await getIMDbIdFromGoogle(googleQuery);
-        return imdbIdFallback && 'tt' + imdbIdFallback.replace(/tt0*([1-9][0-9]*)$/, '$1').padStart(7, '0');
-    }
-}
-
-function getIMDbIdFromNameToImdb(name, year, type) {
-    return new Promise((resolve, reject) => {
-        nameToImdb({ name, year, type }, function(err, res) {
-            if (res) {
-                resolve(res);
-            } else {
-                reject(err || new Error('Failed IMDbId search'));
-            }
-        });
-    });
-}
-
-async function getIMDbIdFromGoogle(query) {
-    try {
-        const searchResults = await search({ query: query });
-        for (const result of searchResults) {
-            if (result.link.includes('imdb.com/title/')) {
-                const match = result.link.match(/imdb\.com\/title\/(tt\d+)/);
-                if (match) {
-                    return match[1];
-                }
-            }
-        }
-        return undefined;
-    }
-    catch (error) {
-        throw new Error('Failed to find IMDb ID from Google search');
-    }
-}
-
-export async function getKitsuId(info) {
-  const title = escapeTitle(info.title.replace(/\s\|\s.*/, ''));
-  const year = info.year ? ` ${info.year}` : '';
-  const season = info.season > 1 ? ` S${info.season}` : '';
-  const key = `${title}${year}${season}`;
-  const query = encodeURIComponent(key);
-
-  return cacheWrapKitsuId(key,
-      () => axios.get(`${KITSU_URL}/catalog/series/kitsu-anime-list/search=${query}.json`, { timeout: 60000 })
-          .then((response) => {
-            const body = response.data;
-            if (body && body.metas && body.metas.length) {
-              return body.metas[0].id.replace('kitsu:', '');
-            } else {
-              throw new Error('No search results');
-            }
-          }));
-}
-
-export async function isEpisodeImdbId(imdbId) {
-  if (!imdbId) {
-    return false;
-  }
-  return axios.get(`https://www.imdb.com/title/${imdbId}/`, { timeout: 10000 })
-      .then(response => !!(response.data && response.data.includes('video.episode')))
-      .catch(() => false);
-}

src/node/consumer/src/lib/models/configuration/cache_config.ts

@@ -0,0 +1,15 @@
+import {BooleanHelpers} from "@helpers/boolean_helpers";
+
+export const cacheConfig = {
+    MONGODB_HOST: process.env.MONGODB_HOST || 'mongodb',
+    MONGODB_PORT: process.env.MONGODB_PORT || '27017',
+    MONGODB_DB: process.env.MONGODB_DB || 'knightcrawler',
+    MONGO_INITDB_ROOT_USERNAME: process.env.MONGO_INITDB_ROOT_USERNAME || 'mongo',
+    MONGO_INITDB_ROOT_PASSWORD: process.env.MONGO_INITDB_ROOT_PASSWORD || 'mongo',
+    NO_CACHE: BooleanHelpers.parseBool(process.env.NO_CACHE, false),
+    COLLECTION_NAME: process.env.MONGODB_COLLECTION || 'knightcrawler_consumer_collection',
+
+    get MONGO_URI(): string {
+        return `mongodb://${this.MONGO_INITDB_ROOT_USERNAME}:${this.MONGO_INITDB_ROOT_PASSWORD}@${this.MONGODB_HOST}:${this.MONGODB_PORT}/${this.MONGODB_DB}?authSource=admin`;
+    }
+};

src/node/consumer/src/lib/models/configuration/database_config.ts

@@ -0,0 +1,14 @@
+import {BooleanHelpers} from "@helpers/boolean_helpers";
+
+export const databaseConfig = {
+    POSTGRES_HOST: process.env.POSTGRES_HOST || 'postgres',
+    POSTGRES_PORT: parseInt(process.env.POSTGRES_PORT || '5432'),
+    POSTGRES_DB: process.env.POSTGRES_DB || 'knightcrawler',
+    POSTGRES_USER: process.env.POSTGRES_USER || 'postgres',
+    POSTGRES_PASSWORD: process.env.POSTGRES_PASSWORD || 'postgres',
+    AUTO_CREATE_AND_APPLY_MIGRATIONS: BooleanHelpers.parseBool(process.env.AUTO_CREATE_AND_APPLY_MIGRATIONS, false),
+
+    get POSTGRES_URI(): string {
+        return `postgres://${this.POSTGRES_USER}:${this.POSTGRES_PASSWORD}@${this.POSTGRES_HOST}:${this.POSTGRES_PORT}/${this.POSTGRES_DB}`;
+    }
+};

src/node/consumer/src/lib/models/configuration/job_config.ts

@@ -0,0 +1,6 @@
+import {BooleanHelpers} from "@helpers/boolean_helpers";
+
+export const jobConfig = {
+    JOB_CONCURRENCY: parseInt(process.env.JOB_CONCURRENCY || "1", 10),
+    JOBS_ENABLED: BooleanHelpers.parseBool(process.env.JOBS_ENABLED, true)
+};

src/node/consumer/src/lib/models/configuration/metadata_config.ts

@@ -0,0 +1,4 @@
+export const metadataConfig = {
+    IMDB_CONCURRENT: parseInt(process.env.IMDB_CONCURRENT || "1", 10),
+    IMDB_INTERVAL_MS: parseInt(process.env.IMDB_INTERVAL_MS || "1000", 10)
+};

src/node/consumer/src/lib/models/configuration/rabbit_config.ts

@@ -0,0 +1,4 @@
+export const rabbitConfig = {
+    RABBIT_URI: process.env.RABBIT_URI || 'amqp://localhost',
+    QUEUE_NAME: process.env.QUEUE_NAME || 'test-queue'
+};

src/node/consumer/src/lib/models/configuration/torrent_config.ts

@@ -0,0 +1,4 @@
+export const torrentConfig = {
+    MAX_CONNECTIONS_PER_TORRENT: parseInt(process.env.MAX_CONNECTIONS_PER_TORRENT || "20", 10),
+    TIMEOUT: parseInt(process.env.TORRENT_TIMEOUT || "30000", 10)
+};

src/node/consumer/src/lib/models/configuration/tracker_config.ts

@@ -0,0 +1,6 @@
+import {BooleanHelpers} from "@helpers/boolean_helpers";
+
+export const trackerConfig = {
+    TRACKERS_URL: process.env.TRACKERS_URL || 'https://ngosang.github.io/trackerslist/trackers_all.txt',
+    UDP_ENABLED: BooleanHelpers.parseBool(process.env.UDP_TRACKERS_ENABLED, false)
+};

src/node/consumer/src/lib/parseHelper.js

@@ -1,98 +0,0 @@
-import { parse } from 'parse-torrent-title';
-import { TorrentType } from './types.js';
-
-const MULTIPLE_FILES_SIZE = 4 * 1024 * 1024 * 1024; // 4 GB
-
-export function parseSeriesVideos(torrent, videos) {
-  const parsedTorrentName = parse(torrent.title);
-  const hasMovies = parsedTorrentName.complete || !!torrent.title.match(/movies?(?:\W|$)/i);
-  const parsedVideos = videos.map(video => parseSeriesVideo(video, parsedTorrentName));
-  return parsedVideos.map(video => ({ ...video, isMovie: isMovieVideo(video, parsedVideos, torrent.type, hasMovies) }));
-}
-
-function parseSeriesVideo(video, parsedTorrentName) {
-  const videoInfo = parse(video.name);
-  // the episode may be in a folder containing season number
-  if (!Number.isInteger(videoInfo.season) && video.path.includes('/')) {
-    const folders = video.path.split('/');
-    const pathInfo = parse(folders[folders.length - 2]);
-    videoInfo.season = pathInfo.season;
-  }
-  if (!Number.isInteger(videoInfo.season) && parsedTorrentName.season) {
-    videoInfo.season = parsedTorrentName.season;
-  }
-  if (!Number.isInteger(videoInfo.season) && videoInfo.seasons && videoInfo.seasons.length > 1) {
-    // in case single file was interpreted as having multiple seasons
-    videoInfo.season = videoInfo.seasons[0];
-  }
-  if (!Number.isInteger(videoInfo.season) && video.path.includes('/') && parsedTorrentName.seasons
-      && parsedTorrentName.seasons.length > 1) {
-    // russian season are usually named with 'series name-2` i.e. Улицы разбитых фонарей-6/22. Одиночный выстрел.mkv
-    const folderPathSeasonMatch = video.path.match(/[\u0400-\u04ff]-(\d{1,2})(?=.*\/)/);
-    videoInfo.season = folderPathSeasonMatch && parseInt(folderPathSeasonMatch[1], 10) || undefined;
-  }
-  // sometimes video file does not have correct date format as in torrent title
-  if (!videoInfo.episodes && !videoInfo.date && parsedTorrentName.date) {
-    videoInfo.date = parsedTorrentName.date;
-  }
-  // limit number of episodes in case of incorrect parsing
-  if (videoInfo.episodes && videoInfo.episodes.length > 20) {
-    videoInfo.episodes = [videoInfo.episodes[0]];
-    videoInfo.episode = videoInfo.episodes[0];
-  }
-  // force episode to any found number if it was not parsed
-  if (!videoInfo.episodes && !videoInfo.date) {
-    const epMatcher = videoInfo.title.match(
-        /(?<!season\W*|disk\W*|movie\W*|film\W*)(?:^|\W|_)(\d{1,4})(?:a|b|c|v\d)?(?:_|\W|$)(?!disk|movie|film)/i);
-    videoInfo.episodes = epMatcher && [parseInt(epMatcher[1], 10)];
-    videoInfo.episode = videoInfo.episodes && videoInfo.episodes[0];
-  }
-  if (!videoInfo.episodes && !videoInfo.date) {
-    const epMatcher = video.name.match(new RegExp(`(?:\\(${videoInfo.year}\\)|part)[._ ]?(\\d{1,3})(?:\\b|_)`, "i"));
-    videoInfo.episodes = epMatcher && [parseInt(epMatcher[1], 10)];
-    videoInfo.episode = videoInfo.episodes && videoInfo.episodes[0];
-  }
-
-  return { ...video, ...videoInfo };
-}
-
-function isMovieVideo(video, otherVideos, type, hasMovies) {
-  if (Number.isInteger(video.season) && Array.isArray(video.episodes)) {
-    // not movie if video has season
-    return false;
-  }
-  if (video.name.match(/\b(?:\d+[ .]movie|movie[ .]\d+)\b/i)) {
-    // movie if video explicitly has numbered movie keyword in the name, ie. 1 Movie or Movie 1
-    return true;
-  }
-  if (!hasMovies && type !== TorrentType.ANIME) {
-    // not movie if torrent name does not contain movies keyword or is not a pack torrent and is not anime
-    return false;
-  }
-  if (!video.episodes) {
-    // movie if there's no episode info it could be a movie
-    return true;
-  }
-  // movie if contains year info and there aren't more than 3 video with same title and year
-  // as some series titles might contain year in it.
-  return !!video.year
-      && otherVideos.length > 3
-      && otherVideos.filter(other => other.title === video.title && other.year === video.year) < 3;
-}
-
-export function isPackTorrent(torrent) {
-  if (torrent.pack) {
-    return true;
-  }
-  const parsedInfo = parse(torrent.title);
-  if (torrent.type === TorrentType.MOVIE) {
-    return parsedInfo.complete || typeof parsedInfo.year === 'string' || /movies/i.test(torrent.title);
-  }
-  const hasMultipleEpisodes = parsedInfo.complete ||
-      torrent.size > MULTIPLE_FILES_SIZE ||
-      (parsedInfo.seasons && parsedInfo.seasons.length > 1) ||
-      (parsedInfo.episodes && parsedInfo.episodes.length > 1) ||
-      (parsedInfo.seasons && !parsedInfo.episodes);
-  const hasSingleEpisode = Number.isInteger(parsedInfo.episode) || (!parsedInfo.episodes && parsedInfo.date);
-  return hasMultipleEpisodes && !hasSingleEpisode;
-}

src/node/consumer/src/lib/promises.js

@@ -1,55 +0,0 @@
-/**
- * Execute promises in sequence one after another.
- */
-export async function sequence(promises) {
-  return promises.reduce((promise, func) =>
-      promise.then(result => func().then(Array.prototype.concat.bind(result))), Promise.resolve([]));
-}
-
-/**
- * Return first resolved promise as the result.
- */
-export async function first(promises) {
-  return Promise.all(promises.map((p) => {
-    // If a request fails, count that as a resolution so it will keep
-    // waiting for other possible successes. If a request succeeds,
-    // treat it as a rejection so Promise.all immediately bails out.
-    return p.then(
-        (val) => Promise.reject(val),
-        (err) => Promise.resolve(err)
-    );
-  })).then(
-      // If '.all' resolved, we've just got an array of errors.
-      (errors) => Promise.reject(errors),
-      // If '.all' rejected, we've got the result we wanted.
-      (val) => Promise.resolve(val)
-  );
-}
-
-/**
- * Delay promise
- */
-export async function delay(duration) {
-  return new Promise((resolve) => setTimeout(resolve, duration));
-}
-
-/**
- * Timeout promise after a set time in ms
- */
-export async function timeout(timeoutMs, promise, message = 'Timed out') {
-  return Promise.race([
-    promise,
-    new Promise(function (resolve, reject) {
-      setTimeout(function () {
-        reject(message);
-      }, timeoutMs);
-    })
-  ]);
-}
-
-/**
- * Return most common value from given array.
- */
-export function mostCommonValue(array) {
-  return array.sort((a, b) => array.filter(v => v === a).length - array.filter(v => v === b).length).pop();
-}

src/node/consumer/src/lib/repository.js

@@ -1,382 +0,0 @@
-import moment from 'moment';
-import { Sequelize, Op, DataTypes, fn, col, literal } from 'sequelize';
-import { databaseConfig } from './config.js';
-import { logger } from "./logger.js";
-import * as Promises from './promises.js';
-
-const database = new Sequelize(
-    databaseConfig.POSTGRES_URI,
-    {
-        logging: false
-    }
-);
-
-const Provider = database.define('provider', {
-    name: { type: DataTypes.STRING(32), primaryKey: true },
-    lastScraped: { type: DataTypes.DATE },
-    lastScrapedId: { type: DataTypes.STRING(128) }
-});
-
-const IngestedTorrent = database.define('ingested_torrent', {
-    id: { type: DataTypes.BIGINT, autoIncrement: true, primaryKey: true },
-    name: DataTypes.STRING,
-    source: DataTypes.STRING,
-    category: DataTypes.STRING,
-    info_hash: DataTypes.STRING,
-    size: DataTypes.STRING,
-    seeders: DataTypes.INTEGER,
-    leechers: DataTypes.INTEGER,
-    imdb: DataTypes.STRING,
-    processed: {
-        type: DataTypes.BOOLEAN,
-        defaultValue: false
-    }
-},
-    {
-        indexes: [
-            {
-                unique: true,
-                fields: ['source', 'info_hash']
-            }
-        ]
-    })
-
-/* eslint-disable no-unused-vars */
-const IngestedPage = database.define('ingested_page', {
-    id: { type: DataTypes.BIGINT, autoIncrement: true, primaryKey: true },
-    url: { type: DataTypes.STRING, allowNull: false },
-},
-    {
-        indexes: [
-            {
-                unique: true,
-                fields: ['url']
-            }
-        ]
-    })
-/* eslint-enable no-unused-vars */
-
-const Torrent = database.define('torrent',
-    {
-        infoHash: { type: DataTypes.STRING(64), primaryKey: true },
-        provider: { type: DataTypes.STRING(32), allowNull: false },
-        torrentId: { type: DataTypes.STRING(512) },
-        title: { type: DataTypes.STRING(512), allowNull: false },
-        size: { type: DataTypes.BIGINT },
-        type: { type: DataTypes.STRING(16), allowNull: false },
-        uploadDate: { type: DataTypes.DATE, allowNull: false },
-        seeders: { type: DataTypes.SMALLINT },
-        trackers: { type: DataTypes.STRING(8000) },
-        languages: { type: DataTypes.STRING(4096) },
-        resolution: { type: DataTypes.STRING(16) },
-        reviewed: { type: DataTypes.BOOLEAN, allowNull: false, defaultValue: false },
-        opened: { type: DataTypes.BOOLEAN, allowNull: false, defaultValue: false }
-    }
-);
-
-const File = database.define('file',
-    {
-        id: { type: DataTypes.BIGINT, autoIncrement: true, primaryKey: true },
-        infoHash: {
-            type: DataTypes.STRING(64),
-            allowNull: false,
-            references: { model: Torrent, key: 'infoHash' },
-            onDelete: 'CASCADE'
-        },
-        fileIndex: { type: DataTypes.INTEGER },
-        title: { type: DataTypes.STRING(512), allowNull: false },
-        size: { type: DataTypes.BIGINT },
-        imdbId: { type: DataTypes.STRING(32) },
-        imdbSeason: { type: DataTypes.INTEGER },
-        imdbEpisode: { type: DataTypes.INTEGER },
-        kitsuId: { type: DataTypes.INTEGER },
-        kitsuEpisode: { type: DataTypes.INTEGER }
-    },
-    {
-        indexes: [
-            {
-                unique: true,
-                name: 'files_unique_file_constraint',
-                fields: [
-                    col('infoHash'),
-                    fn('COALESCE', (col('fileIndex')), -1),
-                    fn('COALESCE', (col('imdbId')), 'null'),
-                    fn('COALESCE', (col('imdbSeason')), -1),
-                    fn('COALESCE', (col('imdbEpisode')), -1),
-                    fn('COALESCE', (col('kitsuId')), -1),
-                    fn('COALESCE', (col('kitsuEpisode')), -1)
-                ]
-            },
-            { unique: false, fields: ['imdbId', 'imdbSeason', 'imdbEpisode'] },
-            { unique: false, fields: ['kitsuId', 'kitsuEpisode'] }
-        ]
-    }
-);
-
-const Subtitle = database.define('subtitle',
-    {
-        infoHash: {
-            type: DataTypes.STRING(64),
-            allowNull: false,
-            references: { model: Torrent, key: 'infoHash' },
-            onDelete: 'CASCADE'
-        },
-        fileIndex: {
-            type: DataTypes.INTEGER,
-            allowNull: false
-        },
-        fileId: {
-            type: DataTypes.BIGINT,
-            allowNull: true,
-            references: { model: File, key: 'id' },
-            onDelete: 'SET NULL'
-        },
-        title: { type: DataTypes.STRING(512), allowNull: false },
-    },
-    {
-        timestamps: false,
-        indexes: [
-            {
-                unique: true,
-                name: 'subtitles_unique_subtitle_constraint',
-                fields: [
-                    col('infoHash'),
-                    col('fileIndex'),
-                    fn('COALESCE', (col('fileId')), -1)
-                ]
-            },
-            { unique: false, fields: ['fileId'] }
-        ]
-    }
-);
-
-const Content = database.define('content',
-    {
-        infoHash: {
-            type: DataTypes.STRING(64),
-            primaryKey: true,
-            allowNull: false,
-            references: { model: Torrent, key: 'infoHash' },
-            onDelete: 'CASCADE'
-        },
-        fileIndex: {
-            type: DataTypes.INTEGER,
-            primaryKey: true,
-            allowNull: false
-        },
-        path: { type: DataTypes.STRING(512), allowNull: false },
-        size: { type: DataTypes.BIGINT },
-    },
-    {
-        timestamps: false,
-    }
-);
-
-const SkipTorrent = database.define('skip_torrent', {
-    infoHash: { type: DataTypes.STRING(64), primaryKey: true },
-});
-
-Torrent.hasMany(File, { foreignKey: 'infoHash', constraints: false });
-File.belongsTo(Torrent, { foreignKey: 'infoHash', constraints: false });
-Torrent.hasMany(Content, { foreignKey: 'infoHash', constraints: false });
-Content.belongsTo(Torrent, { foreignKey: 'infoHash', constraints: false });
-File.hasMany(Subtitle, { foreignKey: 'fileId', constraints: false });
-Subtitle.belongsTo(File, { foreignKey: 'fileId', constraints: false });
-
-export function connect() {
-    return database.sync({ alter: databaseConfig.AUTO_CREATE_AND_APPLY_MIGRATIONS })
-        .catch(error => {
-            console.error('Failed syncing database: ', error.message);
-            throw error;
-        });
-    // I'm not convinced this code is needed. If anyone can see where it leads, or what it does, please inform me.
-    // For now, I'm commenting it out. I don't think we ever reach this at the moment anyway as the previous ENABLE_SYNC
-    // was always on.
-    // return Promise.resolve();
-}
-
-export function getProvider(provider) {
-    return Provider.findOrCreate({ where: { name: { [Op.eq]: provider.name } }, defaults: provider })
-        .then((result) => result[0])
-        .catch(() => provider);
-}
-
-export function getTorrent(torrent) {
-    const where = torrent.infoHash
-        ? { infoHash: torrent.infoHash }
-        : { provider: torrent.provider, torrentId: torrent.torrentId }
-    return Torrent.findOne({ where: where });
-}
-
-export function getTorrentsBasedOnTitle(titleQuery, type) {
-    return getTorrentsBasedOnQuery({ title: { [Op.regexp]: `${titleQuery}` }, type: type });
-}
-
-export function getTorrentsBasedOnQuery(where) {
-    return Torrent.findAll({ where: where });
-}
-
-export function getFilesBasedOnQuery(where) {
-    return File.findAll({ where: where });
-}
-
-export function getUnprocessedIngestedTorrents() {
-    return IngestedTorrent.findAll({
-        where: {
-            processed: false,
-            category: {
-                [Op.or]: ['tv', 'movies']
-            }
-        },
-
-    });
-}
-
-export function setIngestedTorrentsProcessed(ingestedTorrents) {
-    return Promises.sequence(ingestedTorrents
-        .map(ingestedTorrent => () => {
-            ingestedTorrent.processed = true;
-            return ingestedTorrent.save();
-        }));
-}
-
-export function getTorrentsWithoutSize() {
-    return Torrent.findAll({
-        where: literal(
-            'exists (select 1 from files where files."infoHash" = torrent."infoHash" and files.size = 300000000)'),
-        order: [
-            ['seeders', 'DESC']
-        ]
-    });
-}
-
-export function getUpdateSeedersTorrents(limit = 50) {
-    const until = moment().subtract(7, 'days').format('YYYY-MM-DD');
-    return Torrent.findAll({
-        where: literal(`torrent."updatedAt" < '${until}'`),
-        limit: limit,
-        order: [
-            ['seeders', 'DESC'],
-            ['updatedAt', 'ASC']
-        ]
-    });
-}
-
-export function getUpdateSeedersNewTorrents(limit = 50) {
-    const lastUpdate = moment().subtract(12, 'hours').format('YYYY-MM-DD');
-    const createdAfter = moment().subtract(4, 'days').format('YYYY-MM-DD');
-    return Torrent.findAll({
-        where: literal(`torrent."updatedAt" < '${lastUpdate}' AND torrent."createdAt" > '${createdAfter}'`),
-        limit: limit,
-        order: [
-            ['seeders', 'ASC'],
-            ['updatedAt', 'ASC']
-        ]
-    });
-}
-
-export function getNoContentsTorrents() {
-    return Torrent.findAll({
-        where: { opened: false, seeders: { [Op.gte]: 1 } },
-        limit: 500,
-        order: [[fn('RANDOM')]]
-    });
-}
-
-export function createTorrent(torrent) {
-    return Torrent.upsert(torrent)
-        .then(() => createContents(torrent.infoHash, torrent.contents))
-        .then(() => createSubtitles(torrent.infoHash, torrent.subtitles));
-}
-
-export function setTorrentSeeders(torrent, seeders) {
-    const where = torrent.infoHash
-        ? { infoHash: torrent.infoHash }
-        : { provider: torrent.provider, torrentId: torrent.torrentId }
-    return Torrent.update(
-        { seeders: seeders },
-        { where: where }
-    );
-}
-
-export function deleteTorrent(torrent) {
-    return Torrent.destroy({ where: { infoHash: torrent.infoHash } })
-}
-
-export function createFile(file) {
-    if (file.id) {
-        return (file.dataValues ? file.save() : File.upsert(file))
-            .then(() => upsertSubtitles(file, file.subtitles));
-    }
-    if (file.subtitles && file.subtitles.length) {
-        file.subtitles = file.subtitles.map(subtitle => ({ infoHash: file.infoHash, title: subtitle.path, ...subtitle }));
-    }
-    return File.create(file, { include: [Subtitle], ignoreDuplicates: true });
-}
-
-export function getFiles(torrent) {
-    return File.findAll({ where: { infoHash: torrent.infoHash } });
-}
-
-export function getFilesBasedOnTitle(titleQuery) {
-    return File.findAll({ where: { title: { [Op.regexp]: `${titleQuery}` } } });
-}
-
-export function deleteFile(file) {
-    return File.destroy({ where: { id: file.id } })
-}
-
-export function createSubtitles(infoHash, subtitles) {
-    if (subtitles && subtitles.length) {
-        return Subtitle.bulkCreate(subtitles.map(subtitle => ({ infoHash, title: subtitle.path, ...subtitle })));
-    }
-    return Promise.resolve();
-}
-
-export function upsertSubtitles(file, subtitles) {
-    if (file.id && subtitles && subtitles.length) {
-        return Promises.sequence(subtitles
-            .map(subtitle => {
-                subtitle.fileId = file.id;
-                subtitle.infoHash = subtitle.infoHash || file.infoHash;
-                subtitle.title = subtitle.title || subtitle.path;
-                return subtitle;
-            })
-            .map(subtitle => () => subtitle.dataValues ? subtitle.save() : Subtitle.create(subtitle)));
-    }
-    return Promise.resolve();
-}
-
-export function getSubtitles(torrent) {
-    return Subtitle.findAll({ where: { infoHash: torrent.infoHash } });
-}
-
-export function getUnassignedSubtitles() {
-    return Subtitle.findAll({ where: { fileId: null } });
-}
-
-export function createContents(infoHash, contents) {
-    if (contents && contents.length) {
-        return Content.bulkCreate(contents.map(content => ({ infoHash, ...content })), { ignoreDuplicates: true })
-            .then(() => Torrent.update({ opened: true }, { where: { infoHash: infoHash }, silent: true }));
-    }
-    return Promise.resolve();
-}
-
-export function getContents(torrent) {
-    return Content.findAll({ where: { infoHash: torrent.infoHash } });
-}
-
-export function getSkipTorrent(torrent) {
-    return SkipTorrent.findByPk(torrent.infoHash)
-        .then((result) => {
-            if (!result) {
-                throw new Error(`torrent not found: ${torrent.infoHash}`);
-            }
-            return result.dataValues;
-        })
-}
-
-export function createSkipTorrent(torrent) {
-    return SkipTorrent.upsert({ infoHash: torrent.infoHash });
-}

src/node/consumer/src/lib/repository/database_repository.ts

@@ -0,0 +1,260 @@
+import {PromiseHelpers} from '@helpers/promises_helpers';
+import {ILoggingService} from "@interfaces/logging_service";
+import {IContentCreationAttributes} from "@repository/interfaces/content_attributes";
+import {IDatabaseRepository} from "@repository/interfaces/database_repository";
+import {IFileAttributes, IFileCreationAttributes} from "@repository/interfaces/file_attributes";
+import {ISubtitleAttributes, ISubtitleCreationAttributes} from "@repository/interfaces/subtitle_attributes";
+import {ITorrentAttributes, ITorrentCreationAttributes} from "@repository/interfaces/torrent_attributes";
+import {Content} from "@repository/models/content";
+import {File} from "@repository/models/file";
+import {IngestedPage} from "@repository/models/ingestedPage";
+import {IngestedTorrent} from "@repository/models/ingestedTorrent";
+import {Provider} from "@repository/models/provider";
+import {SkipTorrent} from "@repository/models/skipTorrent";
+import {Subtitle} from "@repository/models/subtitle";
+import {Torrent} from "@repository/models/torrent";
+import {configurationService} from '@services/configuration_service';
+import {IocTypes} from "@setup/ioc_types";
+import {inject, injectable} from "inversify";
+import moment from 'moment';
+import {literal, Op, WhereOptions} from "sequelize";
+import {Model, Sequelize} from 'sequelize-typescript';
+
+@injectable()
+export class DatabaseRepository implements IDatabaseRepository {
+    @inject(IocTypes.ILoggingService) logger: ILoggingService;
+
+    private readonly database: Sequelize;
+
+    private models = [
+        Torrent,
+        Provider,
+        File,
+        Subtitle,
+        Content,
+        SkipTorrent,
+        IngestedTorrent,
+        IngestedPage];
+
+    constructor() {
+        this.database = this.createDatabase();
+    }
+
+    async connect(): Promise<void> {
+        try {
+            await this.database.sync({alter: configurationService.databaseConfig.AUTO_CREATE_AND_APPLY_MIGRATIONS});
+        } catch (error) {
+            this.logger.debug('Failed to sync database', error);
+            this.logger.error('Failed syncing database');
+            process.exit(1);
+        }
+    }
+
+    async getProvider(provider: Provider): Promise<Provider> {
+        try {
+            const [result] = await Provider.findOrCreate({where: {name: {[Op.eq]: provider.name}}, defaults: provider});
+            return result;
+        } catch {
+            return provider as Provider;
+        }
+    }
+
+    async getTorrent(torrent: ITorrentAttributes): Promise<Torrent | null> {
+        const where = torrent.infoHash
+            ? {infoHash: torrent.infoHash}
+            : {provider: torrent.provider, torrentId: torrent.torrentId};
+        return await Torrent.findOne({where});
+    }
+
+    async getTorrentsBasedOnTitle(titleQuery: string, type: string): Promise<Torrent[]> {
+        return this.getTorrentsBasedOnQuery({
+            title: {[Op.regexp]: `${titleQuery}`},
+            type
+        });
+    }
+
+    async getTorrentsBasedOnQuery(where: WhereOptions<ITorrentAttributes>): Promise<Torrent[]> {
+        return await Torrent.findAll({where});
+    }
+
+    async getFilesBasedOnQuery(where: WhereOptions<IFileAttributes>): Promise<File[]> {
+        return await File.findAll({where});
+    }
+
+    async getTorrentsWithoutSize(): Promise<Torrent[]> {
+        return await Torrent.findAll({
+            where: literal(
+                'exists (select 1 from files where files."infoHash" = torrent."infoHash" and files.size = 300000000)'),
+            order: [
+                ['seeders', 'DESC']
+            ]
+        });
+    }
+
+    async getUpdateSeedersTorrents(limit = 50): Promise<Torrent[]> {
+        const until = moment().subtract(7, 'days').format('YYYY-MM-DD');
+        return await Torrent.findAll({
+            where: literal(`torrent."updatedAt" < '${until}'`),
+            limit: limit,
+            order: [
+                ['seeders', 'DESC'],
+                ['updatedAt', 'ASC']
+            ]
+        });
+    }
+
+    async getUpdateSeedersNewTorrents(limit = 50): Promise<Torrent[]> {
+        const lastUpdate = moment().subtract(12, 'hours').format('YYYY-MM-DD');
+        const createdAfter = moment().subtract(4, 'days').format('YYYY-MM-DD');
+        return await Torrent.findAll({
+            where: literal(`torrent."updatedAt" < '${lastUpdate}' AND torrent."createdAt" > '${createdAfter}'`),
+            limit: limit,
+            order: [
+                ['seeders', 'ASC'],
+                ['updatedAt', 'ASC']
+            ]
+        });
+    }
+
+    async getNoContentsTorrents(): Promise<Torrent[]> {
+        return await Torrent.findAll({
+            where: {opened: false, seeders: {[Op.gte]: 1}},
+            limit: 500,
+            order: literal('random()')
+        });
+    }
+
+    async createTorrent(torrent: ITorrentCreationAttributes): Promise<void> {
+        try {
+            await Torrent.upsert(torrent);
+            await this.createContents(torrent.infoHash, torrent.contents);
+            await this.createSubtitles(torrent.infoHash, torrent.subtitles);
+        } catch (error) {
+            this.logger.error(`Failed to create torrent: ${torrent.infoHash}`);
+            this.logger.debug("Error: ", error);
+        }
+    }
+
+    async setTorrentSeeders(torrent: ITorrentAttributes, seeders: number): Promise<[number]> {
+        const where = torrent.infoHash
+            ? {infoHash: torrent.infoHash}
+            : {provider: torrent.provider, torrentId: torrent.torrentId};
+
+        return await Torrent.update(
+            {seeders: seeders},
+            {where: where}
+        );
+    }
+
+    async deleteTorrent(infoHash: string): Promise<number> {
+        return await Torrent.destroy({where: {infoHash: infoHash}});
+    }
+
+    async createFile(file: IFileCreationAttributes): Promise<void> {
+        try {
+            const operatingFile = File.build(file);
+            if (operatingFile.id) {
+                if (operatingFile.dataValues) {
+                    await operatingFile.save();
+                } else {
+                    await File.upsert(file);
+                }
+                await this.upsertSubtitles(operatingFile, file.subtitles);
+            } else {
+                if (operatingFile.subtitles && operatingFile.subtitles.length) {
+                    operatingFile.subtitles = operatingFile.subtitles.map(subtitle => {
+                        subtitle.title = subtitle.path || '';
+                        return subtitle;
+                    });
+                }
+                await File.create(file, {include: [Subtitle], ignoreDuplicates: true});
+            }
+        } catch (error) {
+            this.logger.error(`Failed to create file: ${file.infoHash}`);
+            this.logger.debug("Error: ", error);
+        }
+    }
+
+    async getFiles(infoHash: string): Promise<File[]> {
+        return File.findAll({where: {infoHash: infoHash}});
+    }
+
+    async getFilesBasedOnTitle(titleQuery: string): Promise<File[]> {
+        return File.findAll({where: {title: {[Op.regexp]: `${titleQuery}`}}});
+    }
+
+    async deleteFile(id: number): Promise<number> {
+        return File.destroy({where: {id: id}});
+    }
+
+    async createSubtitles(infoHash: string, subtitles: ISubtitleCreationAttributes[] | undefined): Promise<void | Model<ISubtitleAttributes, ISubtitleCreationAttributes>[]> {
+        if (subtitles && subtitles.length) {
+            return Subtitle.bulkCreate(subtitles.map(subtitle => ({...subtitle, infoHash: infoHash, title: subtitle.path})));
+        }
+        return Promise.resolve();
+    }
+
+    async upsertSubtitles(file: File, subtitles: ISubtitleCreationAttributes[] | undefined): Promise<void> {
+        if (file.id && subtitles && subtitles.length) {
+            await PromiseHelpers.sequence(subtitles
+                .map(subtitle => {
+                    subtitle.fileId = file.id;
+                    subtitle.infoHash = subtitle.infoHash || file.infoHash;
+                    subtitle.title = subtitle.title || subtitle.path || '';
+                    return subtitle;
+                })
+                .map(subtitle => async () => {
+                    const operatingInstance = Subtitle.build(subtitle);
+                    if (operatingInstance.dataValues) {
+                        await operatingInstance.save();
+                    } else {
+                        await Subtitle.create(subtitle);
+                    }
+                }));
+        }
+    }
+
+    async getSubtitles(infoHash: string): Promise<Subtitle[]> {
+        return Subtitle.findAll({where: {infoHash: infoHash}});
+    }
+
+    async getUnassignedSubtitles(): Promise<Subtitle[]> {
+        return Subtitle.findAll({where: {fileId: null}});
+    }
+
+    async createContents(infoHash: string, contents: IContentCreationAttributes[] | undefined): Promise<void> {
+        if (contents && contents.length) {
+            await Content.bulkCreate(contents.map(content => ({...content, infoHash})), {ignoreDuplicates: true});
+            await Torrent.update({opened: true}, {where: {infoHash: infoHash}, silent: true});
+        }
+    }
+
+    async getContents(infoHash: string): Promise<Content[]> {
+        return Content.findAll({where: {infoHash: infoHash}});
+    }
+
+    async getSkipTorrent(infoHash: string): Promise<SkipTorrent> {
+        const result = await SkipTorrent.findByPk(infoHash);
+        if (!result) {
+            throw new Error(`torrent not found: ${infoHash}`);
+        }
+        return result.dataValues as SkipTorrent;
+    }
+
+    async createSkipTorrent(torrent: ITorrentCreationAttributes): Promise<[SkipTorrent, boolean | null]> {
+        return SkipTorrent.upsert({infoHash: torrent.infoHash});
+    }
+
+    private createDatabase = (): Sequelize => {
+        const newDatabase = new Sequelize(
+            configurationService.databaseConfig.POSTGRES_URI,
+            {
+                logging: false
+            }
+        );
+
+        newDatabase.addModels(this.models);
+
+        return newDatabase;
+    };
+}

src/node/consumer/src/lib/repository/interfaces/content_attributes.ts

@@ -0,0 +1,11 @@
+import {Optional} from "sequelize";
+
+export interface IContentAttributes {
+    infoHash: string;
+    fileIndex: number;
+    path: string;
+    size: number;
+}
+
+export interface IContentCreationAttributes extends Optional<IContentAttributes, 'fileIndex' | 'size'> {
+}

src/node/consumer/src/lib/repository/interfaces/database_repository.ts

@@ -0,0 +1,64 @@
+import {IContentCreationAttributes} from "@repository/interfaces/content_attributes";
+import {IFileAttributes, IFileCreationAttributes} from "@repository/interfaces/file_attributes";
+import {ISubtitleAttributes, ISubtitleCreationAttributes} from "@repository/interfaces/subtitle_attributes";
+import {ITorrentAttributes, ITorrentCreationAttributes} from "@repository/interfaces/torrent_attributes";
+import {Content} from "@repository/models/content";
+import {File} from "@repository/models/file";
+import {Provider} from "@repository/models/provider";
+import {SkipTorrent} from "@repository/models/skipTorrent";
+import {Subtitle} from "@repository/models/subtitle";
+import {Torrent} from "@repository/models/torrent";
+import {WhereOptions} from "sequelize";
+import {Model} from "sequelize-typescript";
+
+export interface IDatabaseRepository {
+    connect(): Promise<void>;
+
+    getProvider(provider: Provider): Promise<Provider>;
+
+    getTorrent(torrent: ITorrentAttributes): Promise<Torrent | null>;
+
+    getTorrentsBasedOnTitle(titleQuery: string, type: string): Promise<Torrent[]>;
+
+    getTorrentsBasedOnQuery(where: WhereOptions<ITorrentAttributes>): Promise<Torrent[]>;
+
+    getFilesBasedOnQuery(where: WhereOptions<IFileAttributes>): Promise<File[]>;
+
+    getTorrentsWithoutSize(): Promise<Torrent[]>;
+
+    getUpdateSeedersTorrents(limit: number): Promise<Torrent[]>;
+
+    getUpdateSeedersNewTorrents(limit: number): Promise<Torrent[]>;
+
+    getNoContentsTorrents(): Promise<Torrent[]>;
+
+    createTorrent(torrent: ITorrentCreationAttributes): Promise<void>;
+
+    setTorrentSeeders(torrent: ITorrentAttributes, seeders: number): Promise<[number]>;
+
+    deleteTorrent(infoHash: string): Promise<number>;
+
+    createFile(file: IFileCreationAttributes): Promise<void>;
+
+    getFiles(infoHash: string): Promise<File[]>;
+
+    getFilesBasedOnTitle(titleQuery: string): Promise<File[]>;
+
+    deleteFile(id: number): Promise<number>;
+
+    createSubtitles(infoHash: string, subtitles: ISubtitleCreationAttributes[]): Promise<void | Model<ISubtitleAttributes, ISubtitleCreationAttributes>[]>;
+
+    upsertSubtitles(file: File, subtitles: ISubtitleCreationAttributes[] | undefined): Promise<void>;
+
+    getSubtitles(infoHash: string): Promise<Subtitle[]>;
+
+    getUnassignedSubtitles(): Promise<Subtitle[]>;
+
+    createContents(infoHash: string, contents: IContentCreationAttributes[]): Promise<void>;
+
+    getContents(infoHash: string): Promise<Content[]>;
+
+    getSkipTorrent(infoHash: string): Promise<SkipTorrent>;
+
+    createSkipTorrent(torrent: ITorrentCreationAttributes): Promise<[SkipTorrent, boolean | null]>;
+}

src/node/consumer/src/lib/repository/interfaces/file_attributes.ts

@@ -0,0 +1,22 @@
+import {IParseTorrentTitleResult} from "@interfaces/parse_torrent_title_result";
+import {ISubtitleAttributes} from "@repository/interfaces/subtitle_attributes";
+import {Optional} from "sequelize";
+
+export interface IFileAttributes extends IParseTorrentTitleResult {
+    id?: number;
+    infoHash?: string;
+    fileIndex?: number;
+    title: string;
+    size?: number;
+    imdbId?: string;
+    imdbSeason?: number;
+    imdbEpisode?: number;
+    kitsuId?: number;
+    kitsuEpisode?: number;
+    subtitles?: ISubtitleAttributes[];
+    path?: string;
+    isMovie?: boolean;
+}
+
+export interface IFileCreationAttributes extends Optional<IFileAttributes, 'fileIndex' | 'size' | 'imdbId' | 'imdbSeason' | 'imdbEpisode' | 'kitsuId' | 'kitsuEpisode'> {
+}

src/node/consumer/src/lib/repository/interfaces/ingested_page_attributes.ts

@@ -0,0 +1,6 @@
+export interface IIngestedPageAttributes {
+    url: string;
+}
+
+export interface IIngestedPageCreationAttributes extends IIngestedPageAttributes {
+}

src/node/consumer/src/lib/repository/interfaces/ingested_torrent_attributes.ts

@@ -0,0 +1,17 @@
+import {Optional} from "sequelize";
+
+export interface IIngestedTorrentAttributes {
+    name: string;
+    source: string;
+    category: string;
+    info_hash: string;
+    size: string;
+    seeders: number;
+    leechers: number;
+    imdb: string;
+    processed: boolean;
+    createdAt?: Date;
+}
+
+export interface IIngestedTorrentCreationAttributes extends Optional<IIngestedTorrentAttributes, 'processed'> {
+}

src/node/consumer/src/lib/repository/interfaces/provider_attributes.ts

@@ -0,0 +1,10 @@
+import {Optional} from "sequelize";
+
+export interface IProviderAttributes {
+    name: string;
+    lastScraped: Date;
+    lastScrapedId: string;
+}
+
+export interface IProviderCreationAttributes extends Optional<IProviderAttributes, 'lastScraped' | 'lastScrapedId'> {
+}

src/node/consumer/src/lib/repository/interfaces/skip_torrent_attributes.ts

@@ -0,0 +1,8 @@
+import {Optional} from "sequelize";
+
+export interface ISkipTorrentAttributes {
+    infoHash: string;
+}
+
+export interface ISkipTorrentCreationAttributes extends Optional<ISkipTorrentAttributes, never> {
+}

src/node/consumer/src/lib/repository/interfaces/subtitle_attributes.ts

@@ -0,0 +1,12 @@
+import {Optional} from "sequelize";
+
+export interface ISubtitleAttributes {
+    infoHash: string;
+    fileIndex: number;
+    fileId?: number | null;
+    title: string;
+    path: string;
+}
+
+export interface ISubtitleCreationAttributes extends Optional<ISubtitleAttributes, 'fileId'> {
+}

src/node/consumer/src/lib/repository/interfaces/torrent_attributes.ts

@@ -0,0 +1,26 @@
+import {IContentAttributes} from "@repository/interfaces/content_attributes";
+import {IFileAttributes} from "@repository/interfaces/file_attributes";
+import {ISubtitleAttributes} from "@repository/interfaces/subtitle_attributes";
+import {Optional} from "sequelize";
+
+export interface ITorrentAttributes {
+    infoHash: string;
+    provider?: string | null;
+    torrentId?: string;
+    title?: string;
+    size?: number;
+    type?: string;
+    uploadDate?: Date;
+    seeders?: number;
+    trackers?: string;
+    languages?: string;
+    resolution?: string;
+    reviewed?: boolean;
+    opened?: boolean;
+    contents?: IContentAttributes[];
+    files?: IFileAttributes[];
+    subtitles?: ISubtitleAttributes[];
+}
+
+export interface ITorrentCreationAttributes extends Optional<ITorrentAttributes, 'torrentId' | 'size' | 'seeders' | 'trackers' | 'languages' | 'resolution' | 'reviewed' | 'opened'> {
+}

src/node/consumer/src/lib/repository/models/content.ts

@@ -0,0 +1,22 @@
+import {IContentAttributes, IContentCreationAttributes} from "@repository/interfaces/content_attributes";
+import {Torrent} from "@repository/models/torrent";
+import {BelongsTo, Column, DataType, ForeignKey, Model, Table} from 'sequelize-typescript';
+
+@Table({modelName: 'content', timestamps: false})
+export class Content extends Model<IContentAttributes, IContentCreationAttributes> {
+    @Column({type: DataType.STRING(64), primaryKey: true, allowNull: false, onDelete: 'CASCADE'})
+    @ForeignKey(() => Torrent)
+    declare infoHash: string;
+
+    @Column({type: DataType.INTEGER, primaryKey: true, allowNull: false})
+    declare fileIndex: number;
+
+    @Column({type: DataType.STRING(512), allowNull: false})
+    declare path: string;
+
+    @Column({type: DataType.BIGINT})
+    declare size: number;
+
+    @BelongsTo(() => Torrent, {constraints: false, foreignKey: 'infoHash'})
+    torrent?: Torrent;
+}

src/node/consumer/src/lib/repository/models/file.ts

@@ -0,0 +1,59 @@
+import {IFileAttributes, IFileCreationAttributes} from "@repository/interfaces/file_attributes";
+import {Subtitle} from "@repository/models/subtitle";
+import {Torrent} from "@repository/models/torrent";
+import {BelongsTo, Column, DataType, ForeignKey, HasMany, Model, Table} from 'sequelize-typescript';
+
+const indexes = [
+    {
+        unique: true,
+        name: 'files_unique_file_constraint',
+        fields: [
+            'infoHash',
+            'fileIndex',
+            'imdbId',
+            'imdbSeason',
+            'imdbEpisode',
+            'kitsuId',
+            'kitsuEpisode'
+        ]
+    },
+    {unique: false, fields: ['imdbId', 'imdbSeason', 'imdbEpisode']},
+    {unique: false, fields: ['kitsuId', 'kitsuEpisode']}
+];
+
+@Table({modelName: 'file', timestamps: true, indexes: indexes})
+export class File extends Model<IFileAttributes, IFileCreationAttributes> {
+    @Column({type: DataType.STRING(64), allowNull: false, onDelete: 'CASCADE'})
+    @ForeignKey(() => Torrent)
+    declare infoHash: string;
+
+    @Column({type: DataType.INTEGER})
+    declare fileIndex: number;
+
+    @Column({type: DataType.STRING(512), allowNull: false})
+    declare title: string;
+
+    @Column({type: DataType.BIGINT})
+    declare size: number;
+
+    @Column({type: DataType.STRING(32)})
+    declare imdbId: string;
+
+    @Column({type: DataType.INTEGER})
+    declare imdbSeason: number;
+
+    @Column({type: DataType.INTEGER})
+    declare imdbEpisode: number;
+
+    @Column({type: DataType.INTEGER})
+    declare kitsuId: number;
+
+    @Column({type: DataType.INTEGER})
+    declare kitsuEpisode: number;
+
+    @HasMany(() => Subtitle, {constraints: false, foreignKey: 'fileId'})
+    declare subtitles?: Subtitle[];
+
+    @BelongsTo(() => Torrent, {constraints: false, foreignKey: 'infoHash'})
+    torrent?: Torrent;
+}

src/node/consumer/src/lib/repository/models/ingestedPage.ts

@@ -0,0 +1,16 @@
+import {IIngestedPageAttributes, IIngestedPageCreationAttributes} from "@repository/interfaces/ingested_page_attributes";
+import {Column, DataType, Model, Table} from 'sequelize-typescript';
+
+const indexes = [
+    {
+        unique: true,
+        name: 'ingested_page_unique_url_constraint',
+        fields: ['url']
+    }
+];
+
+@Table({modelName: 'ingested_page', timestamps: true, indexes: indexes})
+export class IngestedPage extends Model<IIngestedPageAttributes, IIngestedPageCreationAttributes> {
+    @Column({type: DataType.STRING(512), allowNull: false})
+    declare url: string;
+}

src/node/consumer/src/lib/repository/models/ingestedTorrent.ts

@@ -0,0 +1,40 @@
+import {IIngestedTorrentAttributes, IIngestedTorrentCreationAttributes} from "@repository/interfaces/ingested_torrent_attributes";
+import {Column, DataType, Model, Table} from 'sequelize-typescript';
+
+const indexes = [
+    {
+        unique: true,
+        name: 'ingested_torrent_unique_source_info_hash_constraint',
+        fields: ['source', 'info_hash']
+    }
+];
+
+@Table({modelName: 'ingested_torrent', timestamps: true, indexes: indexes})
+export class IngestedTorrent extends Model<IIngestedTorrentAttributes, IIngestedTorrentCreationAttributes> {
+    @Column({type: DataType.STRING(512)})
+    declare name: string;
+
+    @Column({type: DataType.STRING(512)})
+    declare source: string;
+
+    @Column({type: DataType.STRING(32)})
+    declare category: string;
+
+    @Column({type: DataType.STRING(64)})
+    declare info_hash: string;
+
+    @Column({type: DataType.STRING(32)})
+    declare size: string;
+
+    @Column({type: DataType.INTEGER})
+    declare seeders: number;
+
+    @Column({type: DataType.INTEGER})
+    declare leechers: number;
+
+    @Column({type: DataType.STRING(32)})
+    declare imdb: string;
+
+    @Column({type: DataType.BOOLEAN, defaultValue: false})
+    declare processed: boolean;
+}

src/node/consumer/src/lib/repository/models/provider.ts

@@ -0,0 +1,15 @@
+import {IProviderAttributes, IProviderCreationAttributes} from "@repository/interfaces/provider_attributes";
+import {Column, DataType, Model, Table} from 'sequelize-typescript';
+
+@Table({modelName: 'provider', timestamps: false})
+export class Provider extends Model<IProviderAttributes, IProviderCreationAttributes> {
+
+    @Column({type: DataType.STRING(32), primaryKey: true})
+    declare name: string;
+
+    @Column({type: DataType.DATE})
+    declare lastScraped: Date;
+
+    @Column({type: DataType.STRING(128)})
+    declare lastScrapedId: string;
+}

src/node/consumer/src/lib/repository/models/skipTorrent.ts

@@ -0,0 +1,9 @@
+import {ISkipTorrentAttributes, ISkipTorrentCreationAttributes} from "@repository/interfaces/skip_torrent_attributes";
+import {Column, DataType, Model, Table} from 'sequelize-typescript';
+
+@Table({modelName: 'skip_torrent', timestamps: false})
+export class SkipTorrent extends Model<ISkipTorrentAttributes, ISkipTorrentCreationAttributes> {
+
+    @Column({type: DataType.STRING(64), primaryKey: true})
+    declare infoHash: string;
+}

src/node/consumer/src/lib/repository/models/subtitle.ts

@@ -0,0 +1,38 @@
+import {ISubtitleAttributes, ISubtitleCreationAttributes} from "@repository/interfaces/subtitle_attributes";
+import {File} from "@repository/models/file";
+import {BelongsTo, Column, DataType, ForeignKey, Model, Table} from 'sequelize-typescript';
+
+const indexes = [
+    {
+        unique: true,
+        name: 'subtitles_unique_subtitle_constraint',
+        fields: [
+            'infoHash',
+            'fileIndex',
+            'fileId'
+        ]
+    },
+    {unique: false, fields: ['fileId']}
+];
+
+@Table({modelName: 'subtitle', timestamps: false, indexes: indexes})
+export class Subtitle extends Model<ISubtitleAttributes, ISubtitleCreationAttributes> {
+
+    @Column({type: DataType.STRING(64), allowNull: false, onDelete: 'CASCADE'})
+    declare infoHash: string;
+
+    @Column({type: DataType.INTEGER, allowNull: false})
+    declare fileIndex: number;
+
+    @Column({type: DataType.BIGINT, allowNull: true, onDelete: 'SET NULL'})
+    @ForeignKey(() => File)
+    declare fileId?: number | null;
+
+    @Column({type: DataType.STRING(512), allowNull: false})
+    declare title: string;
+
+    @BelongsTo(() => File, {constraints: false, foreignKey: 'fileId'})
+    file?: File;
+
+    path?: string;
+}

src/node/consumer/src/lib/repository/models/torrent.ts

@@ -0,0 +1,56 @@
+import {ITorrentAttributes, ITorrentCreationAttributes} from "@repository/interfaces/torrent_attributes";
+import {Content} from "@repository/models/content";
+import {File} from "@repository/models/file";
+import {Subtitle} from "@repository/models/subtitle";
+import {Column, DataType, HasMany, Model, Table} from 'sequelize-typescript';
+
+@Table({modelName: 'torrent', timestamps: true})
+
+export class Torrent extends Model<ITorrentAttributes, ITorrentCreationAttributes> {
+    @Column({type: DataType.STRING(64), primaryKey: true})
+    declare infoHash: string;
+
+    @Column({type: DataType.STRING(32), allowNull: false})
+    declare provider: string;
+
+    @Column({type: DataType.STRING(512)})
+    declare torrentId: string;
+
+    @Column({type: DataType.STRING(512), allowNull: false})
+    declare title: string;
+
+    @Column({type: DataType.BIGINT})
+    declare size: number;
+
+    @Column({type: DataType.STRING(16), allowNull: false})
+    declare type: string;
+
+    @Column({type: DataType.DATE, allowNull: false})
+    declare uploadDate: Date;
+
+    @Column({type: DataType.SMALLINT})
+    declare seeders: number;
+
+    @Column({type: DataType.STRING(8000)})
+    declare trackers: string;
+
+    @Column({type: DataType.STRING(4096)})
+    declare languages: string;
+
+    @Column({type: DataType.STRING(16)})
+    declare resolution: string;
+
+    @Column({type: DataType.BOOLEAN, allowNull: false, defaultValue: false})
+    declare reviewed: boolean;
+
+    @Column({type: DataType.BOOLEAN, allowNull: false, defaultValue: false})
+    declare opened: boolean;
+
+    @HasMany(() => Content, {foreignKey: 'infoHash', constraints: false})
+    contents?: Content[];
+
+    @HasMany(() => File, {foreignKey: 'infoHash', constraints: false})
+    files?: File[];
+
+    subtitles?: Subtitle[];
+}

src/node/consumer/src/lib/services/cache_service.ts

@@ -0,0 +1,111 @@
+import {CacheType} from "@enums/cache_types";
+import {ICacheOptions} from "@interfaces/cache_options";
+import {ICacheService} from "@interfaces/cache_service";
+import {ILoggingService} from "@interfaces/logging_service";
+import {configurationService} from '@services/configuration_service';
+import {IocTypes} from "@setup/ioc_types";
+import {mongoDbStore} from '@tirke/node-cache-manager-mongodb'
+import {Cache, createCache, MemoryCache, memoryStore} from 'cache-manager';
+import {inject, injectable} from "inversify";
+
+const GLOBAL_KEY_PREFIX = 'knightcrawler-consumer';
+const IMDB_ID_PREFIX = `${GLOBAL_KEY_PREFIX}|imdb_id`;
+const KITSU_ID_PREFIX = `${GLOBAL_KEY_PREFIX}|kitsu_id`;
+const METADATA_PREFIX = `${GLOBAL_KEY_PREFIX}|metadata`;
+const TRACKERS_KEY_PREFIX = `${GLOBAL_KEY_PREFIX}|trackers`;
+
+const GLOBAL_TTL: number = Number(process.env.METADATA_TTL) || 7 * 24 * 60 * 60; // 7 days
+const MEMORY_TTL: number = Number(process.env.METADATA_TTL) || 2 * 60 * 60; // 2 hours
+const TRACKERS_TTL: number = 2 * 24 * 60 * 60; // 2 days
+
+/* eslint-disable-next-line @typescript-eslint/no-explicit-any */
+export type CacheMethod = () => any;
+
+@injectable()
+export class CacheService implements ICacheService {
+    @inject(IocTypes.ILoggingService) private logger: ILoggingService;
+
+    private readonly memoryCache: MemoryCache | undefined;
+    private readonly remoteCache: Cache | MemoryCache | undefined;
+
+    constructor() {
+        if (configurationService.cacheConfig.NO_CACHE) {
+            this.logger.info('Cache is disabled');
+            return;
+        }
+
+        this.memoryCache = this.initiateMemoryCache();
+        this.remoteCache = this.initiateRemoteCache();
+    }
+
+    cacheWrapImdbId(key: string, method: CacheMethod): Promise<CacheMethod> {
+        return this.cacheWrap(CacheType.MongoDb, `${IMDB_ID_PREFIX}:${key}`, method, {ttl: GLOBAL_TTL});
+    }
+
+    cacheWrapKitsuId(key: string, method: CacheMethod): Promise<CacheMethod> {
+        return this.cacheWrap(CacheType.MongoDb, `${KITSU_ID_PREFIX}:${key}`, method, {ttl: GLOBAL_TTL});
+    }
+
+    cacheWrapMetadata(id: string, method: CacheMethod): Promise<CacheMethod> {
+        return this.cacheWrap(CacheType.Memory, `${METADATA_PREFIX}:${id}`, method, {ttl: MEMORY_TTL});
+    }
+
+    cacheTrackers(method: CacheMethod): Promise<CacheMethod> {
+        return this.cacheWrap(CacheType.Memory, `${TRACKERS_KEY_PREFIX}`, method, {ttl: TRACKERS_TTL});
+    }
+
+    private initiateMemoryCache = (): MemoryCache =>
+        createCache(memoryStore(), {
+            ttl: MEMORY_TTL
+        });
+
+    private initiateMongoCache = (): Cache => {
+        const store = mongoDbStore({
+            collectionName: configurationService.cacheConfig.COLLECTION_NAME,
+            ttl: GLOBAL_TTL,
+            url: configurationService.cacheConfig.MONGO_URI,
+            mongoConfig: {
+                socketTimeoutMS: 120000,
+                appName: 'knightcrawler-consumer',
+            }
+        });
+
+        return createCache(store, {
+            ttl: GLOBAL_TTL,
+        });
+    }
+
+    private initiateRemoteCache = (): Cache | undefined => {
+        if (configurationService.cacheConfig.NO_CACHE) {
+            this.logger.debug('Cache is disabled');
+            return undefined;
+        }
+
+        return configurationService.cacheConfig.MONGO_URI ? this.initiateMongoCache() : this.initiateMemoryCache();
+    }
+
+    private getCacheType = (cacheType: CacheType): MemoryCache | Cache | undefined => {
+        switch (cacheType) {
+            case CacheType.Memory:
+                return this.memoryCache;
+            case CacheType.MongoDb:
+                return this.remoteCache;
+            default:
+                return undefined;
+        }
+    }
+
+    private cacheWrap = async (cacheType: CacheType, key: string, method: CacheMethod, options: ICacheOptions): Promise<CacheMethod> => {
+        const cache = this.getCacheType(cacheType);
+
+        if (configurationService.cacheConfig.NO_CACHE || !cache) {
+            return method();
+        }
+
+        this.logger.debug(`Cache type: ${cacheType}`);
+        this.logger.debug(`Cache key: ${key}`);
+        this.logger.debug(`Cache options: ${JSON.stringify(options)}`);
+
+        return cache.wrap(key, method, options.ttl);
+    };
+}

src/node/consumer/src/lib/services/configuration_service.ts

@@ -0,0 +1,17 @@
+import {cacheConfig} from "@models/configuration/cache_config";
+import {databaseConfig} from "@models/configuration/database_config";
+import {jobConfig} from "@models/configuration/job_config";
+import {metadataConfig} from "@models/configuration/metadata_config";
+import {rabbitConfig} from "@models/configuration/rabbit_config";
+import {torrentConfig} from "@models/configuration/torrent_config";
+import {trackerConfig} from "@models/configuration/tracker_config";
+
+export const configurationService = {
+    rabbitConfig: rabbitConfig,
+    cacheConfig: cacheConfig,
+    databaseConfig: databaseConfig,
+    jobConfig: jobConfig,
+    metadataConfig: metadataConfig,
+    trackerConfig: trackerConfig,
+    torrentConfig: torrentConfig
+};

src/node/consumer/src/lib/services/logging_service.ts

@@ -0,0 +1,32 @@
+import {ILoggingService} from "@interfaces/logging_service";
+import {injectable} from "inversify";
+import {Logger, pino} from "pino";
+
+/* eslint-disable @typescript-eslint/no-explicit-any */
+@injectable()
+export class LoggingService implements ILoggingService {
+    private readonly logger: Logger;
+
+    constructor() {
+        this.logger = pino({
+            level: process.env.LOG_LEVEL || 'info'
+        });
+    }
+
+    public info = (message: string, ...args: any[]): void => {
+        this.logger.info(message, args);
+    };
+
+    public error = (message: string, ...args: any[]): void => {
+        this.logger.error(message, args);
+    };
+
+    public debug = (message: string, ...args: any[]): void => {
+        this.logger.debug(message, args);
+    };
+
+    public warn = (message: string, ...args: any[]): void => {
+        this.logger.warn(message, args);
+    };
+}
+/* eslint-enable @typescript-eslint/no-explicit-any */

src/node/consumer/src/lib/services/metadata_service.ts

@@ -0,0 +1,234 @@
+import {TorrentType} from '@enums/torrent_types';
+import {ICacheService} from "@interfaces/cache_service";
+import {ICinemetaJsonResponse} from "@interfaces/cinemeta_metadata";
+import {ICommonVideoMetadata} from "@interfaces/common_video_metadata";
+import {IKitsuCatalogJsonResponse} from "@interfaces/kitsu_catalog_metadata";
+import {IKitsuJsonResponse} from "@interfaces/kitsu_metadata";
+import {IMetaDataQuery} from "@interfaces/metadata_query";
+import {IMetadataResponse} from "@interfaces/metadata_response";
+import {IMetadataService} from "@interfaces/metadata_service";
+import {IocTypes} from "@setup/ioc_types";
+import axios from 'axios';
+import {ResultTypes, search} from 'google-sr';
+import {inject, injectable} from "inversify";
+import nameToImdb from 'name-to-imdb';
+
+const CINEMETA_URL = 'https://v3-cinemeta.strem.io';
+const KITSU_URL = 'https://anime-kitsu.strem.fun';
+const TIMEOUT = 60000;
+
+@injectable()
+export class MetadataService implements IMetadataService {
+    @inject(IocTypes.ICacheService) private cacheService: ICacheService;
+
+    async getKitsuId(info: IMetaDataQuery): Promise<number | Error> {
+        const title = this.escapeTitle(info.title!.replace(/\s\|\s.*/, ''));
+        const year = info.year ? ` ${info.year}` : '';
+        const season = info.season || 0 > 1 ? ` S${info.season}` : '';
+        const key = `${title}${year}${season}`;
+        const query = encodeURIComponent(key);
+
+        return this.cacheService.cacheWrapKitsuId(key,
+            () => axios.get(`${KITSU_URL}/catalog/series/kitsu-anime-list/search=${query}.json`, {timeout: TIMEOUT})
+                .then((response) => {
+                    const body = response.data as IKitsuCatalogJsonResponse;
+                    if (body && body.metas && body.metas.length) {
+                        return body.metas[0].id.replace('kitsu:', '');
+                    } else {
+                        throw new Error('No search results');
+                    }
+                }));
+    }
+
+    async getImdbId(info: IMetaDataQuery): Promise<string | undefined> {
+        const name = this.escapeTitle(info.title!);
+        const year = info.year || (info.date && info.date.slice(0, 4));
+        const key = `${name}_${year || 'NA'}_${info.type}`;
+        const query = `${name} ${year || ''} ${info.type} imdb`;
+        const fallbackQuery = `${name} ${info.type} imdb`;
+        const googleQuery = year ? query : fallbackQuery;
+
+        try {
+            const imdbId = await this.cacheService.cacheWrapImdbId(key,
+                () => this.getIMDbIdFromNameToImdb(name, info)
+            );
+            return imdbId && 'tt' + imdbId.replace(/tt0*([1-9][0-9]*)$/, '$1').padStart(7, '0');
+        } catch (error) {
+            const imdbIdFallback = await this.getIMDbIdFromGoogle(googleQuery);
+            return imdbIdFallback && 'tt' + imdbIdFallback.toString().replace(/tt0*([1-9][0-9]*)$/, '$1').padStart(7, '0');
+        }
+    }
+
+    async getMetadata(query: IMetaDataQuery): Promise<IMetadataResponse | Error> {
+        if (!query.id) {
+            return Promise.reject("no valid id provided");
+        }
+
+        const key = Number.isInteger(query.id) || query.id.toString().match(/^\d+$/) ? `kitsu:${query.id}` : query.id;
+        const metaType = query.type === TorrentType.Movie ? TorrentType.Movie : TorrentType.Series;
+        const isImdbId = Boolean(key.toString().match(/^tt\d+$/));
+
+        try {
+            try {
+                return await this.cacheService.cacheWrapMetadata(key.toString(), () => {
+                    switch (isImdbId) {
+                        case true:
+                            return this.requestMetadata(`${CINEMETA_URL}/meta/imdb/${key}.json`, this.handleCinemetaResponse);
+                        default:
+                            return this.requestMetadata(`${KITSU_URL}/meta/${metaType}/${key}.json`, this.handleKitsuResponse)
+                    }
+                });
+            } catch (e) {
+                // try different type in case there was a mismatch
+                const otherType = metaType === TorrentType.Movie ? TorrentType.Series : TorrentType.Movie;
+                return this.requestMetadata(`${CINEMETA_URL}/meta/${otherType}/${key}.json`, this.handleCinemetaResponse)
+            }
+        } catch (error) {
+            throw new Error(`failed metadata query ${key} due: ${error.message}`);
+        }
+    }
+
+    async isEpisodeImdbId(imdbId: string | undefined): Promise<boolean> {
+        if (!imdbId || !imdbId.toString().match(/^tt\d+$/)) {
+            return false;
+        }
+
+        try {
+            const response = await axios.get(`https://www.imdb.com/title/${imdbId}/`, {timeout: TIMEOUT});
+            return response.data.includes('video.episode');
+        } catch (error) {
+            return false;
+        }
+    }
+
+    escapeTitle(title: string): string {
+        return title.toLowerCase()
+            .normalize('NFKD') // normalize non-ASCII characters
+            .replace(/[\u0300-\u036F]/g, '')
+            .replace(/&/g, 'and')
+            .replace(/[;, ~./]+/g, ' ') // replace dots, commas or underscores with spaces
+            .replace(/[^\w \-()×+#@!'\u0400-\u04ff]+/g, '') // remove all non-alphanumeric chars
+            .replace(/^\d{1,2}[.#\s]+(?=(?:\d+[.\s]*)?[\u0400-\u04ff])/i, '') // remove russian movie numbering
+            .replace(/\s{2,}/, ' ') // replace multiple spaces
+            .trim();
+    }
+
+    private requestMetadata = async (url: string, handler: (body: unknown) => IMetadataResponse): Promise<IMetadataResponse> => {
+        try {
+            const response = await axios.get(url, {timeout: TIMEOUT});
+            const body = response.data;
+            return handler(body);
+        } catch (error) {
+            throw new Error(`HTTP error! status: ${error.response?.status}`);
+        }
+    };
+
+    private handleCinemetaResponse = (response: unknown): IMetadataResponse => {
+        const body = response as ICinemetaJsonResponse
+
+        return ({
+            imdbId: parseInt(body.meta?.id || '0'),
+            type: body.meta?.type,
+            title: body.meta?.name,
+            year: parseInt(body.meta?.year || '0'),
+            country: body.meta?.country,
+            genres: body.meta?.genres,
+            status: body.meta?.status,
+            videos: body.meta?.videos
+                ? body.meta.videos.map(video => ({
+                    name: video.name,
+                    season: video.season,
+                    episode: video.episode,
+                    imdbSeason: video.season,
+                    imdbEpisode: video.episode,
+                }))
+                : [],
+            episodeCount: body.meta?.videos
+                ? this.getEpisodeCount(body.meta.videos)
+                : [],
+            totalCount: body.meta?.videos
+                ? body.meta.videos.filter(
+                    entry => entry.season !== 0 && entry.episode !== 0
+                ).length
+                : 0,
+        });
+    };
+
+    private handleKitsuResponse = (response: unknown): IMetadataResponse => {
+        const body = response as IKitsuJsonResponse;
+
+        return ({
+            kitsuId: parseInt(body.meta?.kitsu_id || '0'),
+            type: body.meta?.type,
+            title: body.meta?.name,
+            year: parseInt(body.meta?.year || '0'),
+            country: body.meta?.country,
+            genres: body.meta?.genres,
+            status: body.meta?.status,
+            videos: body.meta?.videos
+                ? body.meta?.videos.map(video => ({
+                    name: video.title,
+                    season: video.season,
+                    episode: video.episode,
+                    kitsuId: video.id,
+                    kitsuEpisode: video.episode,
+                    released: video.released,
+                }))
+                : [],
+            episodeCount: body.meta?.videos
+                ? this.getEpisodeCount(body.meta.videos)
+                : [],
+            totalCount: body.meta?.videos
+                ? body.meta.videos.filter(
+                    entry => entry.season !== 0 && entry.episode !== 0
+                ).length
+                : 0,
+        });
+    };
+
+    private getEpisodeCount = (videos: ICommonVideoMetadata[]): number[] =>
+        Object.values(
+            videos
+                .filter(entry => entry.season !== null && entry.season !== 0 && entry.episode !== 0)
+                .sort((a, b) => (a.season || 0) - (b.season || 0))
+                .reduce((map: Record<number, number>, next) => {
+                    if (next.season || next.season === 0) {
+                        map[next.season] = (map[next.season] || 0) + 1;
+                    }
+                    return map;
+                }, {})
+        );
+
+    private getIMDbIdFromNameToImdb = (name: string, info: IMetaDataQuery): Promise<string | Error> => {
+        const {year} = info;
+        const {type} = info;
+        return new Promise((resolve, reject) => {
+            nameToImdb({name, year, type}, function (err: Error, res: string) {
+                if (res) {
+                    resolve(res);
+                } else {
+                    reject(err || new Error('Failed IMDbId search'));
+                }
+            });
+        });
+    };
+
+    private getIMDbIdFromGoogle = async (query: string): Promise<string | undefined> => {
+        try {
+            const searchResults = await search({query: query});
+            for (const result of searchResults) {
+                if (result.type === ResultTypes.SearchResult) {
+                    if (result.link.includes('imdb.com/title/')) {
+                        const match = result.link.match(/imdb\.com\/title\/(tt\d+)/);
+                        if (match) {
+                            return match[1];
+                        }
+                    }
+                }
+            }
+            return undefined;
+        } catch (error) {
+            throw new Error('Failed to find IMDb ID from Google search');
+        }
+    };
+}

src/node/consumer/src/lib/services/torrent_download_service.ts

@@ -0,0 +1,164 @@
+import {ExtensionHelpers} from '@helpers/extension_helpers';
+import {ILoggingService} from "@interfaces/logging_service";
+import {IParsedTorrent} from "@interfaces/parsed_torrent";
+import {ITorrentDownloadService} from "@interfaces/torrent_download_service";
+import {ITorrentFileCollection} from "@interfaces/torrent_file_collection";
+import {IContentAttributes} from "@repository/interfaces/content_attributes";
+import {IFileAttributes} from "@repository/interfaces/file_attributes";
+import {ISubtitleAttributes} from "@repository/interfaces/subtitle_attributes";
+import {configurationService} from '@services/configuration_service';
+import {IocTypes} from "@setup/ioc_types";
+import {inject, injectable} from "inversify";
+import {encode} from 'magnet-uri';
+import {parse} from "parse-torrent-title";
+// eslint-disable-next-line import/no-extraneous-dependencies
+import * as torrentStream from "torrent-stream";
+import TorrentEngine = TorrentStream.TorrentEngine;
+import TorrentEngineOptions = TorrentStream.TorrentEngineOptions;
+
+interface ITorrentFile {
+    name: string;
+    path: string;
+    length: number;
+    fileIndex: number;
+}
+
+@injectable()
+export class TorrentDownloadService implements ITorrentDownloadService {
+    @inject(IocTypes.ILoggingService) private logger: ILoggingService;
+
+    private engineOptions: TorrentEngineOptions = {
+        connections: configurationService.torrentConfig.MAX_CONNECTIONS_PER_TORRENT,
+        uploads: 0,
+        verify: false,
+        dht: false,
+        tracker: true,
+    };
+
+    async getTorrentFiles(torrent: IParsedTorrent, timeout: number = 30000): Promise<ITorrentFileCollection> {
+        const torrentFiles: ITorrentFile[] = await this.filesFromTorrentStream(torrent, timeout);
+
+        const videos = this.filterVideos(torrent, torrentFiles);
+        const subtitles = this.filterSubtitles(torrent, torrentFiles);
+        const contents = this.createContent(torrent, torrentFiles);
+
+        return {
+            contents: contents,
+            videos: videos,
+            subtitles: subtitles,
+        };
+    }
+
+    private filesFromTorrentStream = async (torrent: IParsedTorrent, timeout: number): Promise<ITorrentFile[]> => {
+        if (!torrent.infoHash) {
+            return Promise.reject(new Error("No infoHash..."));
+        }
+        const magnet = encode({infoHash: torrent.infoHash, announce: torrent.trackers!.split(',')});
+
+        return new Promise((resolve, reject) => {
+            this.logger.debug(`Adding torrent with infoHash ${torrent.infoHash} to torrent engine...`);
+
+            const timeoutId = setTimeout(() => {
+                engine.destroy(() => {
+                });
+                reject(new Error('No available connections for torrent!'));
+            }, timeout);
+
+            const engine: TorrentEngine = torrentStream.default(magnet, this.engineOptions);
+
+            engine.on("ready", () => {
+                const files: ITorrentFile[] = engine.files.map((file, fileId) => ({
+                    fileIndex: fileId,
+                    length: file.length,
+                    name: file.name,
+                    path: file.path,
+                }));
+
+                resolve(files);
+                clearTimeout(timeoutId);
+                engine.destroy(() => {
+                });
+            });
+        });
+    };
+
+    private filterVideos = (torrent: IParsedTorrent, torrentFiles: ITorrentFile[]): IFileAttributes[] => {
+        if (torrentFiles.length === 1 && !Number.isInteger(torrentFiles[0].fileIndex)) {
+            return [this.mapTorrentFileToFileAttributes(torrent, torrentFiles[0])];
+        }
+        const videos = torrentFiles.filter(file => ExtensionHelpers.isVideo(file.path || ''));
+        const maxSize = Math.max(...videos.map((video: ITorrentFile) => video.length));
+        const minSampleRatio = videos.length <= 3 ? 3 : 10;
+        const minAnimeExtraRatio = 5;
+        const minRedundantRatio = videos.length <= 3 ? 30 : Number.MAX_VALUE;
+
+        const isSample = (video: ITorrentFile): boolean => video.path?.toString()?.match(/sample|bonus|promo/i) && maxSize / video.length > minSampleRatio || false;
+        const isRedundant = (video: ITorrentFile): boolean => maxSize / video.length > minRedundantRatio;
+        const isExtra = (video: ITorrentFile): boolean => /extras?\//i.test(video.path?.toString() || "");
+        const isAnimeExtra = (video: ITorrentFile): boolean => {
+            if (!video.path || !video.length) {
+                return false;
+            }
+
+            return video.path.toString()?.match(/(?:\b|_)(?:NC)?(?:ED|OP|PV)(?:v?\d\d?)?(?:\b|_)/i)
+                && maxSize / parseInt(video.length.toString()) > minAnimeExtraRatio || false;
+        };
+        const isWatermark = (video: ITorrentFile): boolean => {
+            if (!video.path || !video.length) {
+                return false;
+            }
+
+            return video.path.toString()?.match(/^[A-Z-]+(?:\.[A-Z]+)?\.\w{3,4}$/)
+                && maxSize / parseInt(video.length.toString()) > minAnimeExtraRatio || false;
+        }
+
+        return videos
+            .filter(video => !isSample(video))
+            .filter(video => !isExtra(video))
+            .filter(video => !isAnimeExtra(video))
+            .filter(video => !isRedundant(video))
+            .filter(video => !isWatermark(video))
+            .map(video => this.mapTorrentFileToFileAttributes(torrent, video));
+    };
+
+    private filterSubtitles = (torrent: IParsedTorrent, torrentFiles: ITorrentFile[]): ISubtitleAttributes[] => torrentFiles.filter(file => ExtensionHelpers.isSubtitle(file.name || ''))
+        .map(file => this.mapTorrentFileToSubtitleAttributes(torrent, file));
+
+    private createContent = (torrent: IParsedTorrent, torrentFiles: ITorrentFile[]): IContentAttributes[] => torrentFiles.map(file => this.mapTorrentFileToContentAttributes(torrent, file));
+
+    private mapTorrentFileToFileAttributes = (torrent: IParsedTorrent, file: ITorrentFile): IFileAttributes => {
+        try {
+            const videoFile: IFileAttributes = {
+                title: file.name,
+                size: file.length,
+                fileIndex: file.fileIndex || 0,
+                path: file.path,
+                infoHash: torrent.infoHash?.toString(),
+                imdbId: torrent.imdbId?.toString() || '',
+                imdbSeason: torrent.season || 0,
+                imdbEpisode: torrent.episode || 0,
+                kitsuId: parseInt(torrent.kitsuId?.toString() || '0') || 0,
+                kitsuEpisode: torrent.episode || 0,
+            };
+
+            return {...videoFile, ...parse(file.name)};
+        } catch (error) {
+            throw new Error(`Error parsing file ${file.name} from torrent ${torrent.infoHash}: ${error}`);
+        }
+    };
+
+    private mapTorrentFileToSubtitleAttributes = (torrent: IParsedTorrent, file: ITorrentFile): ISubtitleAttributes => ({
+        title: file.name,
+        infoHash: torrent.infoHash,
+        fileIndex: file.fileIndex,
+        fileId: file.fileIndex,
+        path: file.path,
+    });
+
+    private mapTorrentFileToContentAttributes = (torrent: IParsedTorrent, file: ITorrentFile): IContentAttributes => ({
+        infoHash: torrent.infoHash,
+        fileIndex: file.fileIndex,
+        path: file.path,
+        size: file.length,
+    });
+}

src/node/consumer/src/lib/services/torrent_entries_service.ts

@@ -0,0 +1,288 @@
+import {TorrentType} from '@enums/torrent_types';
+import {PromiseHelpers} from '@helpers/promises_helpers';
+import {ILoggingService} from "@interfaces/logging_service";
+import {IMetaDataQuery} from "@interfaces/metadata_query";
+import {IMetadataService} from "@interfaces/metadata_service";
+import {IParsedTorrent} from "@interfaces/parsed_torrent";
+import {ITorrentEntriesService} from "@interfaces/torrent_entries_service";
+import {ITorrentFileCollection} from "@interfaces/torrent_file_collection";
+import {ITorrentFileService} from "@interfaces/torrent_file_service";
+import {ITorrentSubtitleService} from "@interfaces/torrent_subtitle_service";
+import {IDatabaseRepository} from "@repository/interfaces/database_repository";
+import {IFileCreationAttributes} from "@repository/interfaces/file_attributes";
+import {ISubtitleAttributes} from "@repository/interfaces/subtitle_attributes";
+import {ITorrentAttributes, ITorrentCreationAttributes} from "@repository/interfaces/torrent_attributes";
+import {File} from "@repository/models/file";
+import {SkipTorrent} from "@repository/models/skipTorrent";
+import {Subtitle} from "@repository/models/subtitle";
+import {Torrent} from "@repository/models/torrent";
+import {IocTypes} from "@setup/ioc_types";
+import {inject, injectable} from "inversify";
+import {parse} from 'parse-torrent-title';
+
+@injectable()
+export class TorrentEntriesService implements ITorrentEntriesService {
+    @inject(IocTypes.IMetadataService) private metadataService: IMetadataService;
+    @inject(IocTypes.ILoggingService) private logger: ILoggingService;
+    @inject(IocTypes.ITorrentFileService) private fileService: ITorrentFileService;
+    @inject(IocTypes.ITorrentSubtitleService) private subtitleService: ITorrentSubtitleService;
+    @inject(IocTypes.IDatabaseRepository) private repository: IDatabaseRepository;
+
+    async createTorrentEntry(torrent: IParsedTorrent, overwrite = false): Promise<void> {
+        if (!torrent.title) {
+            this.logger.warn(`No title found for ${torrent.provider} [${torrent.infoHash}]`);
+            return;
+        }
+
+        const titleInfo = parse(torrent.title);
+
+        if (!torrent.imdbId && torrent.type !== TorrentType.Anime) {
+            const imdbQuery = {
+                title: titleInfo.title,
+                year: titleInfo.year,
+                type: torrent.type
+            };
+            torrent.imdbId = await this.metadataService.getImdbId(imdbQuery)
+                .catch(() => undefined);
+        }
+        if (torrent.imdbId && torrent.imdbId.toString().length < 9) {
+            // pad zeros to imdbId if missing
+            torrent.imdbId = 'tt' + torrent.imdbId.toString().replace('tt', '').padStart(7, '0');
+        }
+        if (torrent.imdbId && torrent.imdbId.toString().length > 9 && torrent.imdbId.toString().startsWith('tt0')) {
+            // sanitize imdbId from redundant zeros
+            torrent.imdbId = torrent.imdbId.toString().replace(/tt0+([0-9]{7,})$/, 'tt$1');
+        }
+        if (!torrent.kitsuId && torrent.type === TorrentType.Anime) {
+            const kitsuQuery = {
+                title: titleInfo.title,
+                year: titleInfo.year,
+                season: titleInfo.season,
+            };
+
+            await this.assignKitsuId(kitsuQuery, torrent);
+        }
+
+        if (!torrent.imdbId && !torrent.kitsuId && !this.fileService.isPackTorrent(torrent)) {
+            this.logger.warn(`imdbId or kitsuId not found:  ${torrent.provider} ${torrent.title}`);
+            return;
+        }
+
+        const fileCollection: ITorrentFileCollection = await this.fileService.parseTorrentFiles(torrent)
+            .then((torrentContents: ITorrentFileCollection) => overwrite ? this.overwriteExistingFiles(torrent, torrentContents) : torrentContents)
+            .then((torrentContents: ITorrentFileCollection) => this.subtitleService.assignSubtitles(torrentContents))
+            .catch(error => {
+                this.logger.warn(`Failed getting files for ${torrent.title}`, error.message);
+                return {};
+            });
+
+        if (!fileCollection.videos || !fileCollection.videos.length) {
+            this.logger.warn(`no video files found for ${torrent.provider} [${torrent.infoHash}] ${torrent.title}`);
+            return;
+        }
+
+        const newTorrent: ITorrentCreationAttributes = ({
+            ...torrent,
+            contents: fileCollection.contents,
+            subtitles: fileCollection.subtitles
+        });
+
+        return this.repository.createTorrent(newTorrent)
+            .then(() => PromiseHelpers.sequence(fileCollection.videos!.map(video => () => {
+                const newVideo: IFileCreationAttributes = {...video, infoHash: video.infoHash, title: video.title};
+                if (!newVideo.kitsuId) {
+                    newVideo.kitsuId = 0;
+                }
+                return this.repository.createFile(newVideo)
+            })))
+            .then(() => this.logger.info(`Created ${torrent.provider} entry for [${torrent.infoHash}] ${torrent.title}`));
+    }
+
+    async createSkipTorrentEntry(torrent: ITorrentCreationAttributes): Promise<[SkipTorrent, boolean | null]> {
+        return this.repository.createSkipTorrent(torrent);
+    }
+
+    async getStoredTorrentEntry(torrent: Torrent): Promise<Torrent | SkipTorrent | null | undefined> {
+        return this.repository.getSkipTorrent(torrent.infoHash)
+            .catch(() => this.repository.getTorrent(torrent.dataValues))
+            .catch(() => undefined);
+    }
+
+    async checkAndUpdateTorrent(torrent: IParsedTorrent): Promise<boolean> {
+        const query: ITorrentAttributes = {
+            infoHash: torrent.infoHash,
+            provider: torrent.provider,
+        }
+
+        const existingTorrent = await this.repository.getTorrent(query).catch(() => undefined);
+
+        if (!existingTorrent) {
+            return false;
+        }
+
+        if (existingTorrent.provider === 'RARBG') {
+            return true;
+        }
+        if (existingTorrent.provider === 'KickassTorrents' && torrent.provider) {
+            existingTorrent.provider = torrent.provider;
+            existingTorrent.torrentId = torrent.torrentId!;
+        }
+
+        if (!existingTorrent.languages && torrent.languages && existingTorrent.provider !== 'RARBG') {
+            existingTorrent.languages = torrent.languages;
+            await existingTorrent.save();
+            this.logger.debug(`Updated [${existingTorrent.infoHash}] ${existingTorrent.title} language to ${torrent.languages}`);
+        }
+
+        return this.createTorrentContents(existingTorrent)
+            .then(() => this.updateTorrentSeeders(existingTorrent.dataValues))
+            .then(() => Promise.resolve(true))
+            .catch(() => Promise.reject(false));
+    }
+
+    async createTorrentContents(torrent: Torrent): Promise<void> {
+        if (torrent.opened) {
+            return;
+        }
+
+        const storedVideos: File[] = await this.repository.getFiles(torrent.infoHash).catch(() => []);
+        if (!storedVideos || !storedVideos.length) {
+            return;
+        }
+        const notOpenedVideo = storedVideos.length === 1 && !Number.isInteger(storedVideos[0].fileIndex);
+        const imdbId: string | undefined = PromiseHelpers.mostCommonValue(storedVideos.map(stored => stored.imdbId));
+        const kitsuId: number = PromiseHelpers.mostCommonValue(storedVideos.map(stored => stored.kitsuId || 0));
+
+        const fileCollection: ITorrentFileCollection = await this.fileService.parseTorrentFiles(torrent)
+            .then(torrentContents => notOpenedVideo ? torrentContents : {
+                ...torrentContents,
+                videos: storedVideos.map(video => video.dataValues)
+            })
+            .then(torrentContents => this.subtitleService.assignSubtitles(torrentContents))
+            .then(torrentContents => this.assignMetaIds(torrentContents, imdbId, kitsuId))
+            .catch(error => {
+                this.logger.warn(`Failed getting contents for [${torrent.infoHash}] ${torrent.title}`, error.message);
+                return {};
+            });
+
+        if (!fileCollection.contents || !fileCollection.contents.length) {
+            return;
+        }
+
+        if (notOpenedVideo && fileCollection.videos?.length === 1) {
+            // if both have a single video and stored one was not opened, update stored one to true metadata and use that
+            storedVideos[0].fileIndex = fileCollection?.videos[0]?.fileIndex || 0;
+            storedVideos[0].title = fileCollection.videos[0].title;
+            storedVideos[0].size = fileCollection.videos[0].size || 0;
+            const subtitles: ISubtitleAttributes[] = fileCollection.videos[0]?.subtitles || [];
+            storedVideos[0].subtitles = subtitles.map(subtitle => Subtitle.build(subtitle));
+            fileCollection.videos[0] = {...storedVideos[0], subtitles: subtitles};
+        }
+        // no videos available or more than one new videos were in the torrent
+        const shouldDeleteOld = notOpenedVideo && fileCollection.videos?.every(video => !video.id) || false;
+
+        const newTorrent: ITorrentCreationAttributes = {
+            ...torrent,
+            files: fileCollection.videos,
+            contents: fileCollection.contents,
+            subtitles: fileCollection.subtitles
+        };
+
+        return this.repository.createTorrent(newTorrent)
+            .then(() => {
+                if (shouldDeleteOld) {
+                    this.logger.debug(`Deleting old video for [${torrent.infoHash}] ${torrent.title}`)
+                    return storedVideos[0].destroy();
+                }
+                return Promise.resolve();
+            })
+            .then(() => {
+                const promises = fileCollection.videos!.map(video => {
+                    const newVideo: IFileCreationAttributes = {...video, infoHash: video.infoHash, title: video.title};
+                    return this.repository.createFile(newVideo);
+                });
+                return Promise.all(promises);
+            })
+            .then(() => this.logger.info(`Created contents for ${torrent.provider} [${torrent.infoHash}] ${torrent.title}`))
+            .catch(error => this.logger.error(`Failed saving contents for [${torrent.infoHash}] ${torrent.title}`, error));
+    }
+
+    async updateTorrentSeeders(torrent: ITorrentAttributes): Promise<[number]> {
+        if (!(torrent.infoHash || (torrent.provider && torrent.torrentId)) || !Number.isInteger(torrent.seeders)) {
+            return [0];
+        }
+
+        if (torrent.seeders === undefined) {
+            this.logger.warn(`Seeders not found for ${torrent.provider} [${torrent.infoHash}] ${torrent.title}`);
+            return [0];
+        }
+
+
+        return this.repository.setTorrentSeeders(torrent, torrent.seeders)
+            .catch(error => {
+                this.logger.warn('Failed updating seeders:', error);
+                return [0];
+            });
+    }
+
+    private assignKitsuId = async (kitsuQuery: IMetaDataQuery, torrent: IParsedTorrent): Promise<void> => {
+        await this.metadataService.getKitsuId(kitsuQuery)
+            .then((result: number | Error) => {
+                if (typeof result === 'number') {
+                    torrent.kitsuId = result;
+                } else {
+                    torrent.kitsuId = 0;
+                }
+            })
+            .catch((error: Error) => {
+                this.logger.debug(`Failed getting kitsuId for ${torrent.title}`, error.message);
+                torrent.kitsuId = 0;
+            });
+    };
+
+    private assignMetaIds = (fileCollection: ITorrentFileCollection, imdbId: string | undefined, kitsuId: number): ITorrentFileCollection => {
+        if (fileCollection && fileCollection.videos && fileCollection.videos.length) {
+            fileCollection.videos.forEach(video => {
+                video.imdbId = imdbId || '';
+                video.kitsuId = kitsuId || 0;
+            });
+        }
+
+        return fileCollection;
+    };
+
+    private overwriteExistingFiles = async (torrent: IParsedTorrent, torrentContents: ITorrentFileCollection): Promise<ITorrentFileCollection> => {
+        const videos = torrentContents && torrentContents.videos;
+        if (videos && videos.length) {
+            const existingFiles = await this.repository.getFiles(torrent.infoHash)
+                .then((existing) => existing.reduce<{ [key: number]: File[] }>((map, next) => {
+                    const fileIndex = next.fileIndex !== undefined ? next.fileIndex : null;
+                    if (fileIndex !== null) {
+                        map[fileIndex] = (map[fileIndex] || []).concat(next);
+                    }
+                    return map;
+                }, {}))
+                .catch(() => undefined);
+            if (existingFiles && Object.keys(existingFiles).length) {
+                const overwrittenVideos = videos
+                    .map(file => {
+                        const index = file.fileIndex !== undefined ? file.fileIndex : null;
+                        let mapping;
+                        if (index !== null) {
+                            mapping = videos.length === 1 && Object.keys(existingFiles).length === 1
+                                ? Object.values(existingFiles)[0]
+                                : existingFiles[index];
+                        }
+                        if (mapping) {
+                            const originalFile = mapping.shift();
+                            return {id: originalFile!.id, ...file};
+                        }
+                        return file;
+                    });
+                return {...torrentContents, videos: overwrittenVideos};
+            }
+            return torrentContents;
+        }
+        return Promise.reject(`No video files found for: ${torrent.title}`);
+    };
+}

src/node/consumer/src/lib/services/torrent_file_service.ts

@@ -0,0 +1,733 @@
+import {TorrentType} from '@enums/torrent_types';
+import {ExtensionHelpers} from '@helpers/extension_helpers';
+import {PromiseHelpers} from '@helpers/promises_helpers';
+import {ICommonVideoMetadata} from "@interfaces/common_video_metadata";
+import {ILoggingService} from "@interfaces/logging_service";
+import {IMetaDataQuery} from "@interfaces/metadata_query";
+import {IMetadataResponse} from "@interfaces/metadata_response";
+import {IMetadataService} from "@interfaces/metadata_service";
+import {IParsedTorrent} from "@interfaces/parsed_torrent";
+import {ITorrentDownloadService} from "@interfaces/torrent_download_service";
+import {ITorrentFileCollection} from "@interfaces/torrent_file_collection";
+import {ITorrentFileService} from "@interfaces/torrent_file_service";
+import {IContentAttributes} from "@repository/interfaces/content_attributes";
+import {IFileAttributes} from "@repository/interfaces/file_attributes";
+import {configurationService} from '@services/configuration_service';
+import {IocTypes} from "@setup/ioc_types";
+import Bottleneck from 'bottleneck';
+import {inject, injectable} from "inversify";
+import moment from 'moment';
+import {parse} from 'parse-torrent-title';
+
+const MIN_SIZE: number = 5 * 1024 * 1024; // 5 MB
+const MULTIPLE_FILES_SIZE = 4 * 1024 * 1024 * 1024; // 4 GB
+
+type SeasonEpisodeMap = Record<number, Record<number, ICommonVideoMetadata>>;
+
+@injectable()
+export class TorrentFileService implements ITorrentFileService {
+    @inject(IocTypes.IMetadataService) metadataService: IMetadataService;
+    @inject(IocTypes.ITorrentDownloadService) torrentDownloadService: ITorrentDownloadService;
+    @inject(IocTypes.ILoggingService) logger: ILoggingService;
+
+    private readonly imdb_limiter: Bottleneck = new Bottleneck({
+        maxConcurrent: configurationService.metadataConfig.IMDB_CONCURRENT,
+        minTime: configurationService.metadataConfig.IMDB_INTERVAL_MS
+    });
+
+    async parseTorrentFiles(torrent: IParsedTorrent): Promise<ITorrentFileCollection> {
+        if (!torrent.title) {
+            return Promise.reject(new Error('Torrent title is missing'));
+        }
+
+        if (!torrent.infoHash) {
+            return Promise.reject(new Error('Torrent infoHash is missing'));
+        }
+
+        const parsedTorrentName = parse(torrent.title);
+        const query: IMetaDataQuery = {
+            id: torrent.kitsuId || torrent.imdbId,
+            type: torrent.type || TorrentType.Movie,
+        };
+        const metadata = await this.metadataService.getMetadata(query)
+            .then(meta => Object.assign({}, meta))
+            .catch(() => undefined);
+
+        if (metadata === undefined || metadata instanceof Error) {
+            return Promise.reject(new Error('Failed to retrieve metadata'));
+        }
+
+        if (torrent.type !== TorrentType.Anime && metadata && metadata.type && metadata.type !== torrent.type) {
+            // it's actually a movie/series
+            torrent.type = metadata.type;
+        }
+
+        if (torrent.type === TorrentType.Movie && (!parsedTorrentName.seasons ||
+            parsedTorrentName.season === 5 && [1, 5].includes(parsedTorrentName.episode || 0))) {
+            return this.parseMovieFiles(torrent, metadata);
+        }
+
+        return this.parseSeriesFiles(torrent, metadata)
+    }
+
+    isPackTorrent(torrent: IParsedTorrent): boolean {
+        if (torrent.isPack) {
+            return true;
+        }
+        if (!torrent.title) {
+            return false;
+        }
+        const parsedInfo = parse(torrent.title);
+        if (torrent.type === TorrentType.Movie) {
+            return parsedInfo.complete || typeof parsedInfo.year === 'string' || /movies/i.test(torrent.title);
+        }
+
+        const hasMultipleEpisodes = Boolean(parsedInfo.complete || torrent.size || 0 > MULTIPLE_FILES_SIZE ||
+            (parsedInfo.seasons && parsedInfo.seasons.length > 1) ||
+            (parsedInfo.episodes && parsedInfo.episodes.length > 1) ||
+            (parsedInfo.seasons && !parsedInfo.episodes));
+
+        const hasSingleEpisode: boolean = Boolean(Number.isInteger(parsedInfo.episode) || (!parsedInfo.episodes && parsedInfo.date));
+
+        return hasMultipleEpisodes && !hasSingleEpisode;
+    }
+
+    private parseSeriesVideos = (torrent: IParsedTorrent, videos: IFileAttributes[]): IFileAttributes[] => {
+        const parsedTorrentName = parse(torrent.title!);
+        const hasMovies = parsedTorrentName.complete || !!torrent.title!.match(/movies?(?:\W|$)/i);
+        const parsedVideos = videos.map(video => this.parseSeriesVideo(video));
+
+        return parsedVideos.map(video => ({...video, isMovie: this.isMovieVideo(torrent, video, parsedVideos, hasMovies)}));
+    };
+
+    private parseMovieFiles = async (torrent: IParsedTorrent, metadata: IMetadataResponse): Promise<ITorrentFileCollection> => {
+        const fileCollection: ITorrentFileCollection = await this.getMoviesTorrentContent(torrent);
+        if (fileCollection.videos === undefined || fileCollection.videos.length === 0) {
+            return {...fileCollection, videos: this.getDefaultFileEntries(torrent)};
+        }
+
+        const filteredVideos = fileCollection.videos
+            .filter(video => video.size! > MIN_SIZE)
+            .filter(video => !this.isFeaturette(video));
+        if (this.isSingleMovie(filteredVideos)) {
+            const parsedVideos = filteredVideos.map(video => ({
+                infoHash: torrent.infoHash,
+                fileIndex: video.fileIndex,
+                title: video.title || video.path || video.fileName || '',
+                size: video.size || torrent.size,
+                imdbId: torrent.imdbId?.toString() || metadata && metadata.imdbId?.toString(),
+                kitsuId: parseInt(torrent.kitsuId?.toString() || metadata && metadata.kitsuId?.toString() || '0')
+            }));
+            return {...fileCollection, videos: parsedVideos};
+        }
+
+        const parsedVideos = await PromiseHelpers.sequence(filteredVideos.map(video => () => this.isFeaturette(video)
+            ? Promise.resolve(video)
+            : this.findMovieImdbId(video.title).then(imdbId => ({...video, imdbId: imdbId?.toString() || ''}))))
+            .then(videos => videos.map((video: IFileAttributes) => ({
+                infoHash: torrent.infoHash,
+                fileIndex: video.fileIndex,
+                title: video.title || video.path,
+                size: video.size,
+                imdbId: video.imdbId,
+            })));
+        return {...fileCollection, videos: parsedVideos};
+    };
+
+    private parseSeriesFiles = async (torrent: IParsedTorrent, metadata: IMetadataResponse): Promise<ITorrentFileCollection> => {
+        const fileCollection: ITorrentFileCollection = await this.getSeriesTorrentContent(torrent);
+        if (fileCollection.videos === undefined || fileCollection.videos.length === 0) {
+            return {...fileCollection, videos: this.getDefaultFileEntries(torrent)};
+        }
+
+        const parsedVideos: IFileAttributes[] = await Promise.resolve(fileCollection.videos)
+            .then(videos => videos.filter(video => videos?.length === 1 || video.size! > MIN_SIZE))
+            .then(videos => this.parseSeriesVideos(torrent, videos))
+            .then(videos => this.decomposeEpisodes(torrent, videos, metadata))
+            .then(videos => this.assignKitsuOrImdbEpisodes(torrent, videos, metadata))
+            .then(videos => Promise.all(videos.map(video => video.isMovie
+                ? this.mapSeriesMovie(torrent, video)
+                : this.mapSeriesEpisode(torrent, video, videos))))
+            .then(videos => videos
+                .reduce((a, b) => a.concat(b), [])
+                .map(video => this.isFeaturette(video) ? this.clearInfoFields(video) : video));
+        return {...torrent.fileCollection, videos: parsedVideos};
+    };
+
+    private getMoviesTorrentContent = async (torrent: IParsedTorrent): Promise<ITorrentFileCollection> => {
+        const files = await this.torrentDownloadService.getTorrentFiles(torrent, configurationService.torrentConfig.TIMEOUT)
+            .catch(error => {
+                if (!this.isPackTorrent(torrent)) {
+                    const entries = this.getDefaultFileEntries(torrent);
+                    return {videos: entries, contents: [], subtitles: [], files: entries}
+                }
+                return Promise.reject(error);
+            });
+
+        if (files.contents && files.contents.length && !files.videos?.length && this.isDiskTorrent(files.contents)) {
+            files.videos = this.getDefaultFileEntries(torrent);
+        }
+
+        return files;
+    };
+
+    private getDefaultFileEntries = (torrent: IParsedTorrent): IFileAttributes[] => [{
+        title: torrent.title!,
+        path: torrent.title,
+        size: torrent.size,
+        fileIndex: 0,
+    }];
+
+    private getSeriesTorrentContent = async (torrent: IParsedTorrent): Promise<ITorrentFileCollection> => this.torrentDownloadService.getTorrentFiles(torrent, configurationService.torrentConfig.TIMEOUT)
+        .catch(error => {
+            if (!this.isPackTorrent(torrent)) {
+                return {videos: this.getDefaultFileEntries(torrent), subtitles: [], contents: []}
+            }
+            return Promise.reject(error);
+        });
+
+    private mapSeriesEpisode = async (torrent: IParsedTorrent, file: IFileAttributes, files: IFileAttributes[]): Promise<IFileAttributes[]> => {
+        if (!file.episodes && !file.episodes) {
+            if (files.length === 1 || files.some(f => f.episodes || f.episodes) || parse(torrent.title!).seasons) {
+                return Promise.resolve([{
+                    infoHash: torrent.infoHash,
+                    fileIndex: file.fileIndex,
+                    title: file.path || file.title,
+                    size: file.size,
+                    imdbId: torrent?.imdbId?.toString() || file?.imdbId?.toString() || '',
+                }]);
+            }
+            return Promise.resolve([]);
+        }
+        const episodeIndexes = [...(file.episodes || file.episodes).keys()];
+        return Promise.resolve(episodeIndexes.map((index) => ({
+            infoHash: torrent.infoHash,
+            fileIndex: file.fileIndex,
+            title: file.path || file.title,
+            size: file.size,
+            imdbId: file?.imdbId?.toString() || torrent?.imdbId?.toString() || '',
+            imdbSeason: file.season,
+            season: file.season,
+            imdbEpisode: file.episodes && file.episodes[index],
+            episode: file.episodes && file.episodes[index],
+            kitsuEpisode: file.episodes && file.episodes[index],
+            episodes: file.episodes,
+            kitsuId: parseInt(file.kitsuId?.toString() || torrent.kitsuId?.toString() || '0') || 0,
+        })))
+    };
+
+    private mapSeriesMovie = async (torrent: IParsedTorrent, file: IFileAttributes): Promise<IFileAttributes[]> => {
+        const kitsuId = torrent.type === TorrentType.Anime ? await this.findMovieKitsuId(file)
+            .then(result => {
+                if (result instanceof Error) {
+                    this.logger.warn(`Failed to retrieve kitsuId due to error: ${result.message}`);
+                    return undefined;
+                }
+                return result;
+            }) : undefined;
+
+        const imdbId = !kitsuId ? await this.findMovieImdbId(file) : undefined;
+
+        const query: IMetaDataQuery = {
+            id: kitsuId || imdbId,
+            type: TorrentType.Movie
+        };
+
+        const metadataOrError = await this.metadataService.getMetadata(query);
+        if (metadataOrError instanceof Error) {
+            this.logger.warn(`Failed to retrieve metadata due to error: ${metadataOrError.message}`);
+            // return default result or throw error, depending on your use case
+            return [{
+                infoHash: torrent.infoHash,
+                fileIndex: file.fileIndex,
+                title: file.path || file.title,
+                size: file.size,
+                imdbId: imdbId,
+                kitsuId: parseInt(kitsuId?.toString() || '0') || 0,
+                episodes: undefined,
+                imdbSeason: undefined,
+                imdbEpisode: undefined,
+                kitsuEpisode: undefined
+            }];
+        }
+        // at this point, TypeScript infers that metadataOrError is actually MetadataResponse
+        const metadata = metadataOrError;
+        const hasEpisode = metadata.videos && metadata.videos.length && (file.episode || metadata.videos.length === 1);
+        const episodeVideo = hasEpisode && metadata.videos && metadata.videos[(file.episode || 1) - 1];
+        return [{
+            infoHash: torrent.infoHash,
+            fileIndex: file.fileIndex,
+            title: file.path || file.title,
+            size: file.size,
+            imdbId: metadata.imdbId?.toString() || imdbId || '',
+            kitsuId: parseInt(metadata.kitsuId?.toString() || kitsuId?.toString() || '0') || 0,
+            imdbSeason: episodeVideo && metadata.imdbId ? episodeVideo.season : undefined,
+            imdbEpisode: episodeVideo && metadata.imdbId || episodeVideo && metadata.kitsuId ? episodeVideo.episode || episodeVideo.episode : undefined,
+            kitsuEpisode: episodeVideo && metadata.imdbId || episodeVideo && metadata.kitsuId ? episodeVideo.episode || episodeVideo.episode : undefined,
+        }];
+    };
+
+    private decomposeEpisodes = async (torrent: IParsedTorrent, files: IFileAttributes[], metadata: IMetadataResponse = {episodeCount: []}): Promise<IFileAttributes[]> => {
+        if (files.every(file => !file.episodes && !file.date)) {
+            return files;
+        }
+
+        this.preprocessEpisodes(files);
+
+        if (torrent.type === TorrentType.Anime && torrent.kitsuId) {
+            if (this.needsCinemetaMetadataForAnime(files, metadata)) {
+                // In some cases anime could be resolved to wrong kitsuId
+                // because of imdb season naming/absolute per series naming/multiple seasons
+                // So in these cases we need to fetch cinemeta based metadata and decompose episodes using that
+                await this.updateToCinemetaMetadata(metadata);
+                if (files.some(file => Number.isInteger(file.season))) {
+                    // sometimes multi season anime torrents don't include season 1 naming
+                    files
+                        .filter(file => !Number.isInteger(file.season) && file.episodes)
+                        .forEach(file => file.season = 1);
+                }
+            } else {
+                // otherwise for anime type episodes are always absolute and for a single season
+                files
+                    .filter(file => file.episodes && file.season !== 0)
+                    .forEach(file => file.season = 1);
+                return files;
+            }
+        }
+
+        const sortedEpisodes = files
+            .map(file => !file.isMovie && file.episodes || [])
+            .reduce((a, b) => a.concat(b), [])
+            .sort((a, b) => a - b);
+
+        if (this.isConcatSeasonAndEpisodeFiles(files, sortedEpisodes, metadata)) {
+            this.decomposeConcatSeasonAndEpisodeFiles(files, metadata);
+        } else if (this.isDateEpisodeFiles(files, metadata)) {
+            this.decomposeDateEpisodeFiles(files, metadata);
+        } else if (this.isAbsoluteEpisodeFiles(torrent, files, metadata)) {
+            this.decomposeAbsoluteEpisodeFiles(torrent, files, metadata);
+        }
+        // decomposeEpisodeTitleFiles(torrent, files, metadata);
+
+        return files;
+    };
+
+    private preprocessEpisodes = (files: IFileAttributes[]): void => {
+        // reverse special episode naming when they named with 0 episode, ie. S02E00
+        files
+            .filter(file => Number.isInteger(file.season) && file.episode === 0)
+            .forEach(file => {
+                file.episode = file.season
+                file.episodes = [file.season || 0];
+                file.season = 0;
+            })
+    };
+
+    private isConcatSeasonAndEpisodeFiles = (files: IFileAttributes[], sortedEpisodes: number[], metadata: IMetadataResponse): boolean => {
+        if (metadata.kitsuId !== undefined) {
+            // anime does not use this naming scheme in 99% of cases;
+            return false;
+        }
+        // decompose concat season and episode files (ex. 101=S01E01) in case:
+        // 1. file has a season, but individual files are concatenated with that season (ex. path Season 5/511 - Prize
+        // Fighters.avi)
+        // 2. file does not have a season and the episode does not go out of range for the concat season
+        // episode count
+        const thresholdAbove = Math.max(Math.ceil(files.length * 0.05), 5);
+        const thresholdSorted = Math.max(Math.ceil(files.length * 0.8), 8);
+        const threshold = Math.max(Math.ceil(files.length * 0.8), 5);
+        const sortedConcatEpisodes = sortedEpisodes
+            .filter(ep => ep > 100)
+            .filter(ep => metadata.episodeCount && metadata.episodeCount[this.div100(ep) - 1] < ep)
+            .filter(ep => metadata.episodeCount && metadata.episodeCount[this.div100(ep) - 1] >= this.mod100(ep));
+        const concatFileEpisodes = files
+            .filter(file => !file.isMovie && file.episodes)
+            .filter(file => !file.season || file.episodes?.every(ep => this.div100(ep) === file.season));
+        const concatAboveTotalEpisodeCount = files
+            .filter(file => !file.isMovie && file.episodes && file.episodes.every(ep => ep > 100))
+            .filter(file => file.episodes?.every(ep => ep > metadata.totalCount!));
+        return sortedConcatEpisodes.length >= thresholdSorted && concatFileEpisodes.length >= threshold
+            || concatAboveTotalEpisodeCount.length >= thresholdAbove;
+    };
+
+    private isDateEpisodeFiles = (files: IFileAttributes[], metadata: IMetadataResponse): boolean => files.every(file => (!file.season || metadata.episodeCount && !metadata.episodeCount[file.season - 1]) && file.date);
+
+    private isAbsoluteEpisodeFiles = (torrent: IParsedTorrent, files: IFileAttributes[], metadata: IMetadataResponse): boolean => {
+        const threshold = Math.ceil(files.length / 5);
+        const isAnime = torrent.type === TorrentType.Anime && torrent.kitsuId;
+        const nonMovieEpisodes = files.filter(file => !file.isMovie && file.episodes);
+        const absoluteEpisodes = files
+            .filter(file => file.season && file.episodes)
+            .filter(file => file.episodes?.every(ep =>
+                metadata.episodeCount && file.season && metadata.episodeCount[file.season - 1] < ep));
+        return nonMovieEpisodes.every(file => !file.season)
+            || (isAnime && nonMovieEpisodes.every(file =>
+                metadata.episodeCount && file.season && file.season > metadata.episodeCount.length))
+            || absoluteEpisodes.length >= threshold;
+    };
+
+    private isNewEpisodeNotInMetadata = (torrent: IParsedTorrent, video: IFileAttributes, metadata: IMetadataResponse): boolean => {
+        const isAnime = torrent.type === TorrentType.Anime && torrent.kitsuId;
+        return !!(!isAnime && !video.isMovie && video.episodes && video.season !== 1
+            && metadata.status && /continuing|current/i.test(metadata.status)
+            && metadata.episodeCount && video.season && video.season >= metadata.episodeCount.length
+            && video.episodes.every(ep => metadata.episodeCount && video.season && ep > (metadata.episodeCount[video.season - 1] || 0)));
+    };
+
+    private decomposeConcatSeasonAndEpisodeFiles = (files: IFileAttributes[], metadata: IMetadataResponse): void => {
+        files
+            .filter(file => file.episodes && file.season !== 0 && file.episodes.every(ep => ep > 100))
+            .filter(file => file.episodes && metadata?.episodeCount &&
+                ((file.season || this.div100(file.episodes[0])) - 1) >= 0 &&
+                metadata.episodeCount[(file.season || this.div100(file.episodes[0])) - 1] < 100)
+            .filter(file => (file.season && file.episodes && file.episodes.every(ep => this.div100(ep) === file.season)) || !file.season)
+            .forEach(file => {
+                if (file.episodes) {
+                    file.season = this.div100(file.episodes[0]);
+                    file.episodes = file.episodes.map(ep => this.mod100(ep));
+                }
+            });
+    };
+
+    private decomposeAbsoluteEpisodeFiles = (torrent: IParsedTorrent, videos: IFileAttributes[], metadata: IMetadataResponse): void => {
+        if (metadata.episodeCount?.length === 0) {
+            videos
+                .filter(file => !Number.isInteger(file.season) && file.episodes && !file.isMovie)
+                .forEach(file => {
+                    file.season = 1;
+                });
+            return;
+        }
+        if (!metadata.episodeCount) return;
+
+        videos
+            .filter(file => file.episodes && !file.isMovie && file.season !== 0)
+            .filter(file => !this.isNewEpisodeNotInMetadata(torrent, file, metadata))
+            .filter(file => {
+                if (!file.episodes || !metadata.episodeCount) return false;
+                return !file.season || (metadata.episodeCount[file.season - 1] || 0) < file.episodes[0];
+            })
+            .forEach(file => {
+                if (!file.episodes || !metadata.episodeCount) return;
+
+                let seasonIdx = metadata.episodeCount
+                    .map((_, i) => i)
+                    .find(i => metadata.episodeCount && file.episodes && metadata.episodeCount.slice(0, i + 1).reduce((a, b) => a + b) >= file.episodes[0]);
+
+                seasonIdx = (seasonIdx || 1 || metadata.episodeCount.length) - 1;
+
+                file.season = seasonIdx + 1;
+                file.episodes = file.episodes
+                    .map(ep => ep - (metadata.episodeCount?.slice(0, seasonIdx).reduce((a, b) => a + b, 0) || 0));
+            });
+    };
+
+    private decomposeDateEpisodeFiles = (files: IFileAttributes[], metadata: IMetadataResponse): void => {
+        if (!metadata || !metadata.videos || !metadata.videos.length) {
+            return;
+        }
+
+        const timeZoneOffset = this.getTimeZoneOffset(metadata.country);
+        const offsetVideos: { [key: string]: ICommonVideoMetadata } = metadata.videos
+            .reduce((map: { [key: string]: ICommonVideoMetadata }, video: ICommonVideoMetadata) => {
+                const releaseDate = moment(video.released).utcOffset(timeZoneOffset).format('YYYY-MM-DD');
+                map[releaseDate] = video;
+                return map;
+            }, {});
+
+        files
+            .filter(file => file.date)
+            .forEach(file => {
+                const video = offsetVideos[file.date!];
+                if (video) {
+                    file.season = video.season;
+                    file.episodes = [video.episode || 0];
+                }
+            });
+    };
+
+    private getTimeZoneOffset = (country: string | undefined): string => {
+        switch (country) {
+            case 'United States':
+            case 'USA':
+                return '-08:00';
+            default:
+                return '00:00';
+        }
+    };
+
+    private assignKitsuOrImdbEpisodes = (torrent: IParsedTorrent, files: IFileAttributes[], metadata: IMetadataResponse): IFileAttributes[] => {
+        if (!metadata || !metadata.videos || !metadata.videos.length) {
+            if (torrent.type === TorrentType.Anime) {
+                // assign episodes as kitsu episodes for anime when no metadata available for imdb mapping
+                files
+                    .filter(file => file.season && file.episodes)
+                    .forEach(file => {
+                        file.season = undefined;
+                        file.episodes = undefined;
+                    })
+                if (metadata.type === TorrentType.Movie && files.every(file => !file.imdbId)) {
+                    // sometimes a movie has episode naming, thus not recognized as a movie and imdbId not assigned
+                    files.forEach(file => file.imdbId = metadata.imdbId?.toString());
+                }
+            }
+            return files;
+        }
+
+        const seriesMapping = metadata.videos
+            .filter(video => video.season !== undefined && Number.isInteger(video.season) && video.episode !== undefined && Number.isInteger(video.episode))
+            .reduce<SeasonEpisodeMap>((map, video) => {
+                if (video.season !== undefined && video.episode !== undefined) {
+                    const episodeMap = map[video.season] || {};
+                    episodeMap[video.episode] = video;
+                    map[video.season] = episodeMap;
+                }
+                return map;
+            }, {});
+
+
+        if (metadata.videos.some(video => Number.isInteger(video.season)) || !metadata.imdbId) {
+            files.filter(file => file && Number.isInteger(file.season) && file.episodes)
+                .map(file => {
+                    const seasonMapping = file && file.season && seriesMapping[file.season] || null;
+                    const episodeMapping = seasonMapping && file && file.episodes && file.episodes[0] && seasonMapping[file.episodes[0]] || null;
+
+                    if (episodeMapping && Number.isInteger(episodeMapping.season)) {
+                        file.imdbId = metadata.imdbId?.toString();
+                        file.season = episodeMapping.season;
+                        file.episodes = file.episodes && file.episodes.map(ep => (seasonMapping && seasonMapping[ep]) ? Number(seasonMapping[ep].episode) : 0);
+                    } else {
+                        file.season = undefined;
+                        file.episodes = undefined;
+                    }
+                });
+        } else if (metadata.videos.some(video => video.episode)) {
+            // imdb episode info is base
+            files
+                .filter(file => Number.isInteger(file.season) && file.episodes)
+                .forEach(file => {
+                    if (!file.season || !file.episodes) {
+                        return;
+                    }
+                    if (seriesMapping[file.season]) {
+
+                        const seasonMapping = seriesMapping[file.season];
+                        file.imdbId = metadata.imdbId?.toString();
+                        file.kitsuId = seasonMapping[file.episodes[0]] && parseInt(seasonMapping[file.episodes[0]].id || '0') || 0;
+                        file.episodes = file.episodes.map(ep => seasonMapping[ep]?.episode)
+                            .filter((ep): ep is number => ep !== undefined);
+                    } else if (seriesMapping[file.season - 1]) {
+                        // sometimes a second season might be a continuation of the previous season
+                        const seasonMapping = seriesMapping[file.season - 1] as ICommonVideoMetadata;
+                        const episodes = Object.values(seasonMapping);
+                        const firstKitsuId = episodes.length && episodes[0];
+                        const differentTitlesCount = new Set(episodes.map(ep => ep.id)).size
+                        const skippedCount = episodes.filter(ep => ep.id === firstKitsuId).length;
+                        const emptyArray: number[] = [];
+                        const seasonEpisodes = files
+                            .filter((otherFile: IFileAttributes) => otherFile.season === file.season && otherFile.episodes)
+                            .reduce((a, b) => a.concat(b.episodes || []), emptyArray);
+                        const isAbsoluteOrder = seasonEpisodes.every(ep => ep > skippedCount && ep <= episodes.length)
+                        const isNormalOrder = seasonEpisodes.every(ep => ep + skippedCount <= episodes.length)
+                        if (differentTitlesCount >= 1 && (isAbsoluteOrder || isNormalOrder)) {
+                            const {season} = file;
+                            const [episode] = file.episodes;
+                            file.imdbId = metadata.imdbId?.toString();
+                            file.season = file.season - 1;
+                            file.episodes = file.episodes.map(ep => isAbsoluteOrder ? ep : ep + skippedCount);
+                            const currentEpisode = seriesMapping[season][episode];
+                            file.kitsuId = currentEpisode ? parseInt(currentEpisode.id || '0') : 0;
+                            if (typeof season === 'number' && Array.isArray(file.episodes)) {
+                                file.episodes = file.episodes.map(ep =>
+                                    seriesMapping[season]
+                                    && seriesMapping[season][ep]
+                                    && seriesMapping[season][ep].episode
+                                    || ep);
+                            }
+                        }
+                    } else if (Object.values(seriesMapping).length === 1 && seriesMapping[1]) {
+                        // sometimes series might be named with sequel season but it's not a season on imdb and a new title
+                        // eslint-disable-next-line prefer-destructuring
+                        const seasonMapping = seriesMapping[1];
+                        file.imdbId = metadata.imdbId?.toString();
+                        file.season = 1;
+                        file.kitsuId = parseInt(seasonMapping[file.episodes[0]].id || '0') || 0;
+                        file.episodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].episode)
+                            .filter((ep): ep is number => ep !== undefined);
+                    }
+                });
+        }
+        return files;
+    };
+
+    private needsCinemetaMetadataForAnime = (files: IFileAttributes[], metadata: IMetadataResponse): boolean => {
+        if (!metadata || !metadata.imdbId || !metadata.videos || !metadata.videos.length) {
+            return false;
+        }
+
+        const seasons = metadata.videos
+            .map(video => video.season)
+            .filter((season): season is number => season !== null && season !== undefined);
+
+        // Using || 0 instead of || Number.MAX_VALUE to match previous logic
+        const minSeason = Math.min(...seasons) || 0;
+        const maxSeason = Math.max(...seasons) || 0;
+        const differentSeasons = new Set(seasons.filter(season => Number.isInteger(season))).size;
+
+        const total = metadata.totalCount || Number.MAX_VALUE;
+
+        return differentSeasons > 1 || files
+            .filter(file => !file.isMovie && file.episodes)
+            .some(file => file.season || 0 < minSeason || file.season || 0 > maxSeason || file.episodes?.every(ep => ep > total));
+    };
+
+    private updateToCinemetaMetadata = async (metadata: IMetadataResponse): Promise<IMetadataResponse> => {
+        const query: IMetaDataQuery = {
+            id: metadata.imdbId,
+            type: metadata.type
+        };
+
+        return await this.metadataService.getMetadata(query)
+            .then((newMetadataOrError) => {
+                if (newMetadataOrError instanceof Error) {
+                    // handle error
+                    this.logger.warn(`Failed ${metadata.imdbId} metadata cinemeta update due: ${newMetadataOrError.message}`);
+                    return metadata;    // or throw newMetadataOrError to propagate error up the call stack
+                }
+                // At this point TypeScript infers newMetadataOrError to be of type MetadataResponse
+                const newMetadata = newMetadataOrError;
+                if (!newMetadata.videos || !newMetadata.videos.length) {
+                    return metadata;
+                } else {
+                    metadata.videos = newMetadata.videos;
+                    metadata.episodeCount = newMetadata.episodeCount;
+                    metadata.totalCount = newMetadata.totalCount;
+                    return metadata;
+                }
+            })
+    };
+
+    private findMovieImdbId = (title: IFileAttributes | string): Promise<string | undefined> => {
+        const parsedTitle = typeof title === 'string' ? parse(title) : title;
+        this.logger.debug(`Finding movie imdbId for ${title}`);
+        return this.imdb_limiter.schedule(async () => {
+            const imdbQuery = {
+                title: parsedTitle.title,
+                year: parsedTitle.year,
+                type: TorrentType.Movie
+            };
+            try {
+                return await this.metadataService.getImdbId(imdbQuery);
+            } catch (e) {
+                return undefined;
+            }
+        });
+    };
+
+    private findMovieKitsuId = async (title: IFileAttributes | string): Promise<number | Error | undefined> => {
+        const parsedTitle = typeof title === 'string' ? parse(title) : title;
+        const kitsuQuery = {
+            title: parsedTitle.title,
+            year: parsedTitle.year,
+            season: parsedTitle.season,
+            type: TorrentType.Movie
+        };
+        try {
+            return await this.metadataService.getKitsuId(kitsuQuery);
+        } catch (e) {
+            return undefined;
+        }
+    };
+
+    private isDiskTorrent = (contents: IContentAttributes[]): boolean => contents.some(content => ExtensionHelpers.isDisk(content.path));
+
+    private isSingleMovie = (videos: IFileAttributes[]): boolean => videos.length === 1 ||
+        (videos.length === 2 &&
+            videos.find(v => /\b(?:part|disc|cd)[ ._-]?0?1\b|^0?1\.\w{2,4}$/i.test(v.path!)) &&
+            videos.find(v => /\b(?:part|disc|cd)[ ._-]?0?2\b|^0?2\.\w{2,4}$/i.test(v.path!))) !== undefined;
+
+    private isFeaturette = (video: IFileAttributes): boolean => /featurettes?\/|extras-grym/i.test(video.path!);
+
+    private parseSeriesVideo = (video: IFileAttributes): IFileAttributes => {
+        const videoInfo = parse(video.title);
+        // the episode may be in a folder containing season number
+        if (!Number.isInteger(videoInfo.season) && video.path?.includes('/')) {
+            const folders = video.path?.split('/');
+            const pathInfo = parse(folders[folders.length - 2]);
+            videoInfo.season = pathInfo.season;
+        }
+        if (!Number.isInteger(videoInfo.season) && video.season) {
+            videoInfo.season = video.season;
+        }
+        if (!Number.isInteger(videoInfo.season) && videoInfo.seasons && videoInfo.seasons.length > 1) {
+            // in case single file was interpreted as having multiple seasons
+            [videoInfo.season] = videoInfo.seasons;
+        }
+        if (!Number.isInteger(videoInfo.season) && video.path?.includes('/') && video.seasons
+            && video.seasons.length > 1) {
+            // russian season are usually named with 'series name-2` i.e. Улицы разбитых фонарей-6/22. Одиночный выстрел.mkv
+            const folderPathSeasonMatch = video.path?.match(/[\u0400-\u04ff]-(\d{1,2})(?=.*\/)/);
+            videoInfo.season = folderPathSeasonMatch && parseInt(folderPathSeasonMatch[1], 10) || undefined;
+        }
+        // sometimes video file does not have correct date format as in torrent title
+        if (!videoInfo.episodes && !videoInfo.date && video.date) {
+            videoInfo.date = video.date;
+        }
+        // limit number of episodes in case of incorrect parsing
+        if (videoInfo.episodes && videoInfo.episodes.length > 20) {
+            videoInfo.episodes = [videoInfo.episodes[0]];
+            [videoInfo.episode] = videoInfo.episodes;
+        }
+        // force episode to any found number if it was not parsed
+        if (!videoInfo.episodes && !videoInfo.date) {
+            const epMatcher = videoInfo.title.match(
+                /(?<!season\W*|disk\W*|movie\W*|film\W*)(?:^|\W|_)(\d{1,4})(?:a|b|c|v\d)?(?:_|\W|$)(?!disk|movie|film)/i);
+            videoInfo.episodes = epMatcher && [parseInt(epMatcher[1], 10)] || undefined;
+            videoInfo.episode = videoInfo.episodes && videoInfo.episodes[0];
+        }
+        if (!videoInfo.episodes && !videoInfo.date) {
+            const epMatcher = video.title.match(new RegExp(`(?:\\(${videoInfo.year}\\)|part)[._ ]?(\\d{1,3})(?:\\b|_)`, "i"));
+            videoInfo.episodes = epMatcher && [parseInt(epMatcher[1], 10)] || undefined;
+            videoInfo.episode = videoInfo.episodes && videoInfo.episodes[0];
+        }
+
+        return {...video, ...videoInfo};
+    };
+
+    private isMovieVideo = (torrent: IParsedTorrent, video: IFileAttributes, otherVideos: IFileAttributes[], hasMovies: boolean): boolean => {
+        if (Number.isInteger(torrent.season) && Array.isArray(torrent.episodes)) {
+            // not movie if video has season
+            return false;
+        }
+        if (torrent.title?.match(/\b(?:\d+[ .]movie|movie[ .]\d+)\b/i)) {
+            // movie if video explicitly has numbered movie keyword in the name, ie. 1 Movie or Movie 1
+            return true;
+        }
+        if (!hasMovies && torrent.type !== TorrentType.Anime) {
+            // not movie if torrent name does not contain movies keyword or is not a pack torrent and is not anime
+            return false;
+        }
+        if (!torrent.episodes) {
+            // movie if there's no episode info it could be a movie
+            return true;
+        }
+        // movie if contains year info and there aren't more than 3 video with same title and year
+        // as some series titles might contain year in it.
+        return !!torrent.year
+            && otherVideos.length > 3
+            && otherVideos.filter(other => other.title === video.title && other.year === video.year).length < 3;
+    };
+
+    private clearInfoFields = (video: IFileAttributes): IFileAttributes => {
+        video.imdbId = undefined;
+        video.imdbSeason = undefined;
+        video.imdbEpisode = undefined;
+        video.kitsuId = undefined;
+        video.kitsuEpisode = undefined;
+        return video;
+    };
+
+    private div100 = (episode: number): number => (episode / 100 >> 0);
+
+    private mod100 = (episode: number): number => episode % 100;
+}

src/node/consumer/src/lib/services/torrent_processing_service.ts

@@ -0,0 +1,59 @@
+import {TorrentType} from "@enums/torrent_types";
+import {ILoggingService} from "@interfaces/logging_service";
+import {IParsedTorrent} from "@interfaces/parsed_torrent";
+import {ITorrentEntriesService} from "@interfaces/torrent_entries_service";
+import {ITorrentProcessingService} from "@interfaces/torrent_processing_service";
+import {ITrackerService} from "@interfaces/tracker_service";
+import {IIngestedTorrentAttributes} from "@repository/interfaces/ingested_torrent_attributes";
+import {IocTypes} from "@setup/ioc_types";
+import {inject, injectable} from "inversify";
+
+@injectable()
+export class TorrentProcessingService implements ITorrentProcessingService {
+    @inject(IocTypes.ITorrentEntriesService) torrentEntriesService: ITorrentEntriesService;
+    @inject(IocTypes.ILoggingService) logger: ILoggingService;
+    @inject(IocTypes.ITrackerService) trackerService: ITrackerService;
+
+    async processTorrentRecord(torrent: IIngestedTorrentAttributes): Promise<void> {
+        const {category} = torrent;
+        const type = category === 'tv' ? TorrentType.Series : TorrentType.Movie;
+        const torrentInfo: IParsedTorrent = await this.parseTorrent(torrent, type);
+
+        this.logger.info(`Processing torrent ${torrentInfo.title} with infoHash ${torrentInfo.infoHash}`);
+
+        if (await this.torrentEntriesService.checkAndUpdateTorrent(torrentInfo)) {
+            return;
+        }
+
+        return this.torrentEntriesService.createTorrentEntry(torrentInfo, false);
+    }
+
+    private assignTorrentTrackers = async (): Promise<string> => {
+        const trackers = await this.trackerService.getTrackers();
+        return trackers.join(',');
+    }
+
+    private parseTorrent = async (torrent: IIngestedTorrentAttributes, category: string): Promise<IParsedTorrent> => {
+        const infoHash = torrent.info_hash?.trim().toLowerCase()
+        return {
+            title: torrent.name,
+            torrentId: `${torrent.name}_${infoHash}`,
+            infoHash: infoHash,
+            seeders: 100,
+            size: parseInt(torrent.size),
+            uploadDate: torrent.createdAt,
+            imdbId: this.parseImdbId(torrent),
+            type: category,
+            provider: torrent.source,
+            trackers: await this.assignTorrentTrackers(),
+        }
+    };
+
+    private parseImdbId = (torrent: IIngestedTorrentAttributes): string | undefined => {
+        if (torrent.imdb === undefined || torrent.imdb === null) {
+            return undefined;
+        }
+
+        return torrent.imdb;
+    };
+}

src/node/consumer/src/lib/services/torrent_subtitle_service.ts

@@ -0,0 +1,107 @@
+import {ITorrentFileCollection} from "@interfaces/torrent_file_collection";
+import {ITorrentSubtitleService} from "@interfaces/torrent_subtitle_service";
+import {IFileAttributes} from "@repository/interfaces/file_attributes";
+import {ISubtitleAttributes} from "@repository/interfaces/subtitle_attributes";
+import {injectable} from "inversify";
+import {parse} from 'parse-torrent-title';
+
+@injectable()
+export class TorrentSubtitleService implements ITorrentSubtitleService {
+    assignSubtitles(fileCollection: ITorrentFileCollection): ITorrentFileCollection {
+        if (fileCollection.videos && fileCollection.videos.length && fileCollection.subtitles && fileCollection.subtitles.length) {
+            if (fileCollection.videos.length === 1) {
+                const matchingSubtitles = fileCollection.subtitles.filter(subtitle =>
+                    this.mostProbableSubtitleVideos(subtitle, [fileCollection.videos[0]]).length > 0
+                );
+                fileCollection.videos[0].subtitles = matchingSubtitles;
+                const nonMatchingSubtitles = fileCollection.subtitles.filter(subtitle =>
+                    !matchingSubtitles.includes(subtitle)
+                );
+                return {...fileCollection, subtitles: nonMatchingSubtitles};
+            }
+
+            const parsedVideos = fileCollection.videos.map(video => this.parseVideo(video));
+            const assignedSubs = fileCollection.subtitles.map(subtitle => ({
+                subtitle,
+                videos: this.mostProbableSubtitleVideos(subtitle, parsedVideos)
+            }));
+            const unassignedSubs = assignedSubs.filter(assignedSub => !assignedSub.videos).map(assignedSub => assignedSub.subtitle);
+
+            assignedSubs
+                .filter(assignedSub => assignedSub.videos)
+                .forEach(assignedSub => assignedSub.videos.forEach(video => video.subtitles = (video.subtitles || []).concat(assignedSub.subtitle)));
+            return {...fileCollection, subtitles: unassignedSubs};
+        }
+        return fileCollection;
+    }
+
+    private parseVideo = (video: IFileAttributes): IFileAttributes => {
+        const fileName = video.title?.split('/')?.pop()?.replace(/\.(\w{2,4})$/, '') || '';
+        const folderName = video.title?.replace(/\/?[^/]+$/, '') || '';
+        return Object.assign(video, {
+            fileName: fileName,
+            folderName: folderName,
+            ...this.parseFilename(video.title.toString() || '')
+        });
+    }
+
+    private mostProbableSubtitleVideos = (subtitle: ISubtitleAttributes, parsedVideos: IFileAttributes[]): IFileAttributes[] => {
+        const subTitle = (subtitle.title || subtitle.path)?.split('/')?.pop()?.replace(/\.(\w{2,4})$/, '') || '';
+        const parsedSub = this.parsePath(subtitle.title || subtitle.path);
+        const byFileName = parsedVideos.filter(video => subTitle.includes(video.title!));
+        if (byFileName.length === 1) {
+            return byFileName.map(v => v);
+        }
+        const byTitleSeasonEpisode = parsedVideos.filter(video => video.title === parsedSub.title
+            && parsedSub.seasons && parsedSub.episodes
+            && this.arrayEquals(video.seasons || [], parsedSub.seasons)
+            && this.arrayEquals(video.episodes || [], parsedSub.episodes));
+        if (this.singleVideoFile(byTitleSeasonEpisode)) {
+            return byTitleSeasonEpisode.map(v => v);
+        }
+        const bySeasonEpisode = parsedVideos.filter(video => parsedSub.seasons && parsedSub.episodes
+            && this.arrayEquals(video.seasons || [], parsedSub.seasons)
+            && this.arrayEquals(video.episodes || [], parsedSub.episodes));
+        if (this.singleVideoFile(bySeasonEpisode)) {
+            return bySeasonEpisode.map(v => v);
+        }
+        const byTitle = parsedVideos.filter(video => video.title && video.title === parsedSub.title);
+        if (this.singleVideoFile(byTitle)) {
+            return byTitle.map(v => v);
+        }
+        const byEpisode = parsedVideos.filter(video => parsedSub.episodes
+            && this.arrayEquals(video.episodes || [], parsedSub.episodes || []));
+        if (this.singleVideoFile(byEpisode)) {
+            return byEpisode.map(v => v);
+        }
+        const byInfoHash = parsedVideos.filter(video => video.infoHash === subtitle.infoHash);
+        if (this.singleVideoFile(byInfoHash)) {
+            return byInfoHash.map(v => v);
+        }
+        return [];
+    }
+
+    private singleVideoFile = (videos: IFileAttributes[]): boolean => {
+        return new Set(videos.map(v => v.fileIndex)).size === 1;
+    }
+
+    private parsePath = (path: string): IFileAttributes => {
+        const pathParts = path.split('/').map(part => this.parseFilename(part));
+        const parsedWithEpisode = pathParts.find(parsed => parsed.season && parsed.episodes);
+        return parsedWithEpisode || pathParts[pathParts.length - 1];
+    }
+
+    private parseFilename = (filename: string): IFileAttributes => {
+        const parsedInfo = parse(filename)
+        const titleEpisode = parsedInfo.title.match(/(\d+)$/);
+        if (!parsedInfo.episodes && titleEpisode) {
+            parsedInfo.episodes = [parseInt(titleEpisode[1], 10)];
+        }
+        return parsedInfo;
+    }
+
+    private arrayEquals = <T>(array1: T[], array2: T[]): boolean => {
+        if (!array1 || !array2) return array1 === array2;
+        return array1.length === array2.length && array1.every((value, index) => value === array2[index])
+    }
+}

src/node/consumer/src/lib/services/tracker_service.ts

@@ -0,0 +1,36 @@
+import {ICacheService} from "@interfaces/cache_service";
+import {ILoggingService} from "@interfaces/logging_service";
+import {ITrackerService} from "@interfaces/tracker_service";
+import {configurationService} from '@services/configuration_service';
+import {IocTypes} from "@setup/ioc_types";
+import axios, {AxiosResponse} from 'axios';
+import {inject, injectable} from "inversify";
+
+@injectable()
+export class TrackerService implements ITrackerService {
+    @inject(IocTypes.ICacheService) cacheService: ICacheService;
+    @inject(IocTypes.ILoggingService) logger: ILoggingService;
+
+    async getTrackers(): Promise<string[]> {
+        return this.cacheService.cacheTrackers(this.downloadTrackers);
+    }
+
+    private downloadTrackers = async (): Promise<string[]> => {
+        const response: AxiosResponse<string> = await axios.get(configurationService.trackerConfig.TRACKERS_URL);
+        const trackersListText: string = response.data;
+        // Trackers are separated by a newline character
+        let urlTrackers = trackersListText.split("\n");
+        // remove blank lines
+        urlTrackers = urlTrackers.filter(line => line.trim() !== '');
+
+        if (!configurationService.trackerConfig.UDP_ENABLED) {
+            // remove any udp trackers
+            urlTrackers = urlTrackers.filter(line => !line.startsWith('udp://'));
+
+        }
+
+        this.logger.info(`Trackers updated at ${Date.now()}: ${urlTrackers.length} trackers`);
+
+        return urlTrackers;
+    };
+}

src/node/consumer/src/lib/torrent.js

@@ -1,82 +0,0 @@
-import { decode } from 'magnet-uri';
-import torrentStream from 'torrent-stream';
-import { torrentConfig } from './config.js';
-import {isSubtitle, isVideo} from './extension.js';
-
-export async function torrentFiles(torrent, timeout) {
-  return filesFromTorrentStream(torrent, timeout)
-      .then(files => ({
-        contents: files,
-        videos: filterVideos(files),
-        subtitles: filterSubtitles(files)
-      }));
-}
-
-async function filesFromTorrentStream(torrent, timeout) {
-  return filesAndSizeFromTorrentStream(torrent, timeout).then(result => result.files);
-}
-
-const engineOptions = {
-  connections: torrentConfig.MAX_CONNECTIONS_PER_TORRENT,
-  uploads: 0,
-  verify: false,
-  dht: false,
-  tracker: true
-}
-
-function filesAndSizeFromTorrentStream(torrent, timeout = 30000) {
-  if (!torrent.infoHash) {
-    return Promise.reject(new Error("no infoHash..."));
-  }
-  const magnet = decode.encode({ infoHash: torrent.infoHash, announce: torrent.trackers });
-  return new Promise((resolve, rejected) => {
-    const timeoutId = setTimeout(() => {
-      engine.destroy();
-      rejected(new Error('No available connections for torrent!'));
-    }, timeout);
-
-    const engine = new torrentStream(magnet, engineOptions);
-
-    engine.ready(() => {
-      const files = engine.files
-          .map((file, fileId) => ({
-            fileIndex: fileId,
-            name: file.name,
-            path: file.path.replace(/^[^/]+\//, ''),
-            size: file.length
-          }));
-      const size = engine.torrent.length;
-      resolve({ files, size });
-      engine.destroy();
-      clearTimeout(timeoutId);
-    });
-  });
-}
-
-function filterVideos(files) {
-  if (files.length === 1 && !Number.isInteger(files[0].fileIndex)) {
-    return files;
-  }
-  const videos = files.filter(file => isVideo(file.path));
-  const maxSize = Math.max(...videos.map(video => video.size));
-  const minSampleRatio = videos.length <= 3 ? 3 : 10;
-  const minAnimeExtraRatio = 5;
-  const minRedundantRatio = videos.length <= 3 ? 30 : Number.MAX_VALUE;
-  const isSample = video => video.path.match(/sample|bonus|promo/i) && maxSize / parseInt(video.size) > minSampleRatio;
-  const isRedundant = video => maxSize / parseInt(video.size) > minRedundantRatio;
-  const isExtra = video => video.path.match(/extras?\//i);
-  const isAnimeExtra = video => video.path.match(/(?:\b|_)(?:NC)?(?:ED|OP|PV)(?:v?\d\d?)?(?:\b|_)/i)
-      && maxSize / parseInt(video.size) > minAnimeExtraRatio;
-  const isWatermark = video => video.path.match(/^[A-Z-]+(?:\.[A-Z]+)?\.\w{3,4}$/)
-      && maxSize / parseInt(video.size) > minAnimeExtraRatio
- return videos
-      .filter(video => !isSample(video))
-      .filter(video => !isExtra(video))
-      .filter(video => !isAnimeExtra(video))
-      .filter(video => !isRedundant(video))
-      .filter(video => !isWatermark(video));
-}
-
-function filterSubtitles(files) {
-  return files.filter(file => isSubtitle(file.path));
-}

src/node/consumer/src/lib/torrentEntries.js

@@ -1,173 +0,0 @@
-import { parse } from 'parse-torrent-title';
-import { getImdbId, getKitsuId } from './metadata.js';
-import { isPackTorrent } from './parseHelper.js';
-import * as Promises from './promises.js';
-import * as repository from './repository.js';
-import { parseTorrentFiles } from './torrentFiles.js';
-import { assignSubtitles } from './torrentSubtitles.js';
-import { TorrentType } from './types.js';
-import {logger} from "./logger.js";
-
-export async function createTorrentEntry(torrent, overwrite = false) {
-  const titleInfo = parse(torrent.title);
-    
-  if (!torrent.imdbId && torrent.type !== TorrentType.ANIME) {
-    torrent.imdbId = await getImdbId(titleInfo, torrent.type)
-        .catch(() => undefined);
-  }
-  if (torrent.imdbId && torrent.imdbId.length < 9) {
-    // pad zeros to imdbId if missing
-    torrent.imdbId = 'tt' + torrent.imdbId.replace('tt', '').padStart(7, '0');
-  }
-  if (torrent.imdbId && torrent.imdbId.length > 9 && torrent.imdbId.startsWith('tt0')) {
-    // sanitize imdbId from redundant zeros
-    torrent.imdbId = torrent.imdbId.replace(/tt0+([0-9]{7,})$/, 'tt$1');
-  }
-  if (!torrent.kitsuId && torrent.type === TorrentType.ANIME) {
-    torrent.kitsuId = await getKitsuId(titleInfo)
-        .catch(() => undefined);
-  }
-
-  if (!torrent.imdbId && !torrent.kitsuId && !isPackTorrent(torrent)) {
-    logger.warn(`imdbId or kitsuId not found:  ${torrent.provider} ${torrent.title}`);
-    return;
-  }
-
-  const { contents, videos, subtitles } = await parseTorrentFiles(torrent)
-      .then(torrentContents => overwrite ? overwriteExistingFiles(torrent, torrentContents) : torrentContents)
-      .then(torrentContents => assignSubtitles(torrentContents))
-      .catch(error => {
-          logger.warn(`Failed getting files for ${torrent.title}`, error.message);
-        return {};
-      });
-  if (!videos || !videos.length) {
-      logger.warn(`no video files found for ${torrent.provider} [${torrent.infoHash}] ${torrent.title}`);
-    return;
-  }
-
-  return repository.createTorrent({ ...torrent, contents, subtitles })
-      .then(() => Promises.sequence(videos.map(video => () => repository.createFile(video))))
-      .then(() => logger.info(`Created ${torrent.provider} entry for [${torrent.infoHash}] ${torrent.title}`));
-}
-
-async function overwriteExistingFiles(torrent, torrentContents) {
-  const videos = torrentContents && torrentContents.videos;
-  if (videos && videos.length) {
-    const existingFiles = await repository.getFiles({ infoHash: videos[0].infoHash })
-        .then((existing) => existing
-            .reduce((map, next) => {
-              const fileIndex = next.fileIndex !== undefined ? next.fileIndex : null;
-              map[fileIndex] = (map[fileIndex] || []).concat(next);
-              return map;
-            }, {}))
-        .catch(() => undefined);
-    if (existingFiles && Object.keys(existingFiles).length) {
-      const overwrittenVideos = videos
-          .map(file => {
-            const mapping = videos.length === 1 && Object.keys(existingFiles).length === 1
-                ? Object.values(existingFiles)[0]
-                : existingFiles[file.fileIndex !== undefined ? file.fileIndex : null];
-            if (mapping) {
-              const originalFile = mapping.shift();
-              return { id: originalFile.id, ...file };
-            }
-            return file;
-          });
-      return { ...torrentContents, videos: overwrittenVideos };
-    }
-    return torrentContents;
-  }
-  return Promise.reject(`No video files found for: ${torrent.title}`);
-}
-
-export async function createSkipTorrentEntry(torrent) {
-  return repository.createSkipTorrent(torrent);
-}
-
-export async function getStoredTorrentEntry(torrent) {
-  return repository.getSkipTorrent(torrent)
-      .catch(() => repository.getTorrent(torrent))
-      .catch(() => undefined);
-}
-
-export async function checkAndUpdateTorrent(torrent) {
-  const storedTorrent = torrent.dataValues
-      ? torrent
-      : await repository.getTorrent(torrent).catch(() => undefined);
-  if (!storedTorrent) {
-    return false;
-  }
-  if (storedTorrent.provider === 'RARBG') {
-    return true;
-  }
-  if (storedTorrent.provider === 'KickassTorrents' && torrent.provider) {
-    storedTorrent.provider = torrent.provider;
-    storedTorrent.torrentId = torrent.torrentId;
-  }
-  if (!storedTorrent.languages && torrent.languages && storedTorrent.provider !== 'RARBG') {
-    storedTorrent.languages = torrent.languages;
-    await storedTorrent.save();
-    logger.debug(`Updated [${storedTorrent.infoHash}] ${storedTorrent.title} language to ${torrent.languages}`);
-  }
-  return createTorrentContents({ ...storedTorrent.get(), torrentLink: torrent.torrentLink })
-      .then(() => updateTorrentSeeders(torrent));
-}
-
-export async function createTorrentContents(torrent) {
-  if (torrent.opened) {
-    return;
-  }
-  const storedVideos = await repository.getFiles(torrent).catch(() => []);
-  if (!storedVideos || !storedVideos.length) {
-    return;
-  }
-  const notOpenedVideo = storedVideos.length === 1 && !Number.isInteger(storedVideos[0].fileIndex);
-  const imdbId = Promises.mostCommonValue(storedVideos.map(stored => stored.imdbId));
-  const kitsuId = Promises.mostCommonValue(storedVideos.map(stored => stored.kitsuId));
-
-  const { contents, videos, subtitles } = await parseTorrentFiles({ ...torrent, imdbId, kitsuId })
-      .then(torrentContents => notOpenedVideo ? torrentContents : { ...torrentContents, videos: storedVideos })
-      .then(torrentContents => assignSubtitles(torrentContents))
-      .catch(error => {
-        logger.warn(`Failed getting contents for [${torrent.infoHash}] ${torrent.title}`, error.message);
-        return {};
-      });
-
-  if (!contents || !contents.length) {
-    return;
-  }
-  if (notOpenedVideo && videos.length === 1) {
-    // if both have a single video and stored one was not opened, update stored one to true metadata and use that
-    storedVideos[0].fileIndex = videos[0].fileIndex;
-    storedVideos[0].title = videos[0].title;
-    storedVideos[0].size = videos[0].size;
-    storedVideos[0].subtitles = videos[0].subtitles;
-    videos[0] = storedVideos[0];
-  }
-  // no videos available or more than one new videos were in the torrent
-  const shouldDeleteOld = notOpenedVideo && videos.every(video => !video.id);
-
-  return repository.createTorrent({ ...torrent, contents, subtitles })
-      .then(() => {
-        if (shouldDeleteOld) {
-          logger.debug(`Deleting old video for [${torrent.infoHash}] ${torrent.title}`)
-          return storedVideos[0].destroy();
-        }
-        return Promise.resolve();
-      })
-      .then(() => Promises.sequence(videos.map(video => () => repository.createFile(video))))
-      .then(() => logger.info(`Created contents for ${torrent.provider} [${torrent.infoHash}] ${torrent.title}`))
-      .catch(error => logger.error(`Failed saving contents for [${torrent.infoHash}] ${torrent.title}`, error));
-}
-
-export async function updateTorrentSeeders(torrent) {
-  if (!(torrent.infoHash || (torrent.provider && torrent.torrentId)) || !Number.isInteger(torrent.seeders)) {
-    return torrent;
-  }
-
-  return repository.setTorrentSeeders(torrent, torrent.seeders)
-      .catch(error => {
-        logger.warn('Failed updating seeders:', error);
-        return undefined;
-      });
-}

src/node/consumer/src/lib/torrentFiles.js

@@ -1,513 +0,0 @@
-import Bottleneck from 'bottleneck';
-import distance from 'jaro-winkler';
-import moment from 'moment';
-import { parse } from 'parse-torrent-title';
-import { metadataConfig } from './config.js';
-import { isDisk } from './extension.js';
-import { getMetadata, getImdbId, getKitsuId } from './metadata.js';
-import { parseSeriesVideos, isPackTorrent } from './parseHelper.js';
-import * as Promises from './promises.js';
-import {torrentFiles} from "./torrent.js";
-import { TorrentType } from './types.js';
-import {logger} from "./logger.js";
-
-const MIN_SIZE = 5 * 1024 * 1024; // 5 MB
-const imdb_limiter = new Bottleneck({ maxConcurrent: metadataConfig.IMDB_CONCURRENT, minTime: metadataConfig.IMDB_INTERVAL_MS });
-
-export async function parseTorrentFiles(torrent) {
-  const parsedTorrentName = parse(torrent.title);
-  const metadata = await getMetadata(torrent.kitsuId || torrent.imdbId, torrent.type || TorrentType.MOVIE)
-      .then(meta => Object.assign({}, meta))
-      .catch(() => undefined);
-
-  // if (metadata && metadata.type !== torrent.type && torrent.type !== Type.ANIME) {
-  //   throw new Error(`Mismatching entry type for ${torrent.name}: ${torrent.type}!=${metadata.type}`);
-  // }
-  if (torrent.type !== TorrentType.ANIME && metadata && metadata.type && metadata.type !== torrent.type) {
-    // it's actually a movie/series
-    torrent.type = metadata.type;
-  }
-
-  if (torrent.type === TorrentType.MOVIE && (!parsedTorrentName.seasons ||
-      parsedTorrentName.season === 5 && [1, 5].includes(parsedTorrentName.episode))) {
-    return parseMovieFiles(torrent, parsedTorrentName, metadata);
-  }
-
-  return parseSeriesFiles(torrent, parsedTorrentName, metadata)
-}
-
-async function parseMovieFiles(torrent, parsedName, metadata) {
-  const { contents, videos, subtitles } = await getMoviesTorrentContent(torrent);
-  const filteredVideos = videos
-      .filter(video => video.size > MIN_SIZE)
-      .filter(video => !isFeaturette(video));
-  if (isSingleMovie(filteredVideos)) {
-    const parsedVideos = filteredVideos.map(video => ({
-      infoHash: torrent.infoHash,
-      fileIndex: video.fileIndex,
-      title: video.path || torrent.title,
-      size: video.size || torrent.size,
-      imdbId: torrent.imdbId || metadata && metadata.imdbId,
-      kitsuId: torrent.kitsuId || metadata && metadata.kitsuId
-    }));
-    return { contents, videos: parsedVideos, subtitles };
-  }
-
-  const parsedVideos = await Promises.sequence(filteredVideos.map(video => () => isFeaturette(video)
-          ? Promise.resolve(video)
-          : findMovieImdbId(video.name).then(imdbId => ({ ...video, imdbId }))))
-      .then(videos => videos.map(video => ({
-        infoHash: torrent.infoHash,
-        fileIndex: video.fileIndex,
-        title: video.path || video.name,
-        size: video.size,
-        imdbId: video.imdbId,
-      })));
-  return { contents, videos: parsedVideos, subtitles };
-}
-
-async function parseSeriesFiles(torrent, parsedName, metadata) {
-  const { contents, videos, subtitles } = await getSeriesTorrentContent(torrent);
-  const parsedVideos = await Promise.resolve(videos)
-      .then(videos => videos.filter(video => videos.length === 1 || video.size > MIN_SIZE))
-      .then(videos => parseSeriesVideos(torrent, videos))
-      .then(videos => decomposeEpisodes(torrent, videos, metadata))
-      .then(videos => assignKitsuOrImdbEpisodes(torrent, videos, metadata))
-      .then(videos => Promise.all(videos.map(video => video.isMovie
-          ? mapSeriesMovie(video, torrent)
-          : mapSeriesEpisode(video, torrent, videos))))
-      .then(videos => videos
-          .reduce((a, b) => a.concat(b), [])
-          .map(video => isFeaturette(video) ? clearInfoFields(video) : video))
-  return { contents, videos: parsedVideos, subtitles };
-}
-
-async function getMoviesTorrentContent(torrent) {
-  const files = await torrentFiles(torrent)
-      .catch(error => {
-        if (!isPackTorrent(torrent)) {
-          return { videos: [{ name: torrent.title, path: torrent.title, size: torrent.size }] }
-        }
-        return Promise.reject(error);
-      });
-  if (files.contents && files.contents.length && !files.videos.length && isDiskTorrent(files.contents)) {
-    files.videos = [{ name: torrent.title, path: torrent.title, size: torrent.size }];
-  }
-  return files;
-}
-
-async function getSeriesTorrentContent(torrent) {
-  return torrentFiles(torrent)
-      .catch(error => {
-        if (!isPackTorrent(torrent)) {
-          return { videos: [{ name: torrent.title, path: torrent.title, size: torrent.size }] }
-        }
-        return Promise.reject(error);
-      });
-}
-
-async function mapSeriesEpisode(file, torrent, files) {
-  if (!file.episodes && !file.kitsuEpisodes) {
-    if (files.length === 1 || files.some(f => f.episodes || f.kitsuEpisodes) || parse(torrent.title).seasons) {
-      return Promise.resolve({
-        infoHash: torrent.infoHash,
-        fileIndex: file.fileIndex,
-        title: file.path || file.name,
-        size: file.size,
-        imdbId: torrent.imdbId || file.imdbId,
-      });
-    }
-    return Promise.resolve([]);
-  }
-  const episodeIndexes = [...(file.episodes || file.kitsuEpisodes).keys()];
-  return Promise.resolve(episodeIndexes.map((index) => ({
-    infoHash: torrent.infoHash,
-    fileIndex: file.fileIndex,
-    title: file.path || file.name,
-    size: file.size,
-    imdbId: file.imdbId || torrent.imdbId,
-    imdbSeason: file.season,
-    imdbEpisode: file.episodes && file.episodes[index],
-    kitsuId: file.kitsuId || torrent.kitsuId,
-    kitsuEpisode: file.kitsuEpisodes && file.kitsuEpisodes[index]
-  })))
-}
-
-async function mapSeriesMovie(file, torrent) {
-  const kitsuId = torrent.type === TorrentType.ANIME ? await findMovieKitsuId(file) : undefined;
-  const imdbId = !kitsuId ? await findMovieImdbId(file) : undefined;
-  const metadata = await getMetadata(kitsuId || imdbId, TorrentType.MOVIE).catch(() => ({}));
-  const hasEpisode = metadata.videos && metadata.videos.length && (file.episode || metadata.videos.length === 1);
-  const episodeVideo = hasEpisode && metadata.videos[(file.episode || 1) - 1];
-  return [{
-    infoHash: torrent.infoHash,
-    fileIndex: file.fileIndex,
-    title: file.path || file.name,
-    size: file.size,
-    imdbId: metadata.imdbId || imdbId,
-    kitsuId: metadata.kitsuId || kitsuId,
-    imdbSeason: episodeVideo && metadata.imdbId ? episodeVideo.imdbSeason : undefined,
-    imdbEpisode: episodeVideo && metadata.imdbId ? episodeVideo.imdbEpisode || episodeVideo.episode : undefined,
-    kitsuEpisode: episodeVideo && metadata.kitsuId ? episodeVideo.kitsuEpisode || episodeVideo.episode : undefined
-  }];
-}
-
-async function decomposeEpisodes(torrent, files, metadata = { episodeCount: [] }) {
-  if (files.every(file => !file.episodes && !file.date)) {
-    return files;
-  }
-
-  preprocessEpisodes(files);
-
-  if (torrent.type === TorrentType.ANIME && torrent.kitsuId) {
-    if (needsCinemetaMetadataForAnime(files, metadata)) {
-      // In some cases anime could be resolved to wrong kitsuId
-      // because of imdb season naming/absolute per series naming/multiple seasons
-      // So in these cases we need to fetch cinemeta based metadata and decompose episodes using that
-      await updateToCinemetaMetadata(metadata);
-      if (files.some(file => Number.isInteger(file.season))) {
-        // sometimes multi season anime torrents don't include season 1 naming
-        files
-            .filter(file => !Number.isInteger(file.season) && file.episodes)
-            .forEach(file => file.season = 1);
-      }
-    } else {
-      // otherwise for anime type episodes are always absolute and for a single season
-      files
-          .filter(file => file.episodes && file.season !== 0)
-          .forEach(file => file.season = 1);
-      return files;
-    }
-  }
-
-  const sortedEpisodes = files
-      .map(file => !file.isMovie && file.episodes || [])
-      .reduce((a, b) => a.concat(b), [])
-      .sort((a, b) => a - b);
-
-  if (isConcatSeasonAndEpisodeFiles(files, sortedEpisodes, metadata)) {
-    decomposeConcatSeasonAndEpisodeFiles(torrent, files, metadata);
-  } else if (isDateEpisodeFiles(files, metadata)) {
-    decomposeDateEpisodeFiles(torrent, files, metadata);
-  } else if (isAbsoluteEpisodeFiles(torrent, files, metadata)) {
-    decomposeAbsoluteEpisodeFiles(torrent, files, metadata);
-  }
-  // decomposeEpisodeTitleFiles(torrent, files, metadata);
-
-  return files;
-}
-
-function preprocessEpisodes(files) {
-  // reverse special episode naming when they named with 0 episode, ie. S02E00
-  files
-      .filter(file => Number.isInteger(file.season) && file.episode === 0)
-      .forEach(file => {
-        file.episode = file.season
-        file.episodes = [file.season]
-        file.season = 0;
-      })
-}
-
-function isConcatSeasonAndEpisodeFiles(files, sortedEpisodes, metadata) {
-  if (metadata.kitsuId !== undefined) {
-    // anime does not use this naming scheme in 99% of cases;
-    return false;
-  }
-  // decompose concat season and episode files (ex. 101=S01E01) in case:
-  // 1. file has a season, but individual files are concatenated with that season (ex. path Season 5/511 - Prize
-  // Fighters.avi)
-  // 2. file does not have a season and the episode does not go out of range for the concat season
-  // episode count
-  const thresholdAbove = Math.max(Math.ceil(files.length * 0.05), 5);
-  const thresholdSorted = Math.max(Math.ceil(files.length * 0.8), 8);
-  const threshold = Math.max(Math.ceil(files.length * 0.8), 5);
-  const sortedConcatEpisodes = sortedEpisodes
-      .filter(ep => ep > 100)
-      .filter(ep => metadata.episodeCount[div100(ep) - 1] < ep)
-      .filter(ep => metadata.episodeCount[div100(ep) - 1] >= mod100(ep));
-  const concatFileEpisodes = files
-      .filter(file => !file.isMovie && file.episodes)
-      .filter(file => !file.season || file.episodes.every(ep => div100(ep) === file.season));
-  const concatAboveTotalEpisodeCount = files
-      .filter(file => !file.isMovie && file.episodes && file.episodes.every(ep => ep > 100))
-      .filter(file => file.episodes.every(ep => ep > metadata.totalCount));
-  return sortedConcatEpisodes.length >= thresholdSorted && concatFileEpisodes.length >= threshold
-      || concatAboveTotalEpisodeCount.length >= thresholdAbove;
-}
-
-function isDateEpisodeFiles(files, metadata) {
-  return files.every(file => (!file.season || !metadata.episodeCount[file.season - 1]) && file.date);
-}
-
-function isAbsoluteEpisodeFiles(torrent, files, metadata) {
-  const threshold = Math.ceil(files.length / 5);
-  const isAnime = torrent.type === TorrentType.ANIME && torrent.kitsuId;
-  const nonMovieEpisodes = files
-      .filter(file => !file.isMovie && file.episodes);
-  const absoluteEpisodes = files
-      .filter(file => file.season && file.episodes)
-      .filter(file => file.episodes.every(ep => metadata.episodeCount[file.season - 1] < ep))
-  return nonMovieEpisodes.every(file => !file.season)
-      || (isAnime && nonMovieEpisodes.every(file => file.season > metadata.episodeCount.length))
-      || absoluteEpisodes.length >= threshold;
-}
-
-function isNewEpisodeNotInMetadata(torrent, file, metadata) {
-  // new episode might not yet been indexed by cinemeta.
-  // detect this if episode number is larger than the last episode or season is larger than the last one
-  // only for non anime metas
-  const isAnime = torrent.type === TorrentType.ANIME && torrent.kitsuId;
-  return !isAnime && !file.isMovie && file.episodes && file.season !== 1
-      && /continuing|current/i.test(metadata.status)
-      && file.season >= metadata.episodeCount.length
-      && file.episodes.every(ep => ep > (metadata.episodeCount[file.season - 1] || 0));
-}
-
-function decomposeConcatSeasonAndEpisodeFiles(torrent, files, metadata) {
-  files
-      .filter(file => file.episodes && file.season !== 0 && file.episodes.every(ep => ep > 100))
-      .filter(file => metadata.episodeCount[(file.season || div100(file.episodes[0])) - 1] < 100)
-      .filter(file => file.season && file.episodes.every(ep => div100(ep) === file.season) || !file.season)
-      .forEach(file => {
-        file.season = div100(file.episodes[0]);
-        file.episodes = file.episodes.map(ep => mod100(ep))
-      });
-
-}
-
-function decomposeAbsoluteEpisodeFiles(torrent, files, metadata) {
-  if (metadata.episodeCount.length === 0) {
-    files
-        .filter(file => !Number.isInteger(file.season) && file.episodes && !file.isMovie)
-        .forEach(file => {
-          file.season = 1;
-        });
-    return;
-  }
-  files
-      .filter(file => file.episodes && !file.isMovie && file.season !== 0)
-      .filter(file => !isNewEpisodeNotInMetadata(torrent, file, metadata))
-      .filter(file => !file.season || (metadata.episodeCount[file.season - 1] || 0) < file.episodes[0])
-      .forEach(file => {
-        const seasonIdx = ([...metadata.episodeCount.keys()]
-                .find((i) => metadata.episodeCount.slice(0, i + 1).reduce((a, b) => a + b) >= file.episodes[0])
-            + 1 || metadata.episodeCount.length) - 1;
-
-        file.season = seasonIdx + 1;
-        file.episodes = file.episodes
-            .map(ep => ep - metadata.episodeCount.slice(0, seasonIdx).reduce((a, b) => a + b, 0))
-      });
-}
-
-function decomposeDateEpisodeFiles(torrent, files, metadata) {
-  if (!metadata || !metadata.videos || !metadata.videos.length) {
-    return;
-  }
-
-  const timeZoneOffset = getTimeZoneOffset(metadata.country);
-  const offsetVideos = metadata.videos
-      .reduce((map, video) => {
-        const releaseDate = moment(video.released).utcOffset(timeZoneOffset).format('YYYY-MM-DD');
-        map[releaseDate] = video;
-        return map;
-      }, {});
-
-  files
-      .filter(file => file.date)
-      .forEach(file => {
-        const video = offsetVideos[file.date];
-        if (video) {
-          file.season = video.season;
-          file.episodes = [video.episode];
-        }
-      });
-}
-
-
-/* eslint-disable no-unused-vars  */
-function decomposeEpisodeTitleFiles(torrent, files, metadata) {
-  files
-      // .filter(file => !file.season)
-      .map(file => {
-        const episodeTitle = file.name.replace('_', ' ')
-            .replace(/^.*(?:E\d+[abc]?|- )\s?(.+)\.\w{1,4}$/, '$1')
-            .trim();
-        const foundEpisode = metadata.videos
-            .map(video => ({ ...video, distance: distance(episodeTitle, video.name) }))
-            .sort((a, b) => b.distance - a.distance)[0];
-        if (foundEpisode) {
-          file.isMovie = false;
-          file.season = foundEpisode.season;
-          file.episodes = [foundEpisode.episode];
-        }
-      })
-}
-/* eslint-enable no-unused-vars  */
-
-function getTimeZoneOffset(country) {
-  switch (country) {
-    case 'United States':
-    case 'USA':
-      return '-08:00';
-    default:
-      return '00:00';
-  }
-}
-
-function assignKitsuOrImdbEpisodes(torrent, files, metadata) {
-  if (!metadata || !metadata.videos || !metadata.videos.length) {
-    if (torrent.type === TorrentType.ANIME) {
-      // assign episodes as kitsu episodes for anime when no metadata available for imdb mapping
-      files
-          .filter(file => file.season && file.episodes)
-          .forEach(file => {
-            file.kitsuEpisodes = file.episodes;
-            file.season = undefined;
-            file.episodes = undefined;
-          })
-      if (metadata.type === TorrentType.MOVIE && files.every(file => !file.imdbId)) {
-        // sometimes a movie has episode naming, thus not recognized as a movie and imdbId not assigned
-        files.forEach(file => file.imdbId = metadata.imdbId);
-      }
-    }
-    return files;
-  }
-
-  const seriesMapping = metadata.videos
-      .reduce((map, video) => {
-        const episodeMap = map[video.season] || {};
-        episodeMap[video.episode] = video;
-        map[video.season] = episodeMap;
-        return map;
-      }, {});
-
-  if (metadata.videos.some(video => Number.isInteger(video.imdbSeason)) || !metadata.imdbId) {
-    // kitsu episode info is the base
-    files
-        .filter(file => Number.isInteger(file.season) && file.episodes)
-        .map(file => {
-          const seasonMapping = seriesMapping[file.season];
-          const episodeMapping = seasonMapping && seasonMapping[file.episodes[0]];
-          file.kitsuEpisodes = file.episodes;
-          if (episodeMapping && Number.isInteger(episodeMapping.imdbSeason)) {
-            file.imdbId = metadata.imdbId;
-            file.season = episodeMapping.imdbSeason;
-            file.episodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].imdbEpisode);
-          } else {
-            // no imdb mapping available for episode
-            file.season = undefined;
-            file.episodes = undefined;
-          }
-        });
-  } else if (metadata.videos.some(video => video.kitsuEpisode)) {
-    // imdb episode info is base
-    files
-        .filter(file => Number.isInteger(file.season) && file.episodes)
-        .forEach(file => {
-          if (seriesMapping[file.season]) {
-            const seasonMapping = seriesMapping[file.season];
-            file.imdbId = metadata.imdbId;
-            file.kitsuId = seasonMapping[file.episodes[0]] && seasonMapping[file.episodes[0]].kitsuId;
-            file.kitsuEpisodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].kitsuEpisode);
-          } else if (seriesMapping[file.season - 1]) {
-            // sometimes a second season might be a continuation of the previous season
-            const seasonMapping = seriesMapping[file.season - 1];
-            const episodes = Object.values(seasonMapping);
-            const firstKitsuId = episodes.length && episodes[0].kitsuId;
-            const differentTitlesCount = new Set(episodes.map(ep => ep.kitsuId)).size
-            const skippedCount = episodes.filter(ep => ep.kitsuId === firstKitsuId).length;
-            const seasonEpisodes = files
-                .filter(otherFile => otherFile.season === file.season)
-                .reduce((a, b) => a.concat(b.episodes), []);
-            const isAbsoluteOrder = seasonEpisodes.every(ep => ep > skippedCount && ep <= episodes.length)
-            const isNormalOrder = seasonEpisodes.every(ep => ep + skippedCount <= episodes.length)
-            if (differentTitlesCount >= 1 && (isAbsoluteOrder || isNormalOrder)) {
-              file.imdbId = metadata.imdbId;
-              file.season = file.season - 1;
-              file.episodes = file.episodes.map(ep => isAbsoluteOrder ? ep : ep + skippedCount);
-              file.kitsuId = seasonMapping[file.episodes[0]].kitsuId;
-              file.kitsuEpisodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].kitsuEpisode);
-            }
-          } else if (Object.values(seriesMapping).length === 1 && seriesMapping[1]) {
-            // sometimes series might be named with sequel season but it's not a season on imdb and a new title
-            const seasonMapping = seriesMapping[1];
-            file.imdbId = metadata.imdbId;
-            file.season = 1;
-            file.kitsuId = seasonMapping[file.episodes[0]].kitsuId;
-            file.kitsuEpisodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].kitsuEpisode);
-          }
-        });
-  }
-  return files;
-}
-
-function needsCinemetaMetadataForAnime(files, metadata) {
-  if (!metadata || !metadata.imdbId || !metadata.videos || !metadata.videos.length) {
-    return false;
-  }
-
-  const minSeason = Math.min(...metadata.videos.map(video => video.imdbSeason)) || Number.MAX_VALUE;
-  const maxSeason = Math.max(...metadata.videos.map(video => video.imdbSeason)) || Number.MAX_VALUE;
-  const differentSeasons = new Set(metadata.videos
-      .map(video => video.imdbSeason)
-      .filter(season => Number.isInteger(season))).size;
-  const total = metadata.totalCount || Number.MAX_VALUE;
-  return differentSeasons > 1 || files
-      .filter(file => !file.isMovie && file.episodes)
-      .some(file => file.season < minSeason || file.season > maxSeason || file.episodes.every(ep => ep > total));
-}
-
-async function updateToCinemetaMetadata(metadata) {
-  return getMetadata(metadata.imdbId, metadata.type)
-      .then(newMetadata => !newMetadata.videos || !newMetadata.videos.length ? metadata : newMetadata)
-      .then(newMetadata => {
-        metadata.videos = newMetadata.videos;
-        metadata.episodeCount = newMetadata.episodeCount;
-        metadata.totalCount = newMetadata.totalCount;
-        return metadata;
-      })
-      .catch(error => logger.warn(`Failed ${metadata.imdbId} metadata cinemeta update due: ${error.message}`));
-}
-
-function findMovieImdbId(title) {
-  const parsedTitle = typeof title === 'string' ? parse(title) : title;
-  logger.debug(`Finding movie imdbId for ${title}`);
-  return imdb_limiter.schedule(() => getImdbId(parsedTitle, TorrentType.MOVIE).catch(() => undefined));
-}
-
-function findMovieKitsuId(title) {
-  const parsedTitle = typeof title === 'string' ? parse(title) : title;
-  return getKitsuId(parsedTitle, TorrentType.MOVIE).catch(() => undefined);
-}
-
-function isDiskTorrent(contents) {
-  return contents.some(content => isDisk(content.path));
-}
-
-function isSingleMovie(videos) {
-  return videos.length === 1 ||
-      (videos.length === 2 &&
-          videos.find(v => /\b(?:part|disc|cd)[ ._-]?0?1\b|^0?1\.\w{2,4}$/i.test(v.path)) &&
-          videos.find(v => /\b(?:part|disc|cd)[ ._-]?0?2\b|^0?2\.\w{2,4}$/i.test(v.path)));
-}
-
-function isFeaturette(video) {
-  return /featurettes?\/|extras-grym/i.test(video.path);
-}
-
-function clearInfoFields(video) {
-  video.imdbId = undefined;
-  video.imdbSeason = undefined;
-  video.imdbEpisode = undefined;
-  video.kitsuId = undefined;
-  video.kitsuEpisode = undefined;
-  return video;
-}
-
-function div100(episode) {
-  return (episode / 100 >> 0); // floor to nearest int
-}
-
-function mod100(episode) {
-  return episode % 100;
-}

src/node/consumer/src/lib/torrentSubtitles.js

@@ -1,89 +0,0 @@
-import { parse } from 'parse-torrent-title';
-
-export function assignSubtitles({ contents, videos, subtitles }) {
-  if (videos && videos.length && subtitles && subtitles.length) {
-    if (videos.length === 1) {
-      videos[0].subtitles = subtitles;
-      return { contents, videos, subtitles: [] };
-    }
-
-    const parsedVideos = videos
-        .map(video => _parseVideo(video));
-    const assignedSubs = subtitles
-        .map(subtitle => ({ subtitle, videos: _mostProbableSubtitleVideos(subtitle, parsedVideos) }));
-    const unassignedSubs = assignedSubs
-        .filter(assignedSub => !assignedSub.videos)
-        .map(assignedSub => assignedSub.subtitle);
-
-    assignedSubs
-        .filter(assignedSub => assignedSub.videos)
-        .forEach(assignedSub => assignedSub.videos
-            .forEach(video => video.subtitles = (video.subtitles || []).concat(assignedSub.subtitle)));
-    return { contents, videos, subtitles: unassignedSubs };
-  }
-  return { contents, videos, subtitles };
-}
-
-function _parseVideo(video) {
-  const fileName = video.title.split('/').pop().replace(/\.(\w{2,4})$/, '');
-  const folderName = video.title.replace(/\/?[^/]+$/, '');
-  return {
-    videoFile: video,
-    fileName: fileName,
-    folderName: folderName,
-    ...parseFilename(video.title)
-  };
-}
-
-function _mostProbableSubtitleVideos(subtitle, parsedVideos) {
-  const subTitle = (subtitle.title || subtitle.path).split('/').pop().replace(/\.(\w{2,4})$/, '');
-  const parsedSub = parsePath(subtitle.title || subtitle.path);
-  const byFileName = parsedVideos.filter(video => subTitle.includes(video.fileName));
-  if (byFileName.length === 1) {
-    return byFileName.map(v => v.videoFile);
-  }
-  const byTitleSeasonEpisode = parsedVideos.filter(video => video.title === parsedSub.title
-      && arrayEquals(video.seasons, parsedSub.seasons)
-      && arrayEquals(video.episodes, parsedSub.episodes));
-  if (singleVideoFile(byTitleSeasonEpisode)) {
-    return byTitleSeasonEpisode.map(v => v.videoFile);
-  }
-  const bySeasonEpisode = parsedVideos.filter(video => arrayEquals(video.seasons, parsedSub.seasons)
-      && arrayEquals(video.episodes, parsedSub.episodes));
-  if (singleVideoFile(bySeasonEpisode)) {
-    return bySeasonEpisode.map(v => v.videoFile);
-  }
-  const byTitle = parsedVideos.filter(video => video.title && video.title === parsedSub.title);
-  if (singleVideoFile(byTitle)) {
-    return byTitle.map(v => v.videoFile);
-  }
-  const byEpisode = parsedVideos.filter(video => arrayEquals(video.episodes, parsedSub.episodes));
-  if (singleVideoFile(byEpisode)) {
-    return byEpisode.map(v => v.videoFile);
-  }
-  return undefined;
-}
-
-function singleVideoFile(videos) {
-  return new Set(videos.map(v => v.videoFile.fileIndex)).size === 1;
-}
-
-function parsePath(path) {
-  const pathParts = path.split('/').map(part => parseFilename(part));
-  const parsedWithEpisode = pathParts.find(parsed => parsed.season && parsed.episodes);
-  return parsedWithEpisode || pathParts[pathParts.length - 1];
-}
-
-function parseFilename(filename) {
-  const parsedInfo = parse(filename)
-  const titleEpisode = parsedInfo.title.match(/(\d+)$/);
-  if (!parsedInfo.episodes && titleEpisode) {
-    parsedInfo.episodes = [parseInt(titleEpisode[1], 10)];
-  }
-  return parsedInfo;
-}
-
-function arrayEquals(array1, array2) {
-  if (!array1 || !array2) return array1 === array2;
-  return array1.length === array2.length && array1.every((value, index) => value === array2[index])
-}

src/node/consumer/src/lib/trackerService.js

@@ -1,26 +0,0 @@
-import axios from 'axios';
-import {cacheTrackers} from "./cache.js";
-import { trackerConfig } from './config.js';
-import {logger} from "./logger.js";
-
-const downloadTrackers = async () => {
-    const response = await axios.get(trackerConfig.TRACKERS_URL);
-    const trackersListText = response.data;
-    // Trackers are separated by a newline character
-    let urlTrackers = trackersListText.split("\n");
-    // remove blank lines
-    urlTrackers = urlTrackers.filter(line => line.trim() !== '');
-    
-    if (!trackerConfig.UDP_ENABLED) {
-        // remove any udp trackers
-        urlTrackers = urlTrackers.filter(line => !line.startsWith('udp://'));    
-    }
-
-    logger.info(`Trackers updated at ${Date.now()}: ${urlTrackers.length} trackers`);
-
-    return urlTrackers;
-};
-
-export const getTrackers = async () => {
-    return cacheTrackers(downloadTrackers);
-};

src/node/consumer/src/lib/types.js

@@ -1,11 +0,0 @@
-export const TorrentType = {
-  MOVIE: 'movie',
-  SERIES: 'series',
-  ANIME: 'anime',
-  PORN: 'xxx',
-};
-
-export const CacheType = {
-    MEMORY: 'memory',
-    MONGODB: 'mongodb',
-};

src/node/consumer/src/main.ts

@@ -0,0 +1,9 @@
+import "reflect-metadata"; // required
+import {ICompositionalRoot} from "@setup/composition_root";
+import {serviceContainer} from "@setup/inversify_config";
+import {IocTypes} from "@setup/ioc_types";
+
+(async (): Promise<void> => {
+    const compositionalRoot = serviceContainer.get<ICompositionalRoot>(IocTypes.ICompositionalRoot);
+    await compositionalRoot.start();
+})();

src/node/consumer/src/setup/composition_root.ts

@@ -0,0 +1,22 @@
+import {IProcessTorrentsJob} from "@interfaces/process_torrents_job";
+import {ITrackerService} from "@interfaces/tracker_service";
+import {IDatabaseRepository} from "@repository/interfaces/database_repository";
+import {IocTypes} from "@setup/ioc_types";
+import {inject, injectable} from "inversify";
+
+export interface ICompositionalRoot {
+    start(): Promise<void>;
+}
+
+@injectable()
+export class CompositionalRoot implements ICompositionalRoot {
+    @inject(IocTypes.ITrackerService) trackerService: ITrackerService;
+    @inject(IocTypes.IDatabaseRepository) databaseRepository: IDatabaseRepository;
+    @inject(IocTypes.IProcessTorrentsJob) processTorrentsJob: IProcessTorrentsJob;
+
+    async start(): Promise<void> {
+        await this.trackerService.getTrackers();
+        await this.databaseRepository.connect();
+        await this.processTorrentsJob.listenToQueue();
+    }
+}

src/node/consumer/src/setup/inversify_config.ts

@@ -0,0 +1,42 @@
+import {ICacheService} from "@interfaces/cache_service";
+import {ILoggingService} from "@interfaces/logging_service";
+import {IMetadataService} from "@interfaces/metadata_service";
+import {IProcessTorrentsJob} from "@interfaces/process_torrents_job";
+import {ITorrentDownloadService} from "@interfaces/torrent_download_service";
+import {ITorrentEntriesService} from "@interfaces/torrent_entries_service";
+import {ITorrentFileService} from "@interfaces/torrent_file_service";
+import {ITorrentProcessingService} from "@interfaces/torrent_processing_service";
+import {ITorrentSubtitleService} from "@interfaces/torrent_subtitle_service";
+import {ITrackerService} from "@interfaces/tracker_service";
+import {ProcessTorrentsJob} from "@jobs/process_torrents_job";
+import {DatabaseRepository} from "@repository/database_repository";
+import {IDatabaseRepository} from "@repository/interfaces/database_repository";
+import {CacheService} from "@services/cache_service";
+import {LoggingService} from "@services/logging_service";
+import {MetadataService} from "@services/metadata_service";
+import {TorrentDownloadService} from "@services/torrent_download_service";
+import {TorrentEntriesService} from "@services/torrent_entries_service";
+import {TorrentFileService} from "@services/torrent_file_service";
+import {TorrentProcessingService} from "@services/torrent_processing_service";
+import {TorrentSubtitleService} from "@services/torrent_subtitle_service";
+import {TrackerService} from "@services/tracker_service";
+import {ICompositionalRoot, CompositionalRoot} from "@setup/composition_root";
+import {IocTypes} from "@setup/ioc_types";
+import {Container} from "inversify";
+
+const serviceContainer = new Container();
+
+serviceContainer.bind<ICompositionalRoot>(IocTypes.ICompositionalRoot).to(CompositionalRoot).inSingletonScope();
+serviceContainer.bind<ICacheService>(IocTypes.ICacheService).to(CacheService).inSingletonScope();
+serviceContainer.bind<ILoggingService>(IocTypes.ILoggingService).to(LoggingService).inSingletonScope();
+serviceContainer.bind<ITrackerService>(IocTypes.ITrackerService).to(TrackerService).inSingletonScope();
+serviceContainer.bind<ITorrentDownloadService>(IocTypes.ITorrentDownloadService).to(TorrentDownloadService).inSingletonScope();
+serviceContainer.bind<ITorrentFileService>(IocTypes.ITorrentFileService).to(TorrentFileService);
+serviceContainer.bind<ITorrentProcessingService>(IocTypes.ITorrentProcessingService).to(TorrentProcessingService);
+serviceContainer.bind<ITorrentSubtitleService>(IocTypes.ITorrentSubtitleService).to(TorrentSubtitleService);
+serviceContainer.bind<ITorrentEntriesService>(IocTypes.ITorrentEntriesService).to(TorrentEntriesService);
+serviceContainer.bind<IMetadataService>(IocTypes.IMetadataService).to(MetadataService);
+serviceContainer.bind<IDatabaseRepository>(IocTypes.IDatabaseRepository).to(DatabaseRepository);
+serviceContainer.bind<IProcessTorrentsJob>(IocTypes.IProcessTorrentsJob).to(ProcessTorrentsJob);
+
+export {serviceContainer};

src/node/consumer/src/setup/ioc_types.ts

@@ -0,0 +1,18 @@
+export const IocTypes = {
+    // Composition root
+    ICompositionalRoot: Symbol.for("ICompositionalRoot"),
+    // Services
+    ICacheService: Symbol.for("ICacheService"),
+    ILoggingService: Symbol.for("ILoggingService"),
+    IMetadataService: Symbol.for("IMetadataService"),
+    ITorrentDownloadService: Symbol.for("ITorrentDownloadService"),
+    ITorrentEntriesService: Symbol.for("ITorrentEntriesService"),
+    ITorrentFileService: Symbol.for("ITorrentFileService"),
+    ITorrentProcessingService: Symbol.for("ITorrentProcessingService"),
+    ITorrentSubtitleService: Symbol.for("ITorrentSubtitleService"),
+    ITrackerService: Symbol.for("ITrackerService"),
+    // DAL
+    IDatabaseRepository: Symbol.for("IDatabaseRepository"),
+    // Jobs
+    IProcessTorrentsJob: Symbol.for("IProcessTorrentsJob"),
+};

src/node/consumer/test/helpers/boolean_helpers.test.ts

@@ -0,0 +1,35 @@
+import { BooleanHelpers } from '@helpers/boolean_helpers';
+
+describe('BooleanHelpers.parseBool', () => {
+    it('should return true when value is "true"', () => {
+        expect(BooleanHelpers.parseBool('true', false)).toBe(true);
+    });
+
+    it('should return true when value is "1"', () => {
+        expect(BooleanHelpers.parseBool('1', false)).toBe(true);
+    });
+
+    it('should return true when value is "yes"', () => {
+        expect(BooleanHelpers.parseBool('yes', false)).toBe(true);
+    });
+
+    it('should return false when value is "false"', () => {
+        expect(BooleanHelpers.parseBool('false', true)).toBe(false);
+    });
+
+    it('should return false when value is "0"', () => {
+        expect(BooleanHelpers.parseBool('0', true)).toBe(false);
+    });
+
+    it('should return false when value is "no"', () => {
+        expect(BooleanHelpers.parseBool('no', true)).toBe(false);
+    });
+
+    it('should return default value when value is undefined', () => {
+        expect(BooleanHelpers.parseBool(undefined, true)).toBe(true);
+    });
+
+    it('should return default value when value is not "true", "1", "yes", "false", "0", or "no"', () => {
+        expect(BooleanHelpers.parseBool('random', true)).toBe(true);
+    });
+});

src/node/consumer/test/helpers/extension_helpers.test.ts

@@ -0,0 +1,33 @@
+import { ExtensionHelpers } from '@helpers/extension_helpers';
+
+describe('ExtensionHelpers', () => {
+    describe('isVideo', () => {
+        it('should return true when file extension is a video extension', () => {
+            expect(ExtensionHelpers.isVideo('file.mp4')).toBe(true);
+        });
+
+        it('should return false when file extension is not a video extension', () => {
+            expect(ExtensionHelpers.isVideo('file.txt')).toBe(false);
+        });
+    });
+
+    describe('isSubtitle', () => {
+        it('should return true when file extension is a subtitle extension', () => {
+            expect(ExtensionHelpers.isSubtitle('file.srt')).toBe(true);
+        });
+
+        it('should return false when file extension is not a subtitle extension', () => {
+            expect(ExtensionHelpers.isSubtitle('file.txt')).toBe(false);
+        });
+    });
+
+    describe('isDisk', () => {
+        it('should return true when file extension is a disk extension', () => {
+            expect(ExtensionHelpers.isDisk('file.iso')).toBe(true);
+        });
+
+        it('should return false when file extension is not a disk extension', () => {
+            expect(ExtensionHelpers.isDisk('file.txt')).toBe(false);
+        });
+    });
+});

src/node/consumer/test/helpers/promise_helpers.test.ts

@@ -0,0 +1,55 @@
+import { PromiseHelpers } from '@helpers/promises_helpers';
+
+describe('PromiseHelpers', () => {
+    beforeAll(() => {
+        jest.useFakeTimers({timerLimit: 5000});
+    });
+
+    afterAll(() => {
+        jest.useRealTimers();
+    });
+    
+    describe('sequence', () => {
+        it('should resolve promises in sequence', async () => {
+            const promises = [() => Promise.resolve(1), () => Promise.resolve(2), () => Promise.resolve(3)];
+            const result = await PromiseHelpers.sequence(promises);
+            expect(result).toEqual([1, 2, 3]);
+        });
+    });
+
+    describe('first', () => {
+        it('should resolve the first fulfilled promise', async () => {
+            const promises = [Promise.reject('error'), Promise.resolve('success'), Promise.resolve('success2')];
+            const result = await PromiseHelpers.first(promises);
+            expect(result).toBe('success');
+        });
+    });
+
+    describe('delay', () => {
+        it('should delay execution', async () => {
+            const startTime = Date.now();
+            const delayPromise = PromiseHelpers.delay(1000);
+            jest.runAllTimers();
+            await delayPromise;
+            const endTime = Date.now();
+            expect(endTime - startTime).toBeGreaterThanOrEqual(1000);
+        }, 30000);
+    });
+
+    describe('timeout', () => {
+        it('should reject promise after timeout', async () => {
+            const promise = new Promise((resolve) => setTimeout(resolve, 2000));
+            const timeoutPromise = PromiseHelpers.timeout(1000, promise);
+            jest.advanceTimersByTime(1000);
+            await expect(timeoutPromise).rejects.toBe('Timed out');
+        }, 20000);
+    });
+
+    describe('mostCommonValue', () => {
+        it('should return the most common value in an array', () => {
+            const array = [1, 2, 2, 3, 3, 3];
+            const result = PromiseHelpers.mostCommonValue(array);
+            expect(result).toBe(3);
+        });
+    });
+});

src/node/consumer/test/mock-responses/assets/flash-episode-list.json

@@ -0,0 +1,232 @@
+{
+  "meta": {
+    "status": "Ended",
+    "videos": [
+      {
+        "name": "Pilot",
+        "season": 1,
+        "number": 0,
+        "firstAired": "1990-09-20T00:00:00.000Z",
+        "rating": "6.6",
+        "id": "tt0098798:1:0",
+        "overview": "",
+        "imdb_id": "tt0099580"
+      },
+      {
+        "name": "Out of Control",
+        "season": 1,
+        "number": 1,
+        "firstAired": "1990-09-26T00:00:00.000Z",
+        "rating": "6.8",
+        "id": "tt0098798:1:1",
+        "overview": "",
+        "imdb_id": "tt0579962"
+      },
+      {
+        "name": "Watching the Detectives",
+        "season": 1,
+        "number": 2,
+        "firstAired": "1990-10-17T00:00:00.000Z",
+        "rating": "6.9",
+        "id": "tt0098798:1:2",
+        "overview": "",
+        "imdb_id": "tt0579971"
+      },
+      {
+        "name": "Honor Among Thieves",
+        "season": 1,
+        "number": 3,
+        "firstAired": "1990-10-25T00:00:00.000Z",
+        "rating": "6.8",
+        "id": "tt0098798:1:3",
+        "overview": "",
+        "imdb_id": "tt0579961"
+      },
+      {
+        "name": "Double Vision",
+        "season": 1,
+        "number": 4,
+        "firstAired": "1990-11-01T00:00:00.000Z",
+        "rating": "6.6",
+        "id": "tt0098798:1:4",
+        "overview": "",
+        "imdb_id": "tt0579957"
+      },
+      {
+        "name": "Sins of the Father",
+        "season": 1,
+        "number": 5,
+        "firstAired": "1990-11-08T00:00:00.000Z",
+        "rating": "6.9",
+        "id": "tt0098798:1:5",
+        "overview": "",
+        "imdb_id": "tt0579965"
+      },
+      {
+        "name": "Child's Play",
+        "season": 1,
+        "number": 6,
+        "firstAired": "1990-11-15T00:00:00.000Z",
+        "rating": "6.8",
+        "id": "tt0098798:1:6",
+        "overview": "",
+        "imdb_id": "tt0579955"
+      },
+      {
+        "name": "Shroud of Death",
+        "season": 1,
+        "number": 7,
+        "firstAired": "1990-11-29T00:00:00.000Z",
+        "rating": "7.1",
+        "id": "tt0098798:1:7",
+        "overview": "",
+        "imdb_id": "tt0579963"
+      },
+      {
+        "name": "Ghost in the Machine",
+        "season": 1,
+        "number": 8,
+        "firstAired": "1990-12-13T00:00:00.000Z",
+        "rating": "7.7",
+        "id": "tt0098798:1:8",
+        "overview": "",
+        "imdb_id": "tt0579959"
+      },
+      {
+        "name": "Sight Unseen",
+        "season": 1,
+        "number": 9,
+        "firstAired": "1991-01-10T00:00:00.000Z",
+        "rating": "7",
+        "id": "tt0098798:1:9",
+        "overview": "",
+        "imdb_id": "tt0579964"
+      },
+      {
+        "name": "Beat the Clock",
+        "season": 1,
+        "number": 10,
+        "firstAired": "1991-01-31T00:00:00.000Z",
+        "rating": "7",
+        "id": "tt0098798:1:10",
+        "overview": "",
+        "imdb_id": "tt0579953"
+      },
+      {
+        "name": "The Trickster",
+        "season": 1,
+        "number": 11,
+        "firstAired": "1991-02-07T00:00:00.000Z",
+        "rating": "7.7",
+        "id": "tt0098798:1:11",
+        "overview": "",
+        "imdb_id": "tt0579969"
+      },
+      {
+        "name": "Tina, Is That You?",
+        "season": 1,
+        "number": 12,
+        "firstAired": "1991-02-14T00:00:00.000Z",
+        "rating": "6.8",
+        "id": "tt0098798:1:12",
+        "overview": "",
+        "imdb_id": "tt0579967"
+      },
+      {
+        "name": "Be My Baby",
+        "season": 1,
+        "number": 13,
+        "firstAired": "1991-02-20T00:00:00.000Z",
+        "rating": "6.5",
+        "id": "tt0098798:1:13",
+        "overview": "",
+        "imdb_id": "tt0579952"
+      },
+      {
+        "name": "Fast Forward",
+        "season": 1,
+        "number": 14,
+        "firstAired": "1991-02-27T00:00:00.000Z",
+        "rating": "7.9",
+        "id": "tt0098798:1:14",
+        "overview": "",
+        "imdb_id": "tt0579958"
+      },
+      {
+        "name": "Deadly Nightshade",
+        "season": 1,
+        "number": 15,
+        "firstAired": "1991-03-28T00:00:00.000Z",
+        "rating": "7.7",
+        "id": "tt0098798:1:15",
+        "overview": "",
+        "imdb_id": "tt0579966"
+      },
+      {
+        "name": "Captain Cold",
+        "season": 1,
+        "number": 16,
+        "firstAired": "1991-04-05T00:00:00.000Z",
+        "rating": "7.7",
+        "id": "tt0098798:1:16",
+        "overview": "",
+        "imdb_id": "tt0579954"
+      },
+      {
+        "name": "Twin Streaks",
+        "season": 1,
+        "number": 17,
+        "firstAired": "1991-04-12T00:00:00.000Z",
+        "rating": "7.1",
+        "id": "tt0098798:1:17",
+        "overview": "",
+        "imdb_id": "tt0579970"
+      },
+      {
+        "name": "Done with Mirrors",
+        "season": 1,
+        "number": 18,
+        "firstAired": "1991-04-27T00:00:00.000Z",
+        "rating": "7.3",
+        "id": "tt0098798:1:18",
+        "overview": "",
+        "imdb_id": "tt0579956"
+      },
+      {
+        "name": "Good Night, Central City",
+        "season": 1,
+        "number": 19,
+        "firstAired": "1991-05-04T00:00:00.000Z",
+        "rating": "7.1",
+        "id": "tt0098798:1:19",
+        "overview": "",
+        "imdb_id": "tt0579960"
+      },
+      {
+        "name": "Alpha",
+        "season": 1,
+        "number": 20,
+        "firstAired": "1991-05-11T00:00:00.000Z",
+        "rating": "7.5",
+        "id": "tt0098798:1:20",
+        "overview": "",
+        "imdb_id": "tt0579951"
+      },
+      {
+        "name": "Trial of the Trickster",
+        "season": 1,
+        "number": 21,
+        "firstAired": "1991-05-18T00:00:00.000Z",
+        "rating": "7.9",
+        "id": "tt0098798:1:21",
+        "overview": "",
+        "imdb_id": "tt0579968"
+      }
+    ],
+    "id": "tt0098798",
+    "behaviorHints": {
+      "defaultVideoId": null,
+      "hasScheduledVideos": false
+    }
+  }
+}