mirror of
https://github.com/knightcrawler-stremio/knightcrawler.git
synced 2024-12-20 03:29:51 +00:00
Rewritten repository in typescript
This commit is contained in:
iPromKnight
70
src/node/consumer/package-lock.json
generated
70
src/node/consumer/package-lock.json
generated
@@ -14,24 +14,25 @@
|
||||
"axios": "^1.6.1",
|
||||
"bottleneck": "^2.19.5",
|
||||
"cache-manager": "^5.4.0",
|
||||
"dotenv": "^16.4.1",
|
||||
"google-sr": "^3.2.1",
|
||||
"jaro-winkler": "^0.2.8",
|
||||
"magnet-uri": "^6.2.0",
|
||||
"moment": "^2.30.1",
|
||||
"mongodb": "^6.3.0",
|
||||
"name-to-imdb": "^3.0.4",
|
||||
"parse-torrent-title": "https://github.com/TheBeastLT/parse-torrent-title.git#022408972c2a040f846331a912a6a8487746a654",
|
||||
"pg": "^8.11.3",
|
||||
"pg-hstore": "^2.3.4",
|
||||
"pino": "^8.18.0",
|
||||
"sequelize": "^6.31.1",
|
||||
"reflect-metadata": "^0.2.1",
|
||||
"sequelize": "^6.36.0",
|
||||
"sequelize-typescript": "^2.1.6",
|
||||
"torrent-stream": "^1.2.1",
|
||||
"user-agents": "^1.0.1444"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.11.6",
|
||||
"@types/node": "^20.11.16",
|
||||
"@types/stremio-addon-sdk": "^1.6.10",
|
||||
"@types/validator": "^13.11.8",
|
||||
"esbuild": "^0.20.0",
|
||||
"eslint": "^8.56.0",
|
||||
"eslint-plugin-import": "^2.29.1",
|
||||
@@ -581,7 +582,8 @@
|
||||
},
|
||||
"node_modules/@types/node": {
|
||||
"version": "20.11.16",
|
||||
"license": "MIT",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.16.tgz",
|
||||
"integrity": "sha512-gKb0enTmRCzXSSUJDq6/sPcqrfCv2mkkG6Jt/clpn5eiCbKTY+SgZUxo+p8ZKMof5dCp9vHQUAB7wOUTod22wQ==",
|
||||
"dependencies": {
|
||||
"undici-types": "~5.26.4"
|
||||
}
|
||||
@@ -597,7 +599,8 @@
|
||||
},
|
||||
"node_modules/@types/validator": {
|
||||
"version": "13.11.8",
|
||||
"license": "MIT"
|
||||
"resolved": "https://registry.npmjs.org/@types/validator/-/validator-13.11.8.tgz",
|
||||
"integrity": "sha512-c/hzNDBh7eRF+KbCf+OoZxKbnkpaK/cKp9iLQWqB7muXtM+MtL9SUUH8vCFcLn6dH1Qm05jiexK0ofWY7TfOhQ=="
|
||||
},
|
||||
"node_modules/@types/webidl-conversions": {
|
||||
"version": "7.0.3",
|
||||
@@ -1409,17 +1412,6 @@
|
||||
"url": "https://github.com/fb55/domutils?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/dotenv": {
|
||||
"version": "16.4.1",
|
||||
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.1.tgz",
|
||||
"integrity": "sha512-CjA3y+Dr3FyFDOAMnxZEGtnW9KBR2M0JvvUtXNW+dYJL5ROWxP9DUHCwgFqpMk0OXCc0ljhaNTr2w/kutYIcHQ==",
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/motdotla/dotenv?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/dottie": {
|
||||
"version": "2.0.6",
|
||||
"license": "MIT"
|
||||
@@ -3776,6 +3768,11 @@
|
||||
"node": ">= 12.13.0"
|
||||
}
|
||||
},
|
||||
"node_modules/reflect-metadata": {
|
||||
"version": "0.2.1",
|
||||
"resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.2.1.tgz",
|
||||
"integrity": "sha512-i5lLI6iw9AU3Uu4szRNPPEkomnkjRTaVt9hy/bn5g/oSzekBSMeLZblcjP74AW0vBabqERLLIrz+gR8QYR54Tw=="
|
||||
},
|
||||
"node_modules/regexp.prototype.flags": {
|
||||
"version": "1.5.1",
|
||||
"dev": true,
|
||||
@@ -3965,13 +3962,14 @@
|
||||
},
|
||||
"node_modules/sequelize": {
|
||||
"version": "6.36.0",
|
||||
"resolved": "https://registry.npmjs.org/sequelize/-/sequelize-6.36.0.tgz",
|
||||
"integrity": "sha512-PqOa11EHwA/zLmGDU4aynbsavbHJUlgRvFuC/2cA4LhOuV6NHKcQ0IXB+hNdFrGT3rULmvc4kdIwnfCNsrECMQ==",
|
||||
"funding": [
|
||||
{
|
||||
"type": "opencollective",
|
||||
"url": "https://opencollective.com/sequelize"
|
||||
}
|
||||
],
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/debug": "^4.1.8",
|
||||
"@types/validator": "^13.7.17",
|
||||
@@ -4030,6 +4028,42 @@
|
||||
"node": ">= 10.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/sequelize-typescript": {
|
||||
"version": "2.1.6",
|
||||
"resolved": "https://registry.npmjs.org/sequelize-typescript/-/sequelize-typescript-2.1.6.tgz",
|
||||
"integrity": "sha512-Vc2N++3en346RsbGjL3h7tgAl2Y7V+2liYTAOZ8XL0KTw3ahFHsyAUzOwct51n+g70I1TOUDgs06Oh6+XGcFkQ==",
|
||||
"dependencies": {
|
||||
"glob": "7.2.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@types/node": "*",
|
||||
"@types/validator": "*",
|
||||
"reflect-metadata": "*",
|
||||
"sequelize": ">=6.20.1"
|
||||
}
|
||||
},
|
||||
"node_modules/sequelize-typescript/node_modules/glob": {
|
||||
"version": "7.2.0",
|
||||
"resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz",
|
||||
"integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==",
|
||||
"dependencies": {
|
||||
"fs.realpath": "^1.0.0",
|
||||
"inflight": "^1.0.4",
|
||||
"inherits": "2",
|
||||
"minimatch": "^3.0.4",
|
||||
"once": "^1.3.0",
|
||||
"path-is-absolute": "^1.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "*"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/isaacs"
|
||||
}
|
||||
},
|
||||
"node_modules/sequelize/node_modules/semver": {
|
||||
"version": "7.5.4",
|
||||
"license": "ISC",
|
||||
|
||||
@@ -24,18 +24,21 @@
|
||||
"pg": "^8.11.3",
|
||||
"pg-hstore": "^2.3.4",
|
||||
"pino": "^8.18.0",
|
||||
"sequelize": "^6.31.1",
|
||||
"reflect-metadata": "^0.2.1",
|
||||
"sequelize": "^6.36.0",
|
||||
"sequelize-typescript": "^2.1.6",
|
||||
"torrent-stream": "^1.2.1",
|
||||
"user-agents": "^1.0.1444"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.11.6",
|
||||
"@types/node": "^20.11.16",
|
||||
"@types/stremio-addon-sdk": "^1.6.10",
|
||||
"@types/validator": "^13.11.8",
|
||||
"esbuild": "^0.20.0",
|
||||
"eslint": "^8.56.0",
|
||||
"eslint-plugin-import": "^2.29.1",
|
||||
"eslint-plugin-import-helpers": "^1.3.1",
|
||||
"tsx": "^4.7.0",
|
||||
"pino-pretty": "^10.3.1"
|
||||
"pino-pretty": "^10.3.1",
|
||||
"tsx": "^4.7.0"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import { listenToQueue } from './jobs/processTorrents.js';
|
||||
import { connect } from './lib/repository.js';
|
||||
import { repository } from "./repository/database_repository";
|
||||
import { getTrackers } from "./lib/trackerService.js";
|
||||
|
||||
(async () => {
|
||||
await getTrackers();
|
||||
await connect();
|
||||
await repository.connect();
|
||||
await listenToQueue();
|
||||
})();
|
||||
@@ -1,7 +1,7 @@
|
||||
import amqp from 'amqplib'
|
||||
import { rabbitConfig, jobConfig } from '../lib/config.js'
|
||||
import { processTorrentRecord } from "../lib/ingestedTorrent.js";
|
||||
import {logger} from "../lib/logger.js";
|
||||
import {logger} from "../lib/logger";
|
||||
|
||||
const assertQueueOptions = { durable: true }
|
||||
const consumeQueueOptions = { noAck: false }
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { createCache, memoryStore} from 'cache-manager';
|
||||
import { mongoDbStore } from '@tirke/node-cache-manager-mongodb'
|
||||
import { cacheConfig } from './config.js';
|
||||
import { logger } from './logger.js';
|
||||
import { logger } from './logger';
|
||||
import { CacheType } from "./types.js";
|
||||
|
||||
const GLOBAL_KEY_PREFIX = 'knightcrawler-consumer';
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
export const rabbitConfig = {
|
||||
export const rabbitConfig = {
|
||||
URI: process.env.RABBIT_URI || 'amqp://localhost',
|
||||
QUEUE_NAME: process.env.QUEUE_NAME || 'test-queue'
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { createTorrentEntry, checkAndUpdateTorrent } from './torrentEntries.js';
|
||||
import {getTrackers} from "./trackerService.js";
|
||||
import { TorrentType } from './types.js';
|
||||
import {logger} from "./logger.js";
|
||||
import {logger} from "./logger";
|
||||
|
||||
export async function processTorrentRecord(torrent) {
|
||||
const {category} = torrent;
|
||||
|
||||
@@ -1,382 +0,0 @@
|
||||
import moment from 'moment';
|
||||
import { Sequelize, Op, DataTypes, fn, col, literal } from 'sequelize';
|
||||
import { databaseConfig } from './config.js';
|
||||
import { logger } from "./logger.js";
|
||||
import * as Promises from './promises.js';
|
||||
|
||||
const database = new Sequelize(
|
||||
databaseConfig.POSTGRES_URI,
|
||||
{
|
||||
logging: false
|
||||
}
|
||||
);
|
||||
|
||||
const Provider = database.define('provider', {
|
||||
name: { type: DataTypes.STRING(32), primaryKey: true },
|
||||
lastScraped: { type: DataTypes.DATE },
|
||||
lastScrapedId: { type: DataTypes.STRING(128) }
|
||||
});
|
||||
|
||||
const IngestedTorrent = database.define('ingested_torrent', {
|
||||
id: { type: DataTypes.BIGINT, autoIncrement: true, primaryKey: true },
|
||||
name: DataTypes.STRING,
|
||||
source: DataTypes.STRING,
|
||||
category: DataTypes.STRING,
|
||||
info_hash: DataTypes.STRING,
|
||||
size: DataTypes.STRING,
|
||||
seeders: DataTypes.INTEGER,
|
||||
leechers: DataTypes.INTEGER,
|
||||
imdb: DataTypes.STRING,
|
||||
processed: {
|
||||
type: DataTypes.BOOLEAN,
|
||||
defaultValue: false
|
||||
}
|
||||
},
|
||||
{
|
||||
indexes: [
|
||||
{
|
||||
unique: true,
|
||||
fields: ['source', 'info_hash']
|
||||
}
|
||||
]
|
||||
})
|
||||
|
||||
/* eslint-disable no-unused-vars */
|
||||
const IngestedPage = database.define('ingested_page', {
|
||||
id: { type: DataTypes.BIGINT, autoIncrement: true, primaryKey: true },
|
||||
url: { type: DataTypes.STRING, allowNull: false },
|
||||
},
|
||||
{
|
||||
indexes: [
|
||||
{
|
||||
unique: true,
|
||||
fields: ['url']
|
||||
}
|
||||
]
|
||||
})
|
||||
/* eslint-enable no-unused-vars */
|
||||
|
||||
const Torrent = database.define('torrent',
|
||||
{
|
||||
infoHash: { type: DataTypes.STRING(64), primaryKey: true },
|
||||
provider: { type: DataTypes.STRING(32), allowNull: false },
|
||||
torrentId: { type: DataTypes.STRING(512) },
|
||||
title: { type: DataTypes.STRING(512), allowNull: false },
|
||||
size: { type: DataTypes.BIGINT },
|
||||
type: { type: DataTypes.STRING(16), allowNull: false },
|
||||
uploadDate: { type: DataTypes.DATE, allowNull: false },
|
||||
seeders: { type: DataTypes.SMALLINT },
|
||||
trackers: { type: DataTypes.STRING(8000) },
|
||||
languages: { type: DataTypes.STRING(4096) },
|
||||
resolution: { type: DataTypes.STRING(16) },
|
||||
reviewed: { type: DataTypes.BOOLEAN, allowNull: false, defaultValue: false },
|
||||
opened: { type: DataTypes.BOOLEAN, allowNull: false, defaultValue: false }
|
||||
}
|
||||
);
|
||||
|
||||
const File = database.define('file',
|
||||
{
|
||||
id: { type: DataTypes.BIGINT, autoIncrement: true, primaryKey: true },
|
||||
infoHash: {
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
references: { model: Torrent, key: 'infoHash' },
|
||||
onDelete: 'CASCADE'
|
||||
},
|
||||
fileIndex: { type: DataTypes.INTEGER },
|
||||
title: { type: DataTypes.STRING(512), allowNull: false },
|
||||
size: { type: DataTypes.BIGINT },
|
||||
imdbId: { type: DataTypes.STRING(32) },
|
||||
imdbSeason: { type: DataTypes.INTEGER },
|
||||
imdbEpisode: { type: DataTypes.INTEGER },
|
||||
kitsuId: { type: DataTypes.INTEGER },
|
||||
kitsuEpisode: { type: DataTypes.INTEGER }
|
||||
},
|
||||
{
|
||||
indexes: [
|
||||
{
|
||||
unique: true,
|
||||
name: 'files_unique_file_constraint',
|
||||
fields: [
|
||||
col('infoHash'),
|
||||
fn('COALESCE', (col('fileIndex')), -1),
|
||||
fn('COALESCE', (col('imdbId')), 'null'),
|
||||
fn('COALESCE', (col('imdbSeason')), -1),
|
||||
fn('COALESCE', (col('imdbEpisode')), -1),
|
||||
fn('COALESCE', (col('kitsuId')), -1),
|
||||
fn('COALESCE', (col('kitsuEpisode')), -1)
|
||||
]
|
||||
},
|
||||
{ unique: false, fields: ['imdbId', 'imdbSeason', 'imdbEpisode'] },
|
||||
{ unique: false, fields: ['kitsuId', 'kitsuEpisode'] }
|
||||
]
|
||||
}
|
||||
);
|
||||
|
||||
const Subtitle = database.define('subtitle',
|
||||
{
|
||||
infoHash: {
|
||||
type: DataTypes.STRING(64),
|
||||
allowNull: false,
|
||||
references: { model: Torrent, key: 'infoHash' },
|
||||
onDelete: 'CASCADE'
|
||||
},
|
||||
fileIndex: {
|
||||
type: DataTypes.INTEGER,
|
||||
allowNull: false
|
||||
},
|
||||
fileId: {
|
||||
type: DataTypes.BIGINT,
|
||||
allowNull: true,
|
||||
references: { model: File, key: 'id' },
|
||||
onDelete: 'SET NULL'
|
||||
},
|
||||
title: { type: DataTypes.STRING(512), allowNull: false },
|
||||
},
|
||||
{
|
||||
timestamps: false,
|
||||
indexes: [
|
||||
{
|
||||
unique: true,
|
||||
name: 'subtitles_unique_subtitle_constraint',
|
||||
fields: [
|
||||
col('infoHash'),
|
||||
col('fileIndex'),
|
||||
fn('COALESCE', (col('fileId')), -1)
|
||||
]
|
||||
},
|
||||
{ unique: false, fields: ['fileId'] }
|
||||
]
|
||||
}
|
||||
);
|
||||
|
||||
const Content = database.define('content',
|
||||
{
|
||||
infoHash: {
|
||||
type: DataTypes.STRING(64),
|
||||
primaryKey: true,
|
||||
allowNull: false,
|
||||
references: { model: Torrent, key: 'infoHash' },
|
||||
onDelete: 'CASCADE'
|
||||
},
|
||||
fileIndex: {
|
||||
type: DataTypes.INTEGER,
|
||||
primaryKey: true,
|
||||
allowNull: false
|
||||
},
|
||||
path: { type: DataTypes.STRING(512), allowNull: false },
|
||||
size: { type: DataTypes.BIGINT },
|
||||
},
|
||||
{
|
||||
timestamps: false,
|
||||
}
|
||||
);
|
||||
|
||||
const SkipTorrent = database.define('skip_torrent', {
|
||||
infoHash: { type: DataTypes.STRING(64), primaryKey: true },
|
||||
});
|
||||
|
||||
Torrent.hasMany(File, { foreignKey: 'infoHash', constraints: false });
|
||||
File.belongsTo(Torrent, { foreignKey: 'infoHash', constraints: false });
|
||||
Torrent.hasMany(Content, { foreignKey: 'infoHash', constraints: false });
|
||||
Content.belongsTo(Torrent, { foreignKey: 'infoHash', constraints: false });
|
||||
File.hasMany(Subtitle, { foreignKey: 'fileId', constraints: false });
|
||||
Subtitle.belongsTo(File, { foreignKey: 'fileId', constraints: false });
|
||||
|
||||
export function connect() {
|
||||
return database.sync({ alter: databaseConfig.AUTO_CREATE_AND_APPLY_MIGRATIONS })
|
||||
.catch(error => {
|
||||
console.error('Failed syncing database: ', error.message);
|
||||
throw error;
|
||||
});
|
||||
// I'm not convinced this code is needed. If anyone can see where it leads, or what it does, please inform me.
|
||||
// For now, I'm commenting it out. I don't think we ever reach this at the moment anyway as the previous ENABLE_SYNC
|
||||
// was always on.
|
||||
// return Promise.resolve();
|
||||
}
|
||||
|
||||
export function getProvider(provider) {
|
||||
return Provider.findOrCreate({ where: { name: { [Op.eq]: provider.name } }, defaults: provider })
|
||||
.then((result) => result[0])
|
||||
.catch(() => provider);
|
||||
}
|
||||
|
||||
export function getTorrent(torrent) {
|
||||
const where = torrent.infoHash
|
||||
? { infoHash: torrent.infoHash }
|
||||
: { provider: torrent.provider, torrentId: torrent.torrentId }
|
||||
return Torrent.findOne({ where: where });
|
||||
}
|
||||
|
||||
export function getTorrentsBasedOnTitle(titleQuery, type) {
|
||||
return getTorrentsBasedOnQuery({ title: { [Op.regexp]: `${titleQuery}` }, type: type });
|
||||
}
|
||||
|
||||
export function getTorrentsBasedOnQuery(where) {
|
||||
return Torrent.findAll({ where: where });
|
||||
}
|
||||
|
||||
export function getFilesBasedOnQuery(where) {
|
||||
return File.findAll({ where: where });
|
||||
}
|
||||
|
||||
export function getUnprocessedIngestedTorrents() {
|
||||
return IngestedTorrent.findAll({
|
||||
where: {
|
||||
processed: false,
|
||||
category: {
|
||||
[Op.or]: ['tv', 'movies']
|
||||
}
|
||||
},
|
||||
|
||||
});
|
||||
}
|
||||
|
||||
export function setIngestedTorrentsProcessed(ingestedTorrents) {
|
||||
return Promises.sequence(ingestedTorrents
|
||||
.map(ingestedTorrent => () => {
|
||||
ingestedTorrent.processed = true;
|
||||
return ingestedTorrent.save();
|
||||
}));
|
||||
}
|
||||
|
||||
export function getTorrentsWithoutSize() {
|
||||
return Torrent.findAll({
|
||||
where: literal(
|
||||
'exists (select 1 from files where files."infoHash" = torrent."infoHash" and files.size = 300000000)'),
|
||||
order: [
|
||||
['seeders', 'DESC']
|
||||
]
|
||||
});
|
||||
}
|
||||
|
||||
export function getUpdateSeedersTorrents(limit = 50) {
|
||||
const until = moment().subtract(7, 'days').format('YYYY-MM-DD');
|
||||
return Torrent.findAll({
|
||||
where: literal(`torrent."updatedAt" < '${until}'`),
|
||||
limit: limit,
|
||||
order: [
|
||||
['seeders', 'DESC'],
|
||||
['updatedAt', 'ASC']
|
||||
]
|
||||
});
|
||||
}
|
||||
|
||||
export function getUpdateSeedersNewTorrents(limit = 50) {
|
||||
const lastUpdate = moment().subtract(12, 'hours').format('YYYY-MM-DD');
|
||||
const createdAfter = moment().subtract(4, 'days').format('YYYY-MM-DD');
|
||||
return Torrent.findAll({
|
||||
where: literal(`torrent."updatedAt" < '${lastUpdate}' AND torrent."createdAt" > '${createdAfter}'`),
|
||||
limit: limit,
|
||||
order: [
|
||||
['seeders', 'ASC'],
|
||||
['updatedAt', 'ASC']
|
||||
]
|
||||
});
|
||||
}
|
||||
|
||||
export function getNoContentsTorrents() {
|
||||
return Torrent.findAll({
|
||||
where: { opened: false, seeders: { [Op.gte]: 1 } },
|
||||
limit: 500,
|
||||
order: [[fn('RANDOM')]]
|
||||
});
|
||||
}
|
||||
|
||||
export function createTorrent(torrent) {
|
||||
return Torrent.upsert(torrent)
|
||||
.then(() => createContents(torrent.infoHash, torrent.contents))
|
||||
.then(() => createSubtitles(torrent.infoHash, torrent.subtitles));
|
||||
}
|
||||
|
||||
export function setTorrentSeeders(torrent, seeders) {
|
||||
const where = torrent.infoHash
|
||||
? { infoHash: torrent.infoHash }
|
||||
: { provider: torrent.provider, torrentId: torrent.torrentId }
|
||||
return Torrent.update(
|
||||
{ seeders: seeders },
|
||||
{ where: where }
|
||||
);
|
||||
}
|
||||
|
||||
export function deleteTorrent(torrent) {
|
||||
return Torrent.destroy({ where: { infoHash: torrent.infoHash } })
|
||||
}
|
||||
|
||||
export function createFile(file) {
|
||||
if (file.id) {
|
||||
return (file.dataValues ? file.save() : File.upsert(file))
|
||||
.then(() => upsertSubtitles(file, file.subtitles));
|
||||
}
|
||||
if (file.subtitles && file.subtitles.length) {
|
||||
file.subtitles = file.subtitles.map(subtitle => ({ infoHash: file.infoHash, title: subtitle.path, ...subtitle }));
|
||||
}
|
||||
return File.create(file, { include: [Subtitle], ignoreDuplicates: true });
|
||||
}
|
||||
|
||||
export function getFiles(torrent) {
|
||||
return File.findAll({ where: { infoHash: torrent.infoHash } });
|
||||
}
|
||||
|
||||
export function getFilesBasedOnTitle(titleQuery) {
|
||||
return File.findAll({ where: { title: { [Op.regexp]: `${titleQuery}` } } });
|
||||
}
|
||||
|
||||
export function deleteFile(file) {
|
||||
return File.destroy({ where: { id: file.id } })
|
||||
}
|
||||
|
||||
export function createSubtitles(infoHash, subtitles) {
|
||||
if (subtitles && subtitles.length) {
|
||||
return Subtitle.bulkCreate(subtitles.map(subtitle => ({ infoHash, title: subtitle.path, ...subtitle })));
|
||||
}
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
export function upsertSubtitles(file, subtitles) {
|
||||
if (file.id && subtitles && subtitles.length) {
|
||||
return Promises.sequence(subtitles
|
||||
.map(subtitle => {
|
||||
subtitle.fileId = file.id;
|
||||
subtitle.infoHash = subtitle.infoHash || file.infoHash;
|
||||
subtitle.title = subtitle.title || subtitle.path;
|
||||
return subtitle;
|
||||
})
|
||||
.map(subtitle => () => subtitle.dataValues ? subtitle.save() : Subtitle.create(subtitle)));
|
||||
}
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
export function getSubtitles(torrent) {
|
||||
return Subtitle.findAll({ where: { infoHash: torrent.infoHash } });
|
||||
}
|
||||
|
||||
export function getUnassignedSubtitles() {
|
||||
return Subtitle.findAll({ where: { fileId: null } });
|
||||
}
|
||||
|
||||
export function createContents(infoHash, contents) {
|
||||
if (contents && contents.length) {
|
||||
return Content.bulkCreate(contents.map(content => ({ infoHash, ...content })), { ignoreDuplicates: true })
|
||||
.then(() => Torrent.update({ opened: true }, { where: { infoHash: infoHash }, silent: true }));
|
||||
}
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
export function getContents(torrent) {
|
||||
return Content.findAll({ where: { infoHash: torrent.infoHash } });
|
||||
}
|
||||
|
||||
export function getSkipTorrent(torrent) {
|
||||
return SkipTorrent.findByPk(torrent.infoHash)
|
||||
.then((result) => {
|
||||
if (!result) {
|
||||
throw new Error(`torrent not found: ${torrent.infoHash}`);
|
||||
}
|
||||
return result.dataValues;
|
||||
})
|
||||
}
|
||||
|
||||
export function createSkipTorrent(torrent) {
|
||||
return SkipTorrent.upsert({ infoHash: torrent.infoHash });
|
||||
}
|
||||
@@ -2,11 +2,11 @@ import { parse } from 'parse-torrent-title';
|
||||
import { getImdbId, getKitsuId } from './metadata.js';
|
||||
import { isPackTorrent } from './parseHelper.js';
|
||||
import * as Promises from './promises.js';
|
||||
import * as repository from './repository.js';
|
||||
import { repository } from '../repository/database_repository';
|
||||
import { parseTorrentFiles } from './torrentFiles.js';
|
||||
import { assignSubtitles } from './torrentSubtitles.js';
|
||||
import { TorrentType } from './types.js';
|
||||
import {logger} from "./logger.js";
|
||||
import {logger} from "./logger";
|
||||
|
||||
export async function createTorrentEntry(torrent, overwrite = false) {
|
||||
const titleInfo = parse(torrent.title);
|
||||
|
||||
@@ -9,7 +9,7 @@ import { parseSeriesVideos, isPackTorrent } from './parseHelper.js';
|
||||
import * as Promises from './promises.js';
|
||||
import {torrentFiles} from "./torrent.js";
|
||||
import { TorrentType } from './types.js';
|
||||
import {logger} from "./logger.js";
|
||||
import {logger} from "./logger";
|
||||
|
||||
const MIN_SIZE = 5 * 1024 * 1024; // 5 MB
|
||||
const imdb_limiter = new Bottleneck({ maxConcurrent: metadataConfig.IMDB_CONCURRENT, minTime: metadataConfig.IMDB_INTERVAL_MS });
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import axios from 'axios';
|
||||
import {cacheTrackers} from "./cache.js";
|
||||
import { trackerConfig } from './config.js';
|
||||
import {logger} from "./logger.js";
|
||||
import {logger} from "./logger";
|
||||
|
||||
const downloadTrackers = async () => {
|
||||
const response = await axios.get(trackerConfig.TRACKERS_URL);
|
||||
|
||||
259
src/node/consumer/src/repository/database_repository.ts
Normal file
259
src/node/consumer/src/repository/database_repository.ts
Normal file
@@ -0,0 +1,259 @@
|
||||
import moment from 'moment';
|
||||
import {literal, Op, WhereOptions} from "sequelize";
|
||||
import {Model, Sequelize} from 'sequelize-typescript';
|
||||
import {databaseConfig} from '../lib/config';
|
||||
import * as Promises from '../lib/promises.js';
|
||||
import {Provider} from "./models/provider";
|
||||
import {File} from "./models/file";
|
||||
import {Torrent} from "./models/torrent";
|
||||
import {IngestedTorrent} from "./models/ingestedTorrent";
|
||||
import {Subtitle} from "./models/subtitle";
|
||||
import {Content} from "./models/content";
|
||||
import {SkipTorrent} from "./models/skipTorrent";
|
||||
import {FileAttributes} from "./interfaces/file_attributes";
|
||||
import {TorrentAttributes} from "./interfaces/torrent_attributes";
|
||||
import {IngestedPage} from "./models/ingestedPage";
|
||||
import {logger} from "../lib/logger";
|
||||
|
||||
class DatabaseRepository {
|
||||
private readonly database: Sequelize;
|
||||
|
||||
private models = [
|
||||
Torrent,
|
||||
Provider,
|
||||
File,
|
||||
Subtitle,
|
||||
Content,
|
||||
SkipTorrent,
|
||||
IngestedTorrent,
|
||||
IngestedPage];
|
||||
|
||||
constructor() {
|
||||
this.database = this.createDatabase();
|
||||
}
|
||||
|
||||
public async connect(): Promise<void> {
|
||||
try {
|
||||
await this.database.authenticate();
|
||||
logger.info('Database connection has been established successfully.');
|
||||
await this.database.sync({alter: true});
|
||||
} catch (error) {
|
||||
logger.error('Failed syncing database: ', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
public async getProvider(provider: Provider): Promise<Provider> {
|
||||
try {
|
||||
const [result] = await Provider.findOrCreate({ where: { name: { [Op.eq]: provider.name } }, defaults: provider });
|
||||
return result;
|
||||
} catch {
|
||||
return provider as Provider;
|
||||
}
|
||||
}
|
||||
|
||||
public async getTorrent(torrent: Torrent): Promise<Torrent | null> {
|
||||
const where = torrent.infoHash
|
||||
? { infoHash: torrent.infoHash }
|
||||
: { provider: torrent.provider, torrentId: torrent.torrentId };
|
||||
return await Torrent.findOne({ where });
|
||||
}
|
||||
|
||||
public async getTorrentsBasedOnTitle(titleQuery: string, type: string): Promise<Torrent[]> {
|
||||
return this.getTorrentsBasedOnQuery({ title: { [Op.regexp]: `${titleQuery}` }, type });
|
||||
}
|
||||
|
||||
public async getTorrentsBasedOnQuery(where: WhereOptions<TorrentAttributes>): Promise<Torrent[]> {
|
||||
return await Torrent.findAll({ where });
|
||||
}
|
||||
|
||||
public async getFilesBasedOnQuery(where: WhereOptions<FileAttributes>): Promise<File[]> {
|
||||
return await File.findAll({ where });
|
||||
}
|
||||
|
||||
public async getUnprocessedIngestedTorrents(): Promise<IngestedTorrent[]> {
|
||||
return await IngestedTorrent.findAll({
|
||||
where: {
|
||||
processed: false,
|
||||
category: {
|
||||
[Op.or]: ['tv', 'movies']
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
public async setIngestedTorrentsProcessed(ingestedTorrents: IngestedTorrent[]): Promise<void> {
|
||||
await Promises.sequence(ingestedTorrents
|
||||
.map(ingestedTorrent => async () => {
|
||||
ingestedTorrent.processed = true;
|
||||
await ingestedTorrent.save();
|
||||
}));
|
||||
}
|
||||
|
||||
public async getTorrentsWithoutSize(): Promise<Torrent[]> {
|
||||
return await Torrent.findAll({
|
||||
where: literal(
|
||||
'exists (select 1 from files where files."infoHash" = torrent."infoHash" and files.size = 300000000)'),
|
||||
order: [
|
||||
['seeders', 'DESC']
|
||||
]
|
||||
});
|
||||
}
|
||||
|
||||
public async getUpdateSeedersTorrents(limit = 50): Promise<Torrent[]> {
|
||||
const until = moment().subtract(7, 'days').format('YYYY-MM-DD');
|
||||
return await Torrent.findAll({
|
||||
where: literal(`torrent."updatedAt" < '${until}'`),
|
||||
limit: limit,
|
||||
order: [
|
||||
['seeders', 'DESC'],
|
||||
['updatedAt', 'ASC']
|
||||
]
|
||||
});
|
||||
}
|
||||
|
||||
public async getUpdateSeedersNewTorrents(limit = 50): Promise<Torrent[]> {
|
||||
const lastUpdate = moment().subtract(12, 'hours').format('YYYY-MM-DD');
|
||||
const createdAfter = moment().subtract(4, 'days').format('YYYY-MM-DD');
|
||||
return await Torrent.findAll({
|
||||
where: literal(`torrent."updatedAt" < '${lastUpdate}' AND torrent."createdAt" > '${createdAfter}'`),
|
||||
limit: limit,
|
||||
order: [
|
||||
['seeders', 'ASC'],
|
||||
['updatedAt', 'ASC']
|
||||
]
|
||||
});
|
||||
}
|
||||
|
||||
public async getNoContentsTorrents(): Promise<Torrent[]> {
|
||||
return await Torrent.findAll({
|
||||
where: { opened: false, seeders: { [Op.gte]: 1 } },
|
||||
limit: 500,
|
||||
order: literal('random()')
|
||||
});
|
||||
}
|
||||
|
||||
public async createTorrent(torrent: Torrent): Promise<void> {
|
||||
await Torrent.upsert(torrent);
|
||||
await this.createContents(torrent.infoHash, torrent.contents);
|
||||
await this.createSubtitles(torrent.infoHash, torrent.subtitles);
|
||||
}
|
||||
|
||||
public async setTorrentSeeders(torrent: TorrentAttributes, seeders: number): Promise<[number]> {
|
||||
const where = torrent.infoHash
|
||||
? { infoHash: torrent.infoHash }
|
||||
: { provider: torrent.provider, torrentId: torrent.torrentId };
|
||||
|
||||
return await Torrent.update(
|
||||
{ seeders: seeders },
|
||||
{ where: where }
|
||||
);
|
||||
}
|
||||
|
||||
public async deleteTorrent(torrent: TorrentAttributes): Promise<number> {
|
||||
return await Torrent.destroy({ where: { infoHash: torrent.infoHash } });
|
||||
}
|
||||
|
||||
public async createFile(file: File): Promise<void> {
|
||||
if (file.id) {
|
||||
if (file.dataValues) {
|
||||
await file.save();
|
||||
} else {
|
||||
await File.upsert(file);
|
||||
}
|
||||
await this.upsertSubtitles(file, file.subtitles);
|
||||
} else {
|
||||
if (file.subtitles && file.subtitles.length) {
|
||||
file.subtitles = file.subtitles.map(subtitle => {
|
||||
subtitle.title = subtitle.path;
|
||||
return subtitle;
|
||||
});
|
||||
}
|
||||
await File.create(file, { include: [Subtitle], ignoreDuplicates: true });
|
||||
}
|
||||
}
|
||||
|
||||
public async getFiles(torrent: Torrent): Promise<File[]> {
|
||||
return File.findAll({ where: { infoHash: torrent.infoHash } });
|
||||
}
|
||||
|
||||
public async getFilesBasedOnTitle(titleQuery: string): Promise<File[]> {
|
||||
return File.findAll({ where: { title: { [Op.regexp]: `${titleQuery}` } } });
|
||||
}
|
||||
|
||||
public async deleteFile(file: File): Promise<number> {
|
||||
return File.destroy({ where: { id: file.id } });
|
||||
}
|
||||
|
||||
public async createSubtitles(infoHash: string, subtitles: Subtitle[]): Promise<void | Model<any, any>[]> {
|
||||
if (subtitles && subtitles.length) {
|
||||
return Subtitle.bulkCreate(subtitles.map(subtitle => ({ infoHash, title: subtitle.path, ...subtitle })));
|
||||
}
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
public async upsertSubtitles(file: File, subtitles: Subtitle[]): Promise<void> {
|
||||
if (file.id && subtitles && subtitles.length) {
|
||||
await Promises.sequence(subtitles
|
||||
.map(subtitle => {
|
||||
subtitle.fileId = file.id;
|
||||
subtitle.infoHash = subtitle.infoHash || file.infoHash;
|
||||
subtitle.title = subtitle.title || subtitle.path;
|
||||
return subtitle;
|
||||
})
|
||||
.map(subtitle => async () => {
|
||||
if (subtitle.dataValues) {
|
||||
await subtitle.save();
|
||||
} else {
|
||||
await Subtitle.create(subtitle);
|
||||
}
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
public async getSubtitles(torrent: Torrent): Promise<Subtitle[]> {
|
||||
return Subtitle.findAll({ where: { infoHash: torrent.infoHash } });
|
||||
}
|
||||
|
||||
public async getUnassignedSubtitles(): Promise<Subtitle[]> {
|
||||
return Subtitle.findAll({ where: { fileId: null } });
|
||||
}
|
||||
|
||||
public async createContents(infoHash: string, contents: Content[]): Promise<void> {
|
||||
if (contents && contents.length) {
|
||||
await Content.bulkCreate(contents.map(content => ({ infoHash, ...content })), { ignoreDuplicates: true });
|
||||
await Torrent.update({ opened: true }, { where: { infoHash: infoHash }, silent: true });
|
||||
}
|
||||
}
|
||||
|
||||
public async getContents(torrent: Torrent): Promise<Content[]> {
|
||||
return Content.findAll({ where: { infoHash: torrent.infoHash } });
|
||||
}
|
||||
|
||||
public async getSkipTorrent(torrent: Torrent): Promise<SkipTorrent> {
|
||||
const result = await SkipTorrent.findByPk(torrent.infoHash);
|
||||
if (!result) {
|
||||
throw new Error(`torrent not found: ${torrent.infoHash}`);
|
||||
}
|
||||
return result.dataValues as SkipTorrent;
|
||||
}
|
||||
|
||||
public async createSkipTorrent(torrent: Torrent): Promise<[SkipTorrent, boolean]> {
|
||||
return SkipTorrent.upsert({ infoHash: torrent.infoHash });
|
||||
}
|
||||
|
||||
private createDatabase(): Sequelize {
|
||||
const newDatabase = new Sequelize(
|
||||
databaseConfig.POSTGRES_URI,
|
||||
{
|
||||
logging: false
|
||||
}
|
||||
);
|
||||
|
||||
newDatabase.addModels(this.models);
|
||||
|
||||
return newDatabase;
|
||||
}
|
||||
}
|
||||
|
||||
export const repository = new DatabaseRepository();
|
||||
@@ -0,0 +1,11 @@
|
||||
import {Optional} from "sequelize";
|
||||
|
||||
export interface ContentAttributes {
|
||||
infoHash: string;
|
||||
fileIndex: number;
|
||||
path: string;
|
||||
size: number;
|
||||
}
|
||||
|
||||
export interface ContentCreationAttributes extends Optional<ContentAttributes, 'fileIndex' | 'size'> {
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
import {Optional} from "sequelize";
|
||||
import {SubtitleAttributes} from "./subtitle_attributes";
|
||||
|
||||
export interface FileAttributes {
|
||||
id?: number;
|
||||
infoHash: string;
|
||||
fileIndex: number;
|
||||
title: string;
|
||||
size: number;
|
||||
imdbId: string;
|
||||
imdbSeason: number;
|
||||
imdbEpisode: number;
|
||||
kitsuId: number;
|
||||
kitsuEpisode: number;
|
||||
subtitles?: SubtitleAttributes[];
|
||||
}
|
||||
|
||||
export interface FileCreationAttributes extends Optional<FileAttributes, 'fileIndex' | 'size' | 'imdbId' | 'imdbSeason' | 'imdbEpisode' | 'kitsuId' | 'kitsuEpisode'> {
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
export interface IngestedPageAttributes {
|
||||
url: string;
|
||||
}
|
||||
|
||||
export interface IngestedPageCreationAttributes extends IngestedPageAttributes {
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import {Optional} from "sequelize";
|
||||
|
||||
export interface IngestedTorrentAttributes {
|
||||
name: string;
|
||||
source: string;
|
||||
category: string;
|
||||
info_hash: string;
|
||||
size: string;
|
||||
seeders: number;
|
||||
leechers: number;
|
||||
imdb: string;
|
||||
processed: boolean;
|
||||
}
|
||||
|
||||
export interface IngestedTorrentCreationAttributes extends Optional<IngestedTorrentAttributes, 'processed'> {
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
import {Optional} from "sequelize";
|
||||
|
||||
export interface ProviderAttributes {
|
||||
name: string;
|
||||
lastScraped: Date;
|
||||
lastScrapedId: string;
|
||||
}
|
||||
|
||||
export interface ProviderCreationAttributes extends Optional<ProviderAttributes, 'lastScraped' | 'lastScrapedId'> {
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
import {Optional} from "sequelize";
|
||||
|
||||
export interface SkipTorrentAttributes {
|
||||
infoHash: string;
|
||||
}
|
||||
|
||||
export interface SkipTorrentCreationAttributes extends Optional<SkipTorrentAttributes, never> {
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
import {Optional} from "sequelize";
|
||||
|
||||
export interface SubtitleAttributes {
|
||||
infoHash: string;
|
||||
fileIndex: number;
|
||||
fileId?: number;
|
||||
title: string;
|
||||
path: string;
|
||||
}
|
||||
|
||||
export interface SubtitleCreationAttributes extends Optional<SubtitleAttributes, 'fileId'> {
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import {Optional} from "sequelize";
|
||||
import {ContentAttributes} from "./content_attributes";
|
||||
import {SubtitleAttributes} from "./subtitle_attributes";
|
||||
import {FileAttributes} from "./file_attributes";
|
||||
|
||||
export interface TorrentAttributes {
|
||||
infoHash: string;
|
||||
provider: string;
|
||||
torrentId: string;
|
||||
title: string;
|
||||
size: number;
|
||||
type: string;
|
||||
uploadDate: Date;
|
||||
seeders: number;
|
||||
trackers: string;
|
||||
languages: string;
|
||||
resolution: string;
|
||||
reviewed: boolean;
|
||||
opened: boolean;
|
||||
contents: ContentAttributes[];
|
||||
files: FileAttributes[];
|
||||
subtitles?: SubtitleAttributes[];
|
||||
}
|
||||
|
||||
export interface TorrentCreationAttributes extends Optional<TorrentAttributes, 'torrentId' | 'size' | 'seeders' | 'trackers' | 'languages' | 'resolution' | 'reviewed' | 'opened'> {
|
||||
}
|
||||
22
src/node/consumer/src/repository/models/content.ts
Normal file
22
src/node/consumer/src/repository/models/content.ts
Normal file
@@ -0,0 +1,22 @@
|
||||
import {Table, Column, Model, HasMany, DataType, BelongsTo, ForeignKey} from 'sequelize-typescript';
|
||||
import {ContentAttributes, ContentCreationAttributes} from "../interfaces/content_attributes";
|
||||
import {Torrent} from "./torrent";
|
||||
|
||||
@Table({modelName: 'content', timestamps: false})
|
||||
export class Content extends Model<ContentAttributes, ContentCreationAttributes> {
|
||||
@Column({ type: DataType.STRING(64), primaryKey: true, allowNull: false, onDelete: 'CASCADE' })
|
||||
@ForeignKey(() => Torrent)
|
||||
declare infoHash: string;
|
||||
|
||||
@Column({ type: DataType.INTEGER, primaryKey: true, allowNull: false })
|
||||
declare fileIndex: number;
|
||||
|
||||
@Column({ type: DataType.STRING(512), allowNull: false })
|
||||
declare path: string;
|
||||
|
||||
@Column({ type: DataType.BIGINT })
|
||||
declare size: number;
|
||||
|
||||
@BelongsTo(() => Torrent, { constraints: false, foreignKey: 'infoHash' })
|
||||
torrent: Torrent;
|
||||
}
|
||||
60
src/node/consumer/src/repository/models/file.ts
Normal file
60
src/node/consumer/src/repository/models/file.ts
Normal file
@@ -0,0 +1,60 @@
|
||||
import {Table, Column, Model, HasMany, DataType, BelongsTo, ForeignKey} from 'sequelize-typescript';
|
||||
import {FileAttributes, FileCreationAttributes} from "../interfaces/file_attributes";
|
||||
import {Torrent} from "./torrent";
|
||||
import {Subtitle} from "./subtitle";
|
||||
import {SubtitleAttributes} from "../interfaces/subtitle_attributes";
|
||||
|
||||
const indexes = [
|
||||
{
|
||||
unique: true,
|
||||
name: 'files_unique_file_constraint',
|
||||
fields: [
|
||||
'infoHash',
|
||||
'fileIndex',
|
||||
'imdbId',
|
||||
'imdbSeason',
|
||||
'imdbEpisode',
|
||||
'kitsuId',
|
||||
'kitsuEpisode'
|
||||
]
|
||||
},
|
||||
{ unique: false, fields: ['imdbId', 'imdbSeason', 'imdbEpisode'] },
|
||||
{ unique: false, fields: ['kitsuId', 'kitsuEpisode'] }
|
||||
];
|
||||
|
||||
@Table({modelName: 'file', timestamps: true, indexes: indexes })
|
||||
export class File extends Model<FileAttributes, FileCreationAttributes> {
|
||||
@Column({ type: DataType.STRING(64), allowNull: false, onDelete: 'CASCADE' })
|
||||
@ForeignKey(() => Torrent)
|
||||
declare infoHash: string;
|
||||
|
||||
@Column({ type: DataType.INTEGER})
|
||||
declare fileIndex: number;
|
||||
|
||||
@Column({ type: DataType.STRING(512), allowNull: false })
|
||||
declare title: string;
|
||||
|
||||
@Column({ type: DataType.BIGINT })
|
||||
declare size: number;
|
||||
|
||||
@Column({ type: DataType.STRING(32) })
|
||||
declare imdbId: string;
|
||||
|
||||
@Column({ type: DataType.INTEGER })
|
||||
declare imdbSeason: number;
|
||||
|
||||
@Column({ type: DataType.INTEGER })
|
||||
declare imdbEpisode: number;
|
||||
|
||||
@Column({ type: DataType.INTEGER })
|
||||
declare kitsuId: number;
|
||||
|
||||
@Column({ type: DataType.INTEGER })
|
||||
declare kitsuEpisode: number;
|
||||
|
||||
@HasMany(() => Subtitle, { constraints: false, foreignKey: 'fileId'})
|
||||
declare subtitles?: Subtitle[];
|
||||
|
||||
@BelongsTo(() => Torrent, { constraints: false, foreignKey: 'infoHash' })
|
||||
torrent: Torrent;
|
||||
}
|
||||
16
src/node/consumer/src/repository/models/ingestedPage.ts
Normal file
16
src/node/consumer/src/repository/models/ingestedPage.ts
Normal file
@@ -0,0 +1,16 @@
|
||||
import { Table, Column, Model, HasMany, DataType } from 'sequelize-typescript';
|
||||
import {IngestedPageAttributes, IngestedPageCreationAttributes} from "../interfaces/ingested_page_attributes";
|
||||
|
||||
const indexes = [
|
||||
{
|
||||
unique: true,
|
||||
name: 'ingested_page_unique_url_constraint',
|
||||
fields: ['url']
|
||||
}
|
||||
];
|
||||
|
||||
@Table({modelName: 'ingested_page', timestamps: true, indexes: indexes})
|
||||
export class IngestedPage extends Model<IngestedPageAttributes, IngestedPageCreationAttributes> {
|
||||
@Column({ type: DataType.STRING(512), allowNull: false })
|
||||
declare url: string;
|
||||
}
|
||||
40
src/node/consumer/src/repository/models/ingestedTorrent.ts
Normal file
40
src/node/consumer/src/repository/models/ingestedTorrent.ts
Normal file
@@ -0,0 +1,40 @@
|
||||
import { Table, Column, Model, HasMany, DataType } from 'sequelize-typescript';
|
||||
import {IngestedTorrentAttributes, IngestedTorrentCreationAttributes} from "../interfaces/ingested_torrent_attributes";
|
||||
|
||||
const indexes = [
|
||||
{
|
||||
unique: true,
|
||||
name: 'ingested_torrent_unique_source_info_hash_constraint',
|
||||
fields: ['source', 'info_hash']
|
||||
}
|
||||
];
|
||||
|
||||
@Table({modelName: 'ingested_torrent', timestamps: true, indexes: indexes})
|
||||
export class IngestedTorrent extends Model<IngestedTorrentAttributes, IngestedTorrentCreationAttributes> {
|
||||
@Column({ type: DataType.STRING(512) })
|
||||
declare name: string;
|
||||
|
||||
@Column({ type: DataType.STRING(512) })
|
||||
declare source: string;
|
||||
|
||||
@Column({ type: DataType.STRING(32) })
|
||||
declare category: string;
|
||||
|
||||
@Column({ type: DataType.STRING(64) })
|
||||
declare info_hash: string;
|
||||
|
||||
@Column({ type: DataType.STRING(32) })
|
||||
declare size: string;
|
||||
|
||||
@Column({ type: DataType.INTEGER })
|
||||
declare seeders: number;
|
||||
|
||||
@Column({ type: DataType.INTEGER })
|
||||
declare leechers: number;
|
||||
|
||||
@Column({ type: DataType.STRING(32) })
|
||||
declare imdb: string;
|
||||
|
||||
@Column({ type: DataType.BOOLEAN, defaultValue: false })
|
||||
declare processed: boolean;
|
||||
}
|
||||
15
src/node/consumer/src/repository/models/provider.ts
Normal file
15
src/node/consumer/src/repository/models/provider.ts
Normal file
@@ -0,0 +1,15 @@
|
||||
import { Table, Column, Model, HasMany, DataType } from 'sequelize-typescript';
|
||||
import {ProviderAttributes, ProviderCreationAttributes} from "../interfaces/provider_attributes";
|
||||
|
||||
@Table({modelName: 'provider', timestamps: false})
|
||||
export class Provider extends Model<ProviderAttributes, ProviderCreationAttributes> {
|
||||
|
||||
@Column({ type: DataType.STRING(32), primaryKey: true })
|
||||
declare name: string;
|
||||
|
||||
@Column({ type: DataType.DATE })
|
||||
declare lastScraped: Date;
|
||||
|
||||
@Column({ type: DataType.STRING(128) })
|
||||
declare lastScrapedId: string;
|
||||
}
|
||||
10
src/node/consumer/src/repository/models/skipTorrent.ts
Normal file
10
src/node/consumer/src/repository/models/skipTorrent.ts
Normal file
@@ -0,0 +1,10 @@
|
||||
import { Table, Column, Model, HasMany, DataType } from 'sequelize-typescript';
|
||||
import {SkipTorrentAttributes, SkipTorrentCreationAttributes} from "../interfaces/skip_torrent_attributes";
|
||||
|
||||
|
||||
@Table({modelName: 'skip_torrent', timestamps: false})
|
||||
export class SkipTorrent extends Model<SkipTorrentAttributes, SkipTorrentCreationAttributes> {
|
||||
|
||||
@Column({ type: DataType.STRING(64), primaryKey: true })
|
||||
declare infoHash: string;
|
||||
}
|
||||
39
src/node/consumer/src/repository/models/subtitle.ts
Normal file
39
src/node/consumer/src/repository/models/subtitle.ts
Normal file
@@ -0,0 +1,39 @@
|
||||
import {Table, Column, Model, HasMany, DataType, BelongsTo, ForeignKey} from 'sequelize-typescript';
|
||||
import {SubtitleAttributes, SubtitleCreationAttributes} from "../interfaces/subtitle_attributes";
|
||||
import {File} from "./file";
|
||||
import {Torrent} from "./torrent";
|
||||
|
||||
const indexes = [
|
||||
{
|
||||
unique: true,
|
||||
name: 'subtitles_unique_subtitle_constraint',
|
||||
fields: [
|
||||
'infoHash',
|
||||
'fileIndex',
|
||||
'fileId'
|
||||
]
|
||||
},
|
||||
{ unique: false, fields: ['fileId'] }
|
||||
];
|
||||
|
||||
@Table({modelName: 'subtitle', timestamps: false, indexes: indexes})
|
||||
export class Subtitle extends Model<SubtitleAttributes, SubtitleCreationAttributes> {
|
||||
|
||||
@Column({ type: DataType.STRING(64), allowNull: false, onDelete: 'CASCADE' })
|
||||
declare infoHash: string;
|
||||
|
||||
@Column({ type: DataType.INTEGER, allowNull: false })
|
||||
declare fileIndex: number;
|
||||
|
||||
@Column({ type: DataType.BIGINT, allowNull: true, onDelete: 'SET NULL' })
|
||||
@ForeignKey(() => File)
|
||||
declare fileId?: number | null;
|
||||
|
||||
@Column({ type: DataType.STRING(512), allowNull: false })
|
||||
declare title: string;
|
||||
|
||||
@BelongsTo(() => File, { constraints: false, foreignKey: 'fileId' })
|
||||
file: File;
|
||||
|
||||
path: string;
|
||||
}
|
||||
56
src/node/consumer/src/repository/models/torrent.ts
Normal file
56
src/node/consumer/src/repository/models/torrent.ts
Normal file
@@ -0,0 +1,56 @@
|
||||
import { Table, Column, Model, HasMany, DataType } from 'sequelize-typescript';
|
||||
import {TorrentAttributes, TorrentCreationAttributes} from "../interfaces/torrent_attributes";
|
||||
import {Content} from "./content";
|
||||
import {File} from "./file";
|
||||
import {Subtitle} from "./subtitle";
|
||||
|
||||
@Table({modelName: 'torrent', timestamps: true})
|
||||
|
||||
export class Torrent extends Model<TorrentAttributes, TorrentCreationAttributes> {
|
||||
@Column({type: DataType.STRING(64), primaryKey: true})
|
||||
declare infoHash: string;
|
||||
|
||||
@Column({type: DataType.STRING(32), allowNull: false})
|
||||
declare provider: string;
|
||||
|
||||
@Column({type: DataType.STRING(512)})
|
||||
declare torrentId: string;
|
||||
|
||||
@Column({type: DataType.STRING(512), allowNull: false})
|
||||
declare title: string;
|
||||
|
||||
@Column({type: DataType.BIGINT})
|
||||
declare size: number;
|
||||
|
||||
@Column({type: DataType.STRING(16), allowNull: false})
|
||||
declare type: string;
|
||||
|
||||
@Column({type: DataType.DATE, allowNull: false})
|
||||
declare uploadDate: Date;
|
||||
|
||||
@Column({type: DataType.SMALLINT})
|
||||
declare seeders: number;
|
||||
|
||||
@Column({type: DataType.STRING(8000)})
|
||||
declare trackers: string;
|
||||
|
||||
@Column({type: DataType.STRING(4096)})
|
||||
declare languages: string;
|
||||
|
||||
@Column({type: DataType.STRING(16)})
|
||||
declare resolution: string;
|
||||
|
||||
@Column({type: DataType.BOOLEAN, allowNull: false, defaultValue: false})
|
||||
declare reviewed: boolean;
|
||||
|
||||
@Column({type: DataType.BOOLEAN, allowNull: false, defaultValue: false})
|
||||
declare opened: boolean;
|
||||
|
||||
@HasMany(() => Content, { foreignKey: 'infoHash', constraints: false })
|
||||
contents?: Content[];
|
||||
|
||||
@HasMany(() => File, { foreignKey: 'infoHash', constraints: false })
|
||||
files?: File[];
|
||||
|
||||
subtitles?: Subtitle[];
|
||||
}
|
||||
@@ -3,7 +3,7 @@
|
||||
"baseUrl": "./src",
|
||||
"checkJs": true,
|
||||
"isolatedModules": true,
|
||||
"lib": ["es6"],
|
||||
"lib": ["ESNext"],
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "node",
|
||||
"outDir": "./dist",
|
||||
@@ -13,9 +13,12 @@
|
||||
"rootDir": "./src",
|
||||
"skipLibCheck": true,
|
||||
"sourceMap": true,
|
||||
"target": "ES6",
|
||||
"target": "ESNext",
|
||||
"types": ["node"],
|
||||
"typeRoots": ["node_modules/@types", "src/@types"]
|
||||
"typeRoots": ["node_modules/@types", "src/@types"],
|
||||
"experimentalDecorators": true,
|
||||
"emitDecoratorMetadata": true,
|
||||
"allowSyntheticDefaultImports": true
|
||||
},
|
||||
"exclude": ["node_modules"]
|
||||
}
|
||||
Reference in New Issue
Block a user