tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Few fixes with regards to sequelize usage from services

Browse Source
This commit is contained in:
iPromKnight
2024-02-07 14:50:25 +00:00
committed by iPromKnight
parent 7fe9b64f66
commit 189fdd466e
7 changed files with 77 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/node/consumer/src/lib/interfaces/metadata_service.ts
View File

@@ -2,7 +2,7 @@ import {IMetaDataQuery} from "./metadata_query";
import {IMetadataResponse} from "./metadata_response"; import {IMetadataResponse} from "./metadata_response";
export interface IMetadataService { export interface IMetadataService {
getKitsuId(info: IMetaDataQuery): Promise<string | Error>; getKitsuId(info: IMetaDataQuery): Promise<number | Error>;
getImdbId(info: IMetaDataQuery): Promise<string | undefined>; getImdbId(info: IMetaDataQuery): Promise<string | undefined>;

2
src/node/consumer/src/lib/interfaces/parsed_torrent.ts
View File

@@ -6,7 +6,7 @@ export interface IParsedTorrent extends IParseTorrentTitleResult {
size?: number; size?: number;
isPack?: boolean; isPack?: boolean;
imdbId?: string | number; imdbId?: string | number;
kitsuId?: string | number; kitsuId?: number;
trackers?: string; trackers?: string;
provider?: string | null; provider?: string | null;
infoHash: string | null; infoHash: string | null;

3
src/node/consumer/src/lib/services/torrent_download_service.ts
View File

@@ -132,12 +132,13 @@ export class TorrentDownloadService implements ITorrentDownloadService {
title: file.name, title: file.name,
size: file.length, size: file.length,
fileIndex: file.fileIndex || 0, fileIndex: file.fileIndex || 0,
path: file.path,
infoHash: torrent.infoHash, infoHash: torrent.infoHash,
imdbId: torrent.imdbId.toString(), imdbId: torrent.imdbId.toString(),
imdbSeason: torrent.season || 0, imdbSeason: torrent.season || 0,
imdbEpisode: torrent.episode || 0, imdbEpisode: torrent.episode || 0,
kitsuId: parseInt(torrent.kitsuId?.toString()) || 0, kitsuId: parseInt(torrent.kitsuId?.toString()) || 0,
kitsuEpisode: torrent.episode || 0 kitsuEpisode: torrent.episode || 0,
}; };
return {...videoFile, ...parse(file.name)}; return {...videoFile, ...parse(file.name)};

40
src/node/consumer/src/lib/services/torrent_entries_service.ts
View File

@@ -4,7 +4,7 @@ import {TorrentType} from '../enums/torrent_types';
import {ITorrentFileCollection} from "../interfaces/torrent_file_collection"; import {ITorrentFileCollection} from "../interfaces/torrent_file_collection";
import {Torrent} from "../../repository/models/torrent"; import {Torrent} from "../../repository/models/torrent";
import {PromiseHelpers} from '../helpers/promises_helpers'; import {PromiseHelpers} from '../helpers/promises_helpers';
import {ITorrentAttributes} from "../../repository/interfaces/torrent_attributes"; import {ITorrentAttributes, ITorrentCreationAttributes} from "../../repository/interfaces/torrent_attributes";
import {File} from "../../repository/models/file"; import {File} from "../../repository/models/file";
import {Subtitle} from "../../repository/models/subtitle"; import {Subtitle} from "../../repository/models/subtitle";
import {ITorrentEntriesService} from "../interfaces/torrent_entries_service"; import {ITorrentEntriesService} from "../interfaces/torrent_entries_service";
@@ -15,6 +15,8 @@ import {ILoggingService} from "../interfaces/logging_service";
import {ITorrentFileService} from "../interfaces/torrent_file_service"; import {ITorrentFileService} from "../interfaces/torrent_file_service";
import {ITorrentSubtitleService} from "../interfaces/torrent_subtitle_service"; import {ITorrentSubtitleService} from "../interfaces/torrent_subtitle_service";
import {IDatabaseRepository} from "../../repository/interfaces/database_repository"; import {IDatabaseRepository} from "../../repository/interfaces/database_repository";
import {IIngestedTorrentCreationAttributes} from "../../repository/interfaces/ingested_torrent_attributes";
import {IFileCreationAttributes} from "../../repository/interfaces/file_attributes";
@injectable() @injectable()
export class TorrentEntriesService implements ITorrentEntriesService { export class TorrentEntriesService implements ITorrentEntriesService {
@@ -62,8 +64,8 @@ export class TorrentEntriesService implements ITorrentEntriesService {
year: titleInfo.year, year: titleInfo.year,
season: titleInfo.season, season: titleInfo.season,
}; };
torrent.kitsuId = await this.metadataService.getKitsuId(kitsuQuery)
.catch(() => undefined); await this.assignKitsuId(kitsuQuery, torrent);
} }
if (!torrent.imdbId && !torrent.kitsuId && !this.fileService.isPackTorrent(torrent)) { if (!torrent.imdbId && !torrent.kitsuId && !this.fileService.isPackTorrent(torrent)) {
@@ -84,20 +86,38 @@ export class TorrentEntriesService implements ITorrentEntriesService {
return; return;
} }
const newTorrent: Torrent = Torrent.build({ const newTorrent: ITorrentCreationAttributes = ({
...torrent, ...torrent,
contents: fileCollection.contents, contents: fileCollection.contents,
subtitles: fileCollection.subtitles subtitles: fileCollection.subtitles
}); });
return this.repository.createTorrent(newTorrent) return this.repository.createTorrent(newTorrent)
.then(() => PromiseHelpers.sequence(fileCollection.videos.map(video => () => { .then(() => PromiseHelpers.sequence(fileCollection.videos.map(video => () => {
const newVideo = File.build(video); const newVideo: IFileCreationAttributes = {...video, infoHash: video.infoHash, title: video.title};
if (!newVideo.kitsuId) {
newVideo.kitsuId = 0;
}
return this.repository.createFile(newVideo) return this.repository.createFile(newVideo)
}))) })))
.then(() => this.logger.info(`Created ${torrent.provider} entry for [${torrent.infoHash}] ${torrent.title}`)); .then(() => this.logger.info(`Created ${torrent.provider} entry for [${torrent.infoHash}] ${torrent.title}`));
}; };
private assignKitsuId = async (kitsuQuery: { year: number | string; season: number; title: string }, torrent: IParsedTorrent) => {
await this.metadataService.getKitsuId(kitsuQuery)
.then((result: number | Error) => {
if (typeof result === 'number') {
torrent.kitsuId = result;
} else {
torrent.kitsuId = 0;
}
})
.catch((error: Error) => {
this.logger.debug(`Failed getting kitsuId for ${torrent.title}`, error.message);
torrent.kitsuId = 0;
});
};
public createSkipTorrentEntry = async (torrent: Torrent) => this.repository.createSkipTorrent(torrent); public createSkipTorrentEntry = async (torrent: Torrent) => this.repository.createSkipTorrent(torrent);
public getStoredTorrentEntry = async (torrent: Torrent) => this.repository.getSkipTorrent(torrent.infoHash) public getStoredTorrentEntry = async (torrent: Torrent) => this.repository.getSkipTorrent(torrent.infoHash)
@@ -144,7 +164,7 @@ export class TorrentEntriesService implements ITorrentEntriesService {
} }
const notOpenedVideo = storedVideos.length === 1 && !Number.isInteger(storedVideos[0].fileIndex); const notOpenedVideo = storedVideos.length === 1 && !Number.isInteger(storedVideos[0].fileIndex);
const imdbId: string | undefined = PromiseHelpers.mostCommonValue(storedVideos.map(stored => stored.imdbId)); const imdbId: string | undefined = PromiseHelpers.mostCommonValue(storedVideos.map(stored => stored.imdbId));
const kitsuId: number | undefined = PromiseHelpers.mostCommonValue(storedVideos.map(stored => stored.kitsuId)); const kitsuId: number = PromiseHelpers.mostCommonValue(storedVideos.map(stored => stored.kitsuId || 0));
const fileCollection: ITorrentFileCollection = await this.fileService.parseTorrentFiles(torrent) const fileCollection: ITorrentFileCollection = await this.fileService.parseTorrentFiles(torrent)
.then(torrentContents => notOpenedVideo ? torrentContents : {...torrentContents, videos: storedVideos}) .then(torrentContents => notOpenedVideo ? torrentContents : {...torrentContents, videos: storedVideos})
@@ -187,7 +207,7 @@ export class TorrentEntriesService implements ITorrentEntriesService {
return Promise.resolve(); return Promise.resolve();
}) })
.then(() => PromiseHelpers.sequence(fileCollection.videos.map(video => () => { .then(() => PromiseHelpers.sequence(fileCollection.videos.map(video => () => {
const newVideo = File.build(video); const newVideo: IFileCreationAttributes = {...video, infoHash: video.infoHash, title: video.title};
return this.repository.createFile(newVideo) return this.repository.createFile(newVideo)
}))) })))
.then(() => this.logger.info(`Created contents for ${torrent.provider} [${torrent.infoHash}] ${torrent.title}`)) .then(() => this.logger.info(`Created contents for ${torrent.provider} [${torrent.infoHash}] ${torrent.title}`))
@@ -209,8 +229,8 @@ export class TorrentEntriesService implements ITorrentEntriesService {
private assignMetaIds = (fileCollection: ITorrentFileCollection, imdbId: string, kitsuId: number): ITorrentFileCollection => { private assignMetaIds = (fileCollection: ITorrentFileCollection, imdbId: string, kitsuId: number): ITorrentFileCollection => {
if (fileCollection.videos && fileCollection.videos.length) { if (fileCollection.videos && fileCollection.videos.length) {
fileCollection.videos.forEach(video => { fileCollection.videos.forEach(video => {
video.imdbId = imdbId; video.imdbId = imdbId || '';
video.kitsuId = kitsuId; video.kitsuId = kitsuId || 0;
}); });
} }

12
src/node/consumer/src/lib/services/torrent_file_service.ts
View File

@@ -191,7 +191,7 @@ export class TorrentFileService implements ITorrentFileService {
episode: file.episodes && file.episodes[index], episode: file.episodes && file.episodes[index],
kitsuEpisode: file.episodes && file.episodes[index], kitsuEpisode: file.episodes && file.episodes[index],
episodes: file.episodes, episodes: file.episodes,
kitsuId: parseInt(file.kitsuId.toString() || torrent.kitsuId.toString()), kitsuId: parseInt(file.kitsuId.toString() || torrent.kitsuId.toString()) || 0,
}))) })))
}; };
@@ -222,7 +222,7 @@ export class TorrentFileService implements ITorrentFileService {
title: file.path || file.title, title: file.path || file.title,
size: file.size, size: file.size,
imdbId: imdbId, imdbId: imdbId,
kitsuId: parseInt(kitsuId), kitsuId: parseInt(kitsuId) || 0,
episodes: undefined, episodes: undefined,
imdbSeason: undefined, imdbSeason: undefined,
imdbEpisode: undefined, imdbEpisode: undefined,
@@ -239,7 +239,7 @@ export class TorrentFileService implements ITorrentFileService {
title: file.path || file.title, title: file.path || file.title,
size: file.size, size: file.size,
imdbId: metadata.imdbId.toString() || imdbId, imdbId: metadata.imdbId.toString() || imdbId,
kitsuId: parseInt(metadata.kitsuId.toString() || kitsuId), kitsuId: parseInt(metadata.kitsuId.toString() || kitsuId) || 0,
imdbSeason: episodeVideo && metadata.imdbId ? episodeVideo.season : undefined, imdbSeason: episodeVideo && metadata.imdbId ? episodeVideo.season : undefined,
imdbEpisode: episodeVideo && metadata.imdbId | metadata.kitsuId ? episodeVideo.episode || episodeVideo.episode : undefined, imdbEpisode: episodeVideo && metadata.imdbId | metadata.kitsuId ? episodeVideo.episode || episodeVideo.episode : undefined,
kitsuEpisode: episodeVideo && metadata.imdbId | metadata.kitsuId ? episodeVideo.episode || episodeVideo.episode : undefined, kitsuEpisode: episodeVideo && metadata.imdbId | metadata.kitsuId ? episodeVideo.episode || episodeVideo.episode : undefined,
@@ -474,7 +474,7 @@ export class TorrentFileService implements ITorrentFileService {
if (seriesMapping[file.season]) { if (seriesMapping[file.season]) {
const seasonMapping = seriesMapping[file.season]; const seasonMapping = seriesMapping[file.season];
file.imdbId = metadata.imdbId.toString(); file.imdbId = metadata.imdbId.toString();
file.kitsuId = seasonMapping[file.episodes[0]] && seasonMapping[file.episodes[0]].kitsuId; file.kitsuId = seasonMapping[file.episodes[0]] && seasonMapping[file.episodes[0]].kitsuId || 0;
file.episodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].kitsuEpisode); file.episodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].kitsuEpisode);
} else if (seriesMapping[file.season - 1]) { } else if (seriesMapping[file.season - 1]) {
// sometimes a second season might be a continuation of the previous season // sometimes a second season might be a continuation of the previous season
@@ -492,7 +492,7 @@ export class TorrentFileService implements ITorrentFileService {
file.imdbId = metadata.imdbId.toString(); file.imdbId = metadata.imdbId.toString();
file.season = file.season - 1; file.season = file.season - 1;
file.episodes = file.episodes.map(ep => isAbsoluteOrder ? ep : ep + skippedCount); file.episodes = file.episodes.map(ep => isAbsoluteOrder ? ep : ep + skippedCount);
file.kitsuId = seasonMapping[file.episodes[0]].kitsuId; file.kitsuId = seasonMapping[file.episodes[0]].kitsuId || 0;
file.episodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].kitsuEpisode); file.episodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].kitsuEpisode);
} }
} else if (Object.values(seriesMapping).length === 1 && seriesMapping[1]) { } else if (Object.values(seriesMapping).length === 1 && seriesMapping[1]) {
@@ -500,7 +500,7 @@ export class TorrentFileService implements ITorrentFileService {
const seasonMapping = seriesMapping[1]; const seasonMapping = seriesMapping[1];
file.imdbId = metadata.imdbId.toString(); file.imdbId = metadata.imdbId.toString();
file.season = 1; file.season = 1;
file.kitsuId = seasonMapping[file.episodes[0]].kitsuId; file.kitsuId = seasonMapping[file.episodes[0]].kitsuId || 0;
file.episodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].kitsuEpisode); file.episodes = file.episodes.map(ep => seasonMapping[ep] && seasonMapping[ep].kitsuEpisode);
} }
}); });

48
src/node/consumer/src/repository/database_repository.ts
View File

@@ -10,13 +10,15 @@ import {IngestedTorrent} from "./models/ingestedTorrent";
import {Subtitle} from "./models/subtitle"; import {Subtitle} from "./models/subtitle";
import {Content} from "./models/content"; import {Content} from "./models/content";
import {SkipTorrent} from "./models/skipTorrent"; import {SkipTorrent} from "./models/skipTorrent";
import {IFileAttributes} from "./interfaces/file_attributes"; import {IFileAttributes, IFileCreationAttributes} from "./interfaces/file_attributes";
import {ITorrentAttributes} from "./interfaces/torrent_attributes"; import {ITorrentAttributes, ITorrentCreationAttributes} from "./interfaces/torrent_attributes";
import {IngestedPage} from "./models/ingestedPage"; import {IngestedPage} from "./models/ingestedPage";
import {ILoggingService} from "../lib/interfaces/logging_service"; import {ILoggingService} from "../lib/interfaces/logging_service";
import {IocTypes} from "../lib/models/ioc_types"; import {IocTypes} from "../lib/models/ioc_types";
import {inject, injectable} from "inversify"; import {inject, injectable} from "inversify";
import {IDatabaseRepository} from "./interfaces/database_repository"; import {IDatabaseRepository} from "./interfaces/database_repository";
import {IContentCreationAttributes} from "./interfaces/content_attributes";
import {ISubtitleCreationAttributes} from "./interfaces/subtitle_attributes";
@injectable() @injectable()
export class DatabaseRepository implements IDatabaseRepository { export class DatabaseRepository implements IDatabaseRepository {
@@ -112,7 +114,7 @@ export class DatabaseRepository implements IDatabaseRepository {
order: literal('random()') order: literal('random()')
}); });
public createTorrent = async (torrent: Torrent): Promise<void> => { public createTorrent = async (torrent: ITorrentCreationAttributes): Promise<void> => {
try { try {
await Torrent.upsert(torrent); await Torrent.upsert(torrent);
await this.createContents(torrent.infoHash, torrent.contents); await this.createContents(torrent.infoHash, torrent.contents);
@@ -136,22 +138,28 @@ export class DatabaseRepository implements IDatabaseRepository {
public deleteTorrent = async (infoHash: string): Promise<number> => await Torrent.destroy({where: {infoHash: infoHash}}); public deleteTorrent = async (infoHash: string): Promise<number> => await Torrent.destroy({where: {infoHash: infoHash}});
public createFile = async (file: File): Promise<void> => { public createFile = async (file: IFileCreationAttributes): Promise<void> => {
if (file.id) { try {
if (file.dataValues) { const operatingFile = File.build(file);
await file.save(); if (operatingFile.id) {
if (operatingFile.dataValues) {
await operatingFile.save();
} else {
await File.upsert(operatingFile);
}
await this.upsertSubtitles(operatingFile, operatingFile.subtitles);
} else { } else {
await File.upsert(file); if (operatingFile.subtitles && operatingFile.subtitles.length) {
operatingFile.subtitles = operatingFile.subtitles.map(subtitle => {
subtitle.title = subtitle.path;
return subtitle;
});
}
await File.create(file, {include: [Subtitle], ignoreDuplicates: true});
} }
await this.upsertSubtitles(file, file.subtitles); } catch (error) {
} else { this.logger.error(`Failed to create file: ${file.infoHash}`);
if (file.subtitles && file.subtitles.length) { this.logger.debug(error);
file.subtitles = file.subtitles.map(subtitle => {
subtitle.title = subtitle.path;
return subtitle;
});
}
await File.create(file, {include: [Subtitle], ignoreDuplicates: true});
} }
}; };
@@ -161,7 +169,7 @@ export class DatabaseRepository implements IDatabaseRepository {
public deleteFile = async (id: number): Promise<number> => File.destroy({where: {id: id}}); public deleteFile = async (id: number): Promise<number> => File.destroy({where: {id: id}});
public createSubtitles = async (infoHash: string, subtitles: Subtitle[]): Promise<void | Model<any, any>[]> => { public createSubtitles = async (infoHash: string, subtitles: ISubtitleCreationAttributes[]): Promise<void | Model<any, any>[]> => {
if (subtitles && subtitles.length) { if (subtitles && subtitles.length) {
return Subtitle.bulkCreate(subtitles.map(subtitle => ({infoHash, title: subtitle.path, ...subtitle}))); return Subtitle.bulkCreate(subtitles.map(subtitle => ({infoHash, title: subtitle.path, ...subtitle})));
} }
@@ -191,7 +199,7 @@ export class DatabaseRepository implements IDatabaseRepository {
public getUnassignedSubtitles = async (): Promise<Subtitle[]> => Subtitle.findAll({where: {fileId: null}}); public getUnassignedSubtitles = async (): Promise<Subtitle[]> => Subtitle.findAll({where: {fileId: null}});
public createContents = async (infoHash: string, contents: Content[]): Promise<void> => { public createContents = async (infoHash: string, contents: IContentCreationAttributes[]): Promise<void> => {
if (contents && contents.length) { if (contents && contents.length) {
await Content.bulkCreate(contents.map(content => ({infoHash, ...content})), {ignoreDuplicates: true}); await Content.bulkCreate(contents.map(content => ({infoHash, ...content})), {ignoreDuplicates: true});
await Torrent.update({opened: true}, {where: {infoHash: infoHash}, silent: true}); await Torrent.update({opened: true}, {where: {infoHash: infoHash}, silent: true});
@@ -208,7 +216,7 @@ export class DatabaseRepository implements IDatabaseRepository {
return result.dataValues as SkipTorrent; return result.dataValues as SkipTorrent;
}; };
public createSkipTorrent = async (torrent: Torrent): Promise<[SkipTorrent, boolean]> => SkipTorrent.upsert({infoHash: torrent.infoHash}); public createSkipTorrent = async (torrent: ITorrentCreationAttributes): Promise<[SkipTorrent, boolean]> => SkipTorrent.upsert({infoHash: torrent.infoHash});
private createDatabase = (): Sequelize => { private createDatabase = (): Sequelize => {
const newDatabase = new Sequelize( const newDatabase = new Sequelize(

16
src/node/consumer/src/repository/interfaces/database_repository.ts
View File

@@ -1,13 +1,15 @@
import {Provider} from "../models/provider"; import {Provider} from "../models/provider";
import {WhereOptions} from "sequelize"; import {WhereOptions} from "sequelize";
import {ITorrentAttributes} from "./torrent_attributes"; import {ITorrentAttributes, ITorrentCreationAttributes} from "./torrent_attributes";
import {Torrent} from "../models/torrent"; import {Torrent} from "../models/torrent";
import {IFileAttributes} from "./file_attributes"; import {IFileAttributes, IFileCreationAttributes} from "./file_attributes";
import {File} from "../models/file"; import {File} from "../models/file";
import {Subtitle} from "../models/subtitle"; import {Subtitle} from "../models/subtitle";
import {Model} from "sequelize-typescript"; import {Model} from "sequelize-typescript";
import {Content} from "../models/content"; import {Content} from "../models/content";
import {SkipTorrent} from "../models/skipTorrent"; import {SkipTorrent} from "../models/skipTorrent";
import {ISubtitleCreationAttributes} from "./subtitle_attributes";
import {IContentCreationAttributes} from "./content_attributes";
export interface IDatabaseRepository { export interface IDatabaseRepository {
connect(): Promise<void>; connect(): Promise<void>;
@@ -30,13 +32,13 @@ export interface IDatabaseRepository {
getNoContentsTorrents(): Promise<Torrent[]>; getNoContentsTorrents(): Promise<Torrent[]>;
createTorrent(torrent: Torrent): Promise<void>; createTorrent(torrent: ITorrentCreationAttributes): Promise<void>;
setTorrentSeeders(torrent: ITorrentAttributes, seeders: number): Promise<[number]>; setTorrentSeeders(torrent: ITorrentAttributes, seeders: number): Promise<[number]>;
deleteTorrent(infoHash: string): Promise<number>; deleteTorrent(infoHash: string): Promise<number>;
createFile(file: File): Promise<void>; createFile(file: IFileCreationAttributes): Promise<void>;
getFiles(infoHash: string): Promise<File[]>; getFiles(infoHash: string): Promise<File[]>;
@@ -44,7 +46,7 @@ export interface IDatabaseRepository {
deleteFile(id: number): Promise<number>; deleteFile(id: number): Promise<number>;
createSubtitles(infoHash: string, subtitles: Subtitle[]): Promise<void | Model<any, any>[]>; createSubtitles(infoHash: string, subtitles: ISubtitleCreationAttributes[]): Promise<void | Model<any, any>[]>;
upsertSubtitles(file: File, subtitles: Subtitle[]): Promise<void>; upsertSubtitles(file: File, subtitles: Subtitle[]): Promise<void>;
@@ -52,11 +54,11 @@ export interface IDatabaseRepository {
getUnassignedSubtitles(): Promise<Subtitle[]>; getUnassignedSubtitles(): Promise<Subtitle[]>;
createContents(infoHash: string, contents: Content[]): Promise<void>; createContents(infoHash: string, contents: IContentCreationAttributes[]): Promise<void>;
getContents(infoHash: string): Promise<Content[]>; getContents(infoHash: string): Promise<Content[]>;
getSkipTorrent(infoHash: string): Promise<SkipTorrent>; getSkipTorrent(infoHash: string): Promise<SkipTorrent>;
createSkipTorrent(torrent: Torrent): Promise<[SkipTorrent, boolean]>; createSkipTorrent(torrent: ITorrentCreationAttributes): Promise<[SkipTorrent, boolean]>;
} }