tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

meta data seems a bit iffy right now

Browse Source 
Also gone back to torrent-stream
WebTorrent seemed to be throwing the occasional engine crash
This commit is contained in:
iPromKnight
2024-02-07 21:42:31 +00:00
committed by iPromKnight
parent 028bb122e1
commit 6919622c30
9 changed files with 666 additions and 2721 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3272
src/node/consumer/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

11
src/node/consumer/package.json
View File

@@ -29,17 +29,14 @@
"reflect-metadata": "^0.2.1",
"sequelize": "^6.36.0",
"sequelize-typescript": "^2.1.6",
"utp-native": "^2.5.3",
"webtorrent": "^2.1.35"
"torrent-stream": "^1.2.1"
},
"devDependencies": {
"node-gyp": "^10.0.1",
"nodemon": "^3.0.3",
"@types/amqplib": "^0.10.4",
"@types/magnet-uri": "^5.1.5",
"@types/node": "^20.11.16",
"@types/torrent-stream": "^0.0.9",
"@types/validator": "^13.11.8",
"@types/webtorrent": "^0.109.7",
"@typescript-eslint/eslint-plugin": "^6.21.0",
"@typescript-eslint/parser": "^6.21.0",
"concurrently": "^8.2.2",
@@ -47,10 +44,12 @@
"eslint": "^8.56.0",
"eslint-plugin-import": "^2.29.1",
"eslint-plugin-import-helpers": "^1.3.1",
"node-gyp": "^10.0.1",
"nodemon": "^3.0.3",
"pino-pretty": "^10.3.1",
"tsx": "^4.7.0",
"ts-node": "^10.9.2",
"tsconfig-paths": "^4.2.0",
"tsx": "^4.7.0",
"typescript": "^5.3.3"
}
}

1
src/node/consumer/src/lib/interfaces/cinemeta_metadata.ts
View File

@@ -16,7 +16,6 @@ export interface ICinemetaMetaData {
dvdRelease?: null;
genre?: string[];
imdbRating?: string;
imdb_id?: string;
name?: string;
popularity?: number;
poster?: string;

1
src/node/consumer/src/lib/interfaces/common_video_metadata.ts
View File

@@ -5,4 +5,5 @@ export interface ICommonVideoMetadata {
title?: string;
name?: string;
id?: string;
imdb_id?: string;
}

1
src/node/consumer/src/lib/interfaces/kitsu_metadata.ts
View File

@@ -34,7 +34,6 @@ export interface IKitsuMeta {
export interface IKitsuVideo extends ICommonVideoMetadata {
imdbEpisode?: number;
imdbSeason?: number;
imdb_id?: string;
thumbnail?: string;
}

1
src/node/consumer/src/lib/models/configuration/torrent_config.ts
View File

@@ -1,5 +1,4 @@
export const torrentConfig = {
MAX_CONNECTIONS_PER_TORRENT: parseInt(process.env.MAX_CONNECTIONS_PER_TORRENT || "20", 10),
MAX_CONNECTIONS_OVERALL: parseInt(process.env.MAX_CONNECTIONS_OVERALL || "100", 10),
TIMEOUT: parseInt(process.env.TORRENT_TIMEOUT || "30000", 10)
};

29
src/node/consumer/src/lib/services/metadata_service.ts
View File

@@ -8,7 +8,7 @@ import {IMetaDataQuery} from "@interfaces/metadata_query";
import {IMetadataResponse} from "@interfaces/metadata_response";
import {IMetadataService} from "@interfaces/metadata_service";
import {IocTypes} from "@models/ioc_types";
import axios, {AxiosResponse} from 'axios';
import axios from 'axios';
import {ResultTypes, search} from 'google-sr';
import {inject, injectable} from "inversify";
import nameToImdb from 'name-to-imdb';
@@ -70,12 +70,12 @@ export class MetadataService implements IMetadataService {
const key = Number.isInteger(query.id) || query.id.toString().match(/^\d+$/) ? `kitsu:${query.id}` : query.id;
const metaType = query.type === TorrentType.Movie ? TorrentType.Movie : TorrentType.Series;
return this.cacheService.cacheWrapMetadata(key.toString(), () => this.requestMetadata(`${KITSU_URL}/meta/${metaType}/${key}.json`)
.catch(() => this.requestMetadata(`${CINEMETA_URL}/meta/${metaType}/${key}.json`))
return this.cacheService.cacheWrapMetadata(key.toString(), () => this.requestKitsuMetadata(`${KITSU_URL}/meta/${metaType}/${key}.json`)
.catch(() => this.requestCinemetaMetadata(`${CINEMETA_URL}/meta/${metaType}/${key}.json`))
.catch(() => {
// try different type in case there was a mismatch
const otherType = metaType === TorrentType.Movie ? TorrentType.Series : TorrentType.Movie;
return this.requestMetadata(`${CINEMETA_URL}/meta/${otherType}/${key}.json`)
return this.requestCinemetaMetadata(`${CINEMETA_URL}/meta/${otherType}/${key}.json`)
})
.catch((error) => {
throw new Error(`failed metadata query ${key} due: ${error.message}`);
@@ -101,23 +101,20 @@ export class MetadataService implements IMetadataService {
.replace(/\s{2,}/, ' ') // replace multiple spaces
.trim();
private requestMetadata = async (url: string): Promise<IMetadataResponse> => {
const response: AxiosResponse = await axios.get(url, {timeout: TIMEOUT});
let result: IMetadataResponse;
private requestKitsuMetadata = async (url: string): Promise<IMetadataResponse> => {
const response = await axios.get(url, {timeout: TIMEOUT});
const body = response.data;
if ('kitsu_id' in body.meta) {
result = this.handleKitsuResponse(body as IKitsuJsonResponse);
} else if ('imdb_id' in body.meta) {
result = this.handleCinemetaResponse(body as ICinemetaJsonResponse);
} else {
throw new Error('No valid metadata');
}
return this.handleKitsuResponse(body as IKitsuJsonResponse);
};
return result;
private requestCinemetaMetadata = async (url: string): Promise<IMetadataResponse> => {
const response = await axios.get(url, {timeout: TIMEOUT});
const body = response.data;
return this.handleCinemetaResponse(body as ICinemetaJsonResponse);
};
private handleCinemetaResponse = (body: ICinemetaJsonResponse): IMetadataResponse => ({
imdbId: parseInt(body.meta?.imdb_id || '0'),
imdbId: parseInt(body.meta?.id || '0'),
type: body.meta?.type,
title: body.meta?.name,
year: parseInt(body.meta?.year || '0'),

57
src/node/consumer/src/lib/services/torrent_download_service.ts
View File

@@ -11,7 +11,10 @@ import {configurationService} from '@services/configuration_service';
import {inject, injectable} from "inversify";
import {encode} from 'magnet-uri';
import {parse} from "parse-torrent-title";
import WebTorrent from "webtorrent";
// eslint-disable-next-line import/no-extraneous-dependencies
import * as torrentStream from "torrent-stream";
import TorrentEngine = TorrentStream.TorrentEngine;
import TorrentEngineOptions = TorrentStream.TorrentEngineOptions;
interface ITorrentFile {
name: string;
@@ -20,27 +23,19 @@ interface ITorrentFile {
fileIndex: number;
}
const clientOptions : WebTorrent.Options = {
maxConns: configurationService.torrentConfig.MAX_CONNECTIONS_OVERALL,
utp: false,
}
const torrentOptions: WebTorrent.TorrentOptions = {
skipVerify: true,
destroyStoreOnDestroy: true,
private: true,
maxWebConns: configurationService.torrentConfig.MAX_CONNECTIONS_PER_TORRENT,
}
@injectable()
export class TorrentDownloadService implements ITorrentDownloadService {
private torrentClient: WebTorrent.Instance;
private logger: ILoggingService;
private engineOptions: TorrentEngineOptions = {
connections: configurationService.torrentConfig.MAX_CONNECTIONS_PER_TORRENT,
uploads: 0,
verify: false,
dht: false,
tracker: true,
};
constructor(@inject(IocTypes.ILoggingService) logger: ILoggingService) {
this.logger = logger;
this.torrentClient = new WebTorrent(clientOptions);
this.torrentClient.on('error', errors => this.logClientErrors(errors));
}
public getTorrentFiles = async (torrent: IParsedTorrent, timeout: number = 30000): Promise<ITorrentFileCollection> => {
@@ -64,42 +59,30 @@ export class TorrentDownloadService implements ITorrentDownloadService {
const magnet = encode({infoHash: torrent.infoHash, announce: torrent.trackers!.split(',')});
return new Promise((resolve, reject) => {
this.logger.debug(`Adding torrent with infoHash ${torrent.infoHash} to webtorrent client...`);
const currentTorrent = this.torrentClient.add(magnet, torrentOptions);
this.logger.debug(`Adding torrent with infoHash ${torrent.infoHash} to torrent engine...`);
const timeoutId = setTimeout(() => {
this.removeTorrent(currentTorrent, torrent);
engine.destroy(() => {});
reject(new Error('No available connections for torrent!'));
}, timeout);
currentTorrent.on('ready', () => {
const files: ITorrentFile[] = currentTorrent.files.map((file, fileId) => ({
const engine: TorrentEngine = torrentStream.default(magnet, this.engineOptions);
engine.on("ready", () => {
const files: ITorrentFile[] = engine.files.map((file, fileId) => ({
fileIndex: fileId,
length: file.length,
name: file.name,
path: file.path,
}));
this.logger.debug(`Found ${files.length} files in torrent ${torrent.infoHash}`);
resolve(files);
clearTimeout(timeoutId);
this.removeTorrent(currentTorrent, torrent);
engine.destroy(() => {});
});
});
};
private removeTorrent = (currentTorrent: WebTorrent.Torrent, torrent: IParsedTorrent): void => {
try {
this.torrentClient.remove(currentTorrent, {destroyStore: true}, () => {
this.logger.debug(`Removed torrent ${torrent.infoHash} from webtorrent client...`);
});
} catch (error) {
this.logClientErrors(error);
}
};
private filterVideos = (torrent: IParsedTorrent, torrentFiles: ITorrentFile[]): IFileAttributes[] => {
if (torrentFiles.length === 1 && !Number.isInteger(torrentFiles[0].fileIndex)) {
return [this.mapTorrentFileToFileAttributes(torrent, torrentFiles[0])];
@@ -179,9 +162,5 @@ export class TorrentDownloadService implements ITorrentDownloadService {
path: file.path,
size: file.length,
});
private logClientErrors(errors: Error | string | unknown): void {
this.logger.error(`Error in webtorrent client: ${errors}`);
}
}

2
src/node/consumer/src/lib/services/torrent_file_service.ts
View File

@@ -56,8 +56,6 @@ export class TorrentFileService implements ITorrentFileService {
.catch(() => undefined);
if (metadata === undefined || metadata instanceof Error) {
this.logger.warn(`Failed to retrieve metadata for torrent ${torrent.title}`);
this.logger.debug(`Metadata Error: ${torrent.title}`, metadata);
return Promise.reject(new Error('Failed to retrieve metadata'));
}