diff --git a/.gitignore b/.gitignore index 1c58884..2cafa8e 100644 --- a/.gitignore +++ b/.gitignore @@ -612,3 +612,7 @@ fabric.properties # Mac directory indexes .DS_Store deployment/docker/stack.env + +src/producer/src/python/ +src/debrid-collector/python/ +src/qbit-collector/python/ diff --git a/deployment/docker/docker-compose.yaml b/deployment/docker/docker-compose.yaml index 17c8e1b..419b77a 100644 --- a/deployment/docker/docker-compose.yaml +++ b/deployment/docker/docker-compose.yaml @@ -94,7 +94,7 @@ services: condition: service_healthy env_file: stack.env hostname: knightcrawler-addon - image: gabisonfire/knightcrawler-addon:2.0.17 + image: gabisonfire/knightcrawler-addon:2.0.18 labels: logging: promtail networks: @@ -117,7 +117,7 @@ services: redis: condition: service_healthy env_file: stack.env - image: gabisonfire/knightcrawler-consumer:2.0.17 + image: gabisonfire/knightcrawler-consumer:2.0.18 labels: logging: promtail networks: @@ -138,7 +138,7 @@ services: redis: condition: service_healthy env_file: stack.env - image: gabisonfire/knightcrawler-debrid-collector:2.0.17 + image: gabisonfire/knightcrawler-debrid-collector:2.0.18 labels: logging: promtail networks: @@ -152,7 +152,7 @@ services: migrator: condition: service_completed_successfully env_file: stack.env - image: gabisonfire/knightcrawler-metadata:2.0.17 + image: gabisonfire/knightcrawler-metadata:2.0.18 networks: - knightcrawler-network restart: "no" @@ -163,7 +163,7 @@ services: postgres: condition: service_healthy env_file: stack.env - image: gabisonfire/knightcrawler-migrator:2.0.17 + image: gabisonfire/knightcrawler-migrator:2.0.18 networks: - knightcrawler-network restart: "no" @@ -182,7 +182,7 @@ services: redis: condition: service_healthy env_file: stack.env - image: gabisonfire/knightcrawler-producer:2.0.17 + image: gabisonfire/knightcrawler-producer:2.0.18 labels: logging: promtail networks: @@ -207,7 +207,7 @@ services: deploy: replicas: ${QBIT_REPLICAS:-0} env_file: stack.env - image: gabisonfire/knightcrawler-qbit-collector:2.0.17 + image: gabisonfire/knightcrawler-qbit-collector:2.0.18 labels: logging: promtail networks: diff --git a/deployment/docker/src/components/knightcrawler.yaml b/deployment/docker/src/components/knightcrawler.yaml index 3441e69..6e999f5 100644 --- a/deployment/docker/src/components/knightcrawler.yaml +++ b/deployment/docker/src/components/knightcrawler.yaml @@ -20,7 +20,7 @@ x-depends: &knightcrawler-app-depends services: metadata: - image: gabisonfire/knightcrawler-metadata:2.0.17 + image: gabisonfire/knightcrawler-metadata:2.0.18 env_file: ../../.env networks: - knightcrawler-network @@ -30,7 +30,7 @@ services: condition: service_completed_successfully migrator: - image: gabisonfire/knightcrawler-migrator:2.0.17 + image: gabisonfire/knightcrawler-migrator:2.0.18 env_file: ../../.env networks: - knightcrawler-network @@ -40,7 +40,7 @@ services: condition: service_healthy addon: - image: gabisonfire/knightcrawler-addon:2.0.17 + image: gabisonfire/knightcrawler-addon:2.0.18 <<: [*knightcrawler-app, *knightcrawler-app-depends] restart: unless-stopped hostname: knightcrawler-addon @@ -48,22 +48,22 @@ services: - "7000:7000" consumer: - image: gabisonfire/knightcrawler-consumer:2.0.17 + image: gabisonfire/knightcrawler-consumer:2.0.18 <<: [*knightcrawler-app, *knightcrawler-app-depends] restart: unless-stopped debridcollector: - image: gabisonfire/knightcrawler-debrid-collector:2.0.17 + image: gabisonfire/knightcrawler-debrid-collector:2.0.18 <<: [*knightcrawler-app, *knightcrawler-app-depends] restart: unless-stopped producer: - image: gabisonfire/knightcrawler-producer:2.0.17 + image: gabisonfire/knightcrawler-producer:2.0.18 <<: [*knightcrawler-app, *knightcrawler-app-depends] restart: unless-stopped qbitcollector: - image: gabisonfire/knightcrawler-qbit-collector:2.0.17 + image: gabisonfire/knightcrawler-qbit-collector:2.0.18 <<: [*knightcrawler-app, *knightcrawler-app-depends] restart: unless-stopped depends_on: diff --git a/src/debrid-collector/DebridCollector.csproj b/src/debrid-collector/DebridCollector.csproj index 21ce4fa..8ad2c73 100644 --- a/src/debrid-collector/DebridCollector.csproj +++ b/src/debrid-collector/DebridCollector.csproj @@ -17,7 +17,6 @@ - @@ -29,10 +28,30 @@ Always + + Always + + + + + + + + + Always + + + + + + + + + diff --git a/src/debrid-collector/DebridCollector.sln b/src/debrid-collector/DebridCollector.sln index a3dfeb0..d941188 100644 --- a/src/debrid-collector/DebridCollector.sln +++ b/src/debrid-collector/DebridCollector.sln @@ -6,6 +6,12 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedContracts", "..\share EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "shared", "shared", "{2C0A0F53-28E6-404F-9EFE-DADFBEF8338B}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "eng", "eng", "{72A042C3-B4F3-45C5-AC20-041FE8F41EFC}" + ProjectSection(SolutionItems) = preProject + eng\install-python-reqs.ps1 = eng\install-python-reqs.ps1 + eng\install-python-reqs.sh = eng\install-python-reqs.sh + EndProjectSection +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU diff --git a/src/debrid-collector/Dockerfile b/src/debrid-collector/Dockerfile index baab5d7..50f04a0 100644 --- a/src/debrid-collector/Dockerfile +++ b/src/debrid-collector/Dockerfile @@ -9,12 +9,23 @@ RUN dotnet restore -a $TARGETARCH RUN dotnet publish -c Release --no-restore -o /src/out -a $TARGETARCH -FROM mcr.microsoft.com/dotnet/aspnet:8.0-alpine +FROM mcr.microsoft.com/dotnet/aspnet:8.0-alpine3.19 WORKDIR /app + +ENV PYTHONUNBUFFERED=1 + +RUN apk add --update --no-cache python3=~3.11.8-r0 py3-pip && ln -sf python3 /usr/bin/python + COPY --from=build /src/out . + +RUN rm -rf /app/python && mkdir -p /app/python + +RUN pip3 install -r /app/requirements.txt -t /app/python + RUN addgroup -S debrid && adduser -S -G debrid debrid USER debrid HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 \ CMD pgrep -f dotnet || exit 1 +ENV PYTHONNET_PYDLL=/usr/lib/libpython3.11.so.1.0 ENTRYPOINT ["dotnet", "DebridCollector.dll"] diff --git a/src/debrid-collector/Extensions/ServiceCollectionExtensions.cs b/src/debrid-collector/Extensions/ServiceCollectionExtensions.cs index 0dfb73a..be3b07a 100644 --- a/src/debrid-collector/Extensions/ServiceCollectionExtensions.cs +++ b/src/debrid-collector/Extensions/ServiceCollectionExtensions.cs @@ -1,5 +1,3 @@ -using DebridCollector.Features.Configuration; - namespace DebridCollector.Extensions; public static class ServiceCollectionExtensions @@ -17,7 +15,8 @@ public static class ServiceCollectionExtensions var serviceConfiguration = services.LoadConfigurationFromEnv(); services.AddRealDebridClient(serviceConfiguration); - services.AddSingleton(); + services.RegisterPythonEngine(); + services.AddSingleton(); services.AddHostedService(); return services; diff --git a/src/debrid-collector/Features/Debrid/ServiceCollectionExtensions.cs b/src/debrid-collector/Features/Debrid/ServiceCollectionExtensions.cs index dbd1f9b..87ac6ee 100644 --- a/src/debrid-collector/Features/Debrid/ServiceCollectionExtensions.cs +++ b/src/debrid-collector/Features/Debrid/ServiceCollectionExtensions.cs @@ -1,6 +1,4 @@ -using DebridCollector.Features.Configuration; - -namespace DebridCollector.Features.Debrid; +namespace DebridCollector.Features.Debrid; public static class ServiceCollectionExtensions { diff --git a/src/debrid-collector/Features/Worker/DebridMetaToTorrentMeta.cs b/src/debrid-collector/Features/Worker/DebridMetaToTorrentMeta.cs index ce2935a..db83e2c 100644 --- a/src/debrid-collector/Features/Worker/DebridMetaToTorrentMeta.cs +++ b/src/debrid-collector/Features/Worker/DebridMetaToTorrentMeta.cs @@ -3,10 +3,11 @@ namespace DebridCollector.Features.Worker; public static class DebridMetaToTorrentMeta { public static IReadOnlyList MapMetadataToFilesCollection( - IParseTorrentTitle torrentTitle, + IRankTorrentName rankTorrentName, Torrent torrent, string ImdbId, - FileDataDictionary Metadata) + FileDataDictionary Metadata, + ILogger logger) { try { @@ -26,23 +27,30 @@ public static class DebridMetaToTorrentMeta Size = metadataEntry.Value.Filesize.GetValueOrDefault(), }; - var parsedTitle = torrentTitle.Parse(file.Title); + var parsedTitle = rankTorrentName.Parse(file.Title, false); - file.ImdbSeason = parsedTitle.Seasons.FirstOrDefault(); - file.ImdbEpisode = parsedTitle.Episodes.FirstOrDefault(); + if (!parsedTitle.Success) + { + logger.LogWarning("Failed to parse title {Title} for metadata mapping", file.Title); + continue; + } + + file.ImdbSeason = parsedTitle.Response?.Season?.FirstOrDefault() ?? 0; + file.ImdbEpisode = parsedTitle.Response?.Episode?.FirstOrDefault() ?? 0; files.Add(file); } return files; } - catch (Exception) + catch (Exception ex) { + logger.LogWarning("Failed to map metadata to files collection: {Exception}", ex.Message); return []; } } - public static async Task> MapMetadataToSubtitlesCollection(IDataStorage storage, string InfoHash, FileDataDictionary Metadata) + public static async Task> MapMetadataToSubtitlesCollection(IDataStorage storage, string InfoHash, FileDataDictionary Metadata, ILogger logger) { try { @@ -74,8 +82,9 @@ public static class DebridMetaToTorrentMeta return files; } - catch (Exception) + catch (Exception ex) { + logger.LogWarning("Failed to map metadata to subtitles collection: {Exception}", ex.Message); return []; } } diff --git a/src/debrid-collector/Features/Worker/InfohashMetadataSagaStateMachine.cs b/src/debrid-collector/Features/Worker/InfohashMetadataSagaStateMachine.cs index d726dab..a3bea22 100644 --- a/src/debrid-collector/Features/Worker/InfohashMetadataSagaStateMachine.cs +++ b/src/debrid-collector/Features/Worker/InfohashMetadataSagaStateMachine.cs @@ -53,6 +53,12 @@ public class InfohashMetadataSagaStateMachine : MassTransitStateMachine { + if (!context.Message.WithFiles) + { + logger.LogInformation("No files written for torrent {InfoHash} in Saga {SagaId}", context.Saga.Torrent.InfoHash, context.Saga.CorrelationId); + return; + } + logger.LogInformation("Metadata Written for torrent {InfoHash} in Saga {SagaId}", context.Saga.Torrent.InfoHash, context.Saga.CorrelationId); }) .TransitionTo(Completed) diff --git a/src/debrid-collector/Features/Worker/Requests.cs b/src/debrid-collector/Features/Worker/Requests.cs index 6341da5..1d10e2f 100644 --- a/src/debrid-collector/Features/Worker/Requests.cs +++ b/src/debrid-collector/Features/Worker/Requests.cs @@ -16,7 +16,7 @@ public record WriteMetadata(Torrent Torrent, TorrentMetadataResponse Metadata, s } [EntityName("metadata-written-debrid-colloctor")] -public record MetadataWritten(TorrentMetadataResponse Metadata) : CorrelatedBy +public record MetadataWritten(TorrentMetadataResponse Metadata, bool WithFiles) : CorrelatedBy { public Guid CorrelationId { get; init; } = Metadata.CorrelationId; } \ No newline at end of file diff --git a/src/debrid-collector/Features/Worker/WriteMetadataConsumer.cs b/src/debrid-collector/Features/Worker/WriteMetadataConsumer.cs index 7423966..b55430f 100644 --- a/src/debrid-collector/Features/Worker/WriteMetadataConsumer.cs +++ b/src/debrid-collector/Features/Worker/WriteMetadataConsumer.cs @@ -1,25 +1,28 @@ namespace DebridCollector.Features.Worker; -public class WriteMetadataConsumer(IParseTorrentTitle parseTorrentTitle, IDataStorage dataStorage) : IConsumer +public class WriteMetadataConsumer(IRankTorrentName rankTorrentName, IDataStorage dataStorage, ILogger logger) : IConsumer { public async Task Consume(ConsumeContext context) { var request = context.Message; - var torrentFiles = DebridMetaToTorrentMeta.MapMetadataToFilesCollection(parseTorrentTitle, request.Torrent, request.ImdbId, request.Metadata.Metadata); + var torrentFiles = DebridMetaToTorrentMeta.MapMetadataToFilesCollection(rankTorrentName, request.Torrent, request.ImdbId, request.Metadata.Metadata, logger); - if (torrentFiles.Any()) + if (!torrentFiles.Any()) { - await dataStorage.InsertFiles(torrentFiles); - - var subtitles = await DebridMetaToTorrentMeta.MapMetadataToSubtitlesCollection(dataStorage, request.Torrent.InfoHash, request.Metadata.Metadata); - - if (subtitles.Any()) - { - await dataStorage.InsertSubtitles(subtitles); - } + await context.Publish(new MetadataWritten(request.Metadata, false)); + return; } - await context.Publish(new MetadataWritten(request.Metadata)); + await dataStorage.InsertFiles(torrentFiles); + + var subtitles = await DebridMetaToTorrentMeta.MapMetadataToSubtitlesCollection(dataStorage, request.Torrent.InfoHash, request.Metadata.Metadata, logger); + + if (subtitles.Any()) + { + await dataStorage.InsertSubtitles(subtitles); + } + + await context.Publish(new MetadataWritten(request.Metadata, true)); } } \ No newline at end of file diff --git a/src/debrid-collector/GlobalUsings.cs b/src/debrid-collector/GlobalUsings.cs index 0d1ada4..69c08a4 100644 --- a/src/debrid-collector/GlobalUsings.cs +++ b/src/debrid-collector/GlobalUsings.cs @@ -4,17 +4,18 @@ global using System.Text.Json; global using System.Text.Json.Serialization; global using System.Threading.Channels; global using DebridCollector.Extensions; +global using DebridCollector.Features.Configuration; global using DebridCollector.Features.Debrid; global using DebridCollector.Features.Worker; global using MassTransit; -global using MassTransit.Mediator; global using Microsoft.AspNetCore.Builder; global using Microsoft.Extensions.DependencyInjection; global using Polly; global using Polly.Extensions.Http; -global using PromKnight.ParseTorrentTitle; global using SharedContracts.Configuration; global using SharedContracts.Dapper; global using SharedContracts.Extensions; global using SharedContracts.Models; +global using SharedContracts.Python; +global using SharedContracts.Python.RTN; global using SharedContracts.Requests; \ No newline at end of file diff --git a/src/debrid-collector/eng/install-python-reqs.ps1 b/src/debrid-collector/eng/install-python-reqs.ps1 new file mode 100644 index 0000000..38acc13 --- /dev/null +++ b/src/debrid-collector/eng/install-python-reqs.ps1 @@ -0,0 +1,2 @@ +mkdir -p ../python +python -m pip install -r ../requirements.txt -t ../python/ \ No newline at end of file diff --git a/src/debrid-collector/eng/install-python-reqs.sh b/src/debrid-collector/eng/install-python-reqs.sh new file mode 100644 index 0000000..ab24a74 --- /dev/null +++ b/src/debrid-collector/eng/install-python-reqs.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +rm -rf ../python +mkdir -p ../python +python3 -m pip install -r ../requirements.txt -t ../python/ \ No newline at end of file diff --git a/src/debrid-collector/requirements.txt b/src/debrid-collector/requirements.txt new file mode 100644 index 0000000..aaa299e --- /dev/null +++ b/src/debrid-collector/requirements.txt @@ -0,0 +1 @@ +rank-torrent-name==0.2.5 \ No newline at end of file diff --git a/src/producer/src/requirements.txt b/src/producer/src/requirements.txt index 7d700a4..1f2a3a6 100644 --- a/src/producer/src/requirements.txt +++ b/src/producer/src/requirements.txt @@ -1 +1 @@ -rank-torrent-name==0.1.8 \ No newline at end of file +rank-torrent-name==0.2.5 \ No newline at end of file diff --git a/src/qbit-collector/Dockerfile b/src/qbit-collector/Dockerfile index 8e21adc..bb1fb62 100644 --- a/src/qbit-collector/Dockerfile +++ b/src/qbit-collector/Dockerfile @@ -9,12 +9,23 @@ RUN dotnet restore -a $TARGETARCH RUN dotnet publish -c Release --no-restore -o /src/out -a $TARGETARCH -FROM mcr.microsoft.com/dotnet/aspnet:8.0-alpine +FROM mcr.microsoft.com/dotnet/aspnet:8.0-alpine3.19 WORKDIR /app + +ENV PYTHONUNBUFFERED=1 + +RUN apk add --update --no-cache python3=~3.11.8-r0 py3-pip && ln -sf python3 /usr/bin/python + COPY --from=build /src/out . + +RUN rm -rf /app/python && mkdir -p /app/python + +RUN pip3 install -r /app/requirements.txt -t /app/python + RUN addgroup -S qbit && adduser -S -G qbit qbit USER qbit HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 \ CMD pgrep -f dotnet || exit 1 +ENV PYTHONNET_PYDLL=/usr/lib/libpython3.11.so.1.0 ENTRYPOINT ["dotnet", "QBitCollector.dll"] diff --git a/src/qbit-collector/Extensions/ServiceCollectionExtensions.cs b/src/qbit-collector/Extensions/ServiceCollectionExtensions.cs index 3f3f41a..5836567 100644 --- a/src/qbit-collector/Extensions/ServiceCollectionExtensions.cs +++ b/src/qbit-collector/Extensions/ServiceCollectionExtensions.cs @@ -13,7 +13,8 @@ public static class ServiceCollectionExtensions internal static IServiceCollection AddServiceConfiguration(this IServiceCollection services) { services.AddQBitTorrentClient(); - services.AddSingleton(); + services.RegisterPythonEngine(); + services.AddSingleton(); services.AddSingleton(); services.AddHttpClient(); services.AddSingleton(); diff --git a/src/qbit-collector/Features/Worker/QbitMetaToTorrentMeta.cs b/src/qbit-collector/Features/Worker/QbitMetaToTorrentMeta.cs index ac250fb..73fc56e 100644 --- a/src/qbit-collector/Features/Worker/QbitMetaToTorrentMeta.cs +++ b/src/qbit-collector/Features/Worker/QbitMetaToTorrentMeta.cs @@ -3,10 +3,11 @@ namespace QBitCollector.Features.Worker; public static class QbitMetaToTorrentMeta { public static IReadOnlyList MapMetadataToFilesCollection( - IParseTorrentTitle torrentTitle, + IRankTorrentName rankTorrentName, Torrent torrent, string ImdbId, - IReadOnlyList Metadata) + IReadOnlyList Metadata, + ILogger logger) { try { @@ -24,23 +25,31 @@ public static class QbitMetaToTorrentMeta Size = metadataEntry.Size, }; - var parsedTitle = torrentTitle.Parse(file.Title); + var parsedTitle = rankTorrentName.Parse(file.Title, false); + + if (!parsedTitle.Success) + { + logger.LogWarning("Failed to parse title {Title} for metadata mapping", file.Title); + continue; + } - file.ImdbSeason = parsedTitle.Seasons.FirstOrDefault(); - file.ImdbEpisode = parsedTitle.Episodes.FirstOrDefault(); + file.ImdbSeason = parsedTitle.Response?.Season?.FirstOrDefault() ?? 0; + file.ImdbEpisode = parsedTitle.Response?.Episode?.FirstOrDefault() ?? 0; files.Add(file); } return files; } - catch (Exception) + catch (Exception ex) { + logger.LogWarning("Failed to map metadata to files collection: {Exception}", ex.Message); return []; } } - public static async Task> MapMetadataToSubtitlesCollection(IDataStorage storage, string InfoHash, IReadOnlyList Metadata) + public static async Task> MapMetadataToSubtitlesCollection(IDataStorage storage, string InfoHash, IReadOnlyList Metadata, + ILogger logger) { try { @@ -70,8 +79,9 @@ public static class QbitMetaToTorrentMeta return files; } - catch (Exception) + catch (Exception ex) { + logger.LogWarning("Failed to map metadata to subtitles collection: {Exception}", ex.Message); return []; } } diff --git a/src/qbit-collector/Features/Worker/QbitMetadataSagaStateMachine.cs b/src/qbit-collector/Features/Worker/QbitMetadataSagaStateMachine.cs index c7c5e7f..5dcd842 100644 --- a/src/qbit-collector/Features/Worker/QbitMetadataSagaStateMachine.cs +++ b/src/qbit-collector/Features/Worker/QbitMetadataSagaStateMachine.cs @@ -53,6 +53,12 @@ public class QbitMetadataSagaStateMachine : MassTransitStateMachine { + if (!context.Message.WithFiles) + { + logger.LogInformation("No files written for torrent {InfoHash} in Saga {SagaId}", context.Saga.Torrent.InfoHash, context.Saga.CorrelationId); + return; + } + logger.LogInformation("Metadata Written for torrent {InfoHash} in Saga {SagaId}", context.Saga.Torrent.InfoHash, context.Saga.CorrelationId); }) .TransitionTo(Completed) diff --git a/src/qbit-collector/Features/Worker/Requests.cs b/src/qbit-collector/Features/Worker/Requests.cs index 7cfdbb7..2c4bb97 100644 --- a/src/qbit-collector/Features/Worker/Requests.cs +++ b/src/qbit-collector/Features/Worker/Requests.cs @@ -16,7 +16,7 @@ public record WriteQbitMetadata(Torrent Torrent, QBitMetadataResponse Metadata, } [EntityName("metadata-written-qbit-collector")] -public record QbitMetadataWritten(QBitMetadataResponse Metadata) : CorrelatedBy +public record QbitMetadataWritten(QBitMetadataResponse Metadata, bool WithFiles) : CorrelatedBy { public Guid CorrelationId { get; init; } = Metadata.CorrelationId; diff --git a/src/qbit-collector/Features/Worker/WriteQbitMetadataConsumer.cs b/src/qbit-collector/Features/Worker/WriteQbitMetadataConsumer.cs index 4096395..b2e0b4d 100644 --- a/src/qbit-collector/Features/Worker/WriteQbitMetadataConsumer.cs +++ b/src/qbit-collector/Features/Worker/WriteQbitMetadataConsumer.cs @@ -1,25 +1,30 @@ namespace QBitCollector.Features.Worker; -public class WriteQbitMetadataConsumer(IParseTorrentTitle parseTorrentTitle, IDataStorage dataStorage) : IConsumer +public class WriteQbitMetadataConsumer(IRankTorrentName rankTorrentName, IDataStorage dataStorage, ILogger logger) : IConsumer { public async Task Consume(ConsumeContext context) { var request = context.Message; - - var torrentFiles = QbitMetaToTorrentMeta.MapMetadataToFilesCollection(parseTorrentTitle, request.Torrent, request.ImdbId, request.Metadata.Metadata); - if (torrentFiles.Any()) + var torrentFiles = QbitMetaToTorrentMeta.MapMetadataToFilesCollection( + rankTorrentName, request.Torrent, request.ImdbId, request.Metadata.Metadata, logger); + + if (!torrentFiles.Any()) { - await dataStorage.InsertFiles(torrentFiles); - - var subtitles = await QbitMetaToTorrentMeta.MapMetadataToSubtitlesCollection(dataStorage, request.Torrent.InfoHash, request.Metadata.Metadata); - - if (subtitles.Any()) - { - await dataStorage.InsertSubtitles(subtitles); - } + await context.Publish(new QbitMetadataWritten(request.Metadata, false)); + return; } - - await context.Publish(new QbitMetadataWritten(request.Metadata)); + + await dataStorage.InsertFiles(torrentFiles); + + var subtitles = await QbitMetaToTorrentMeta.MapMetadataToSubtitlesCollection( + dataStorage, request.Torrent.InfoHash, request.Metadata.Metadata, logger); + + if (subtitles.Any()) + { + await dataStorage.InsertSubtitles(subtitles); + } + + await context.Publish(new QbitMetadataWritten(request.Metadata, true)); } } \ No newline at end of file diff --git a/src/qbit-collector/GlobalUsings.cs b/src/qbit-collector/GlobalUsings.cs index 27fe7bd..5afd145 100644 --- a/src/qbit-collector/GlobalUsings.cs +++ b/src/qbit-collector/GlobalUsings.cs @@ -1,17 +1,11 @@ // Global using directives global using System.Text.Json; -global using System.Text.Json.Serialization; -global using System.Threading.Channels; global using MassTransit; -global using MassTransit.Mediator; global using Microsoft.AspNetCore.Builder; global using Microsoft.Extensions.Caching.Distributed; global using Microsoft.Extensions.Caching.Memory; global using Microsoft.Extensions.DependencyInjection; -global using Polly; -global using Polly.Extensions.Http; -global using PromKnight.ParseTorrentTitle; global using QBitCollector.Extensions; global using QBitCollector.Features.Qbit; global using QBitCollector.Features.Trackers; @@ -21,4 +15,6 @@ global using SharedContracts.Configuration; global using SharedContracts.Dapper; global using SharedContracts.Extensions; global using SharedContracts.Models; +global using SharedContracts.Python; +global using SharedContracts.Python.RTN; global using SharedContracts.Requests; \ No newline at end of file diff --git a/src/qbit-collector/QBitCollector.csproj b/src/qbit-collector/QBitCollector.csproj index 9e53242..6c018bf 100644 --- a/src/qbit-collector/QBitCollector.csproj +++ b/src/qbit-collector/QBitCollector.csproj @@ -18,7 +18,6 @@ - @@ -31,10 +30,30 @@ Always + + + + Always + + + + + Always + + + + + + + + + + + diff --git a/src/qbit-collector/QBitCollector.sln b/src/qbit-collector/QBitCollector.sln index 7bb998a..1553d5b 100644 --- a/src/qbit-collector/QBitCollector.sln +++ b/src/qbit-collector/QBitCollector.sln @@ -6,6 +6,12 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "shared", "shared", "{2C0A0F EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "QBitCollector", "QBitCollector.csproj", "{1EF124BE-6EBE-4D9E-846C-FFF814999F3B}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "eng", "eng", "{2F2EA33A-1303-405D-939B-E9394D262BC9}" + ProjectSection(SolutionItems) = preProject + eng\install-python-reqs.ps1 = eng\install-python-reqs.ps1 + eng\install-python-reqs.sh = eng\install-python-reqs.sh + EndProjectSection +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU diff --git a/src/qbit-collector/eng/install-python-reqs.ps1 b/src/qbit-collector/eng/install-python-reqs.ps1 new file mode 100644 index 0000000..68c6563 --- /dev/null +++ b/src/qbit-collector/eng/install-python-reqs.ps1 @@ -0,0 +1,3 @@ +Remove-Item -Recurse -Force ../python +mkdir -p ../python +python -m pip install -r ../requirements.txt -t ../python/ \ No newline at end of file diff --git a/src/qbit-collector/eng/install-python-reqs.sh b/src/qbit-collector/eng/install-python-reqs.sh new file mode 100644 index 0000000..ab24a74 --- /dev/null +++ b/src/qbit-collector/eng/install-python-reqs.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +rm -rf ../python +mkdir -p ../python +python3 -m pip install -r ../requirements.txt -t ../python/ \ No newline at end of file diff --git a/src/qbit-collector/requirements.txt b/src/qbit-collector/requirements.txt new file mode 100644 index 0000000..aaa299e --- /dev/null +++ b/src/qbit-collector/requirements.txt @@ -0,0 +1 @@ +rank-torrent-name==0.2.5 \ No newline at end of file diff --git a/src/shared/Dapper/DapperDataStorage.cs b/src/shared/Dapper/DapperDataStorage.cs index 828b4ec..5cc710c 100644 --- a/src/shared/Dapper/DapperDataStorage.cs +++ b/src/shared/Dapper/DapperDataStorage.cs @@ -167,12 +167,7 @@ public class DapperDataStorage(PostgresConfiguration configuration, RabbitMqConf INSERT INTO subtitles ("infoHash", "fileIndex", "fileId", "title") VALUES - (@InfoHash, @FileIndex, @FileId, @Title) - ON CONFLICT - ("infoHash", "fileIndex") - DO UPDATE SET - "fileId" = COALESCE(subtitles."fileId", EXCLUDED."fileId"), - "title" = COALESCE(subtitles."title", EXCLUDED."title"); + (@InfoHash, @FileIndex, @FileId, @Title); """; await connection.ExecuteAsync(query, subtitles); diff --git a/src/shared/Python/RTN/IRankTorrentName.cs b/src/shared/Python/RTN/IRankTorrentName.cs index b529e15..820bf9b 100644 --- a/src/shared/Python/RTN/IRankTorrentName.cs +++ b/src/shared/Python/RTN/IRankTorrentName.cs @@ -2,5 +2,5 @@ namespace SharedContracts.Python.RTN; public interface IRankTorrentName { - ParseTorrentTitleResponse Parse(string title); + ParseTorrentTitleResponse Parse(string title, bool trashGarbage = true); } \ No newline at end of file diff --git a/src/shared/Python/RTN/RankTorrentName.cs b/src/shared/Python/RTN/RankTorrentName.cs index 8c78e68..1e42d09 100644 --- a/src/shared/Python/RTN/RankTorrentName.cs +++ b/src/shared/Python/RTN/RankTorrentName.cs @@ -13,14 +13,14 @@ public class RankTorrentName : IRankTorrentName InitModules(); } - public ParseTorrentTitleResponse Parse(string title) => + public ParseTorrentTitleResponse Parse(string title, bool trashGarbage = true) => _pythonEngineService.ExecutePythonOperationWithDefault( () => { - var result = _rtn?.parse(title); + var result = _rtn?.parse(title, trashGarbage); return ParseResult(result); }, new ParseTorrentTitleResponse(false, null), nameof(Parse), throwOnErrors: false, logErrors: false); - + private static ParseTorrentTitleResponse ParseResult(dynamic result) { if (result == null) @@ -34,9 +34,18 @@ public class RankTorrentName : IRankTorrentName { return new(false, null); } + + var mediaType = result.GetAttr("type")?.As(); + + if (string.IsNullOrEmpty(mediaType)) + { + return new(false, null); + } var response = JsonSerializer.Deserialize(json); - + + response.IsMovie = mediaType.Equals("movie", StringComparison.OrdinalIgnoreCase); + return new(true, response); } diff --git a/src/shared/Python/RTN/RtnResponse.cs b/src/shared/Python/RTN/RtnResponse.cs index d2e1d85..041ef7a 100644 --- a/src/shared/Python/RTN/RtnResponse.cs +++ b/src/shared/Python/RTN/RtnResponse.cs @@ -76,23 +76,8 @@ public class RtnResponse [JsonPropertyName("extended")] public bool Extended { get; set; } - - // [JsonPropertyName("is_show")] - // public bool IsTvShow { get; set; } - // - // [JsonPropertyName("is_movie")] - // public bool IsMovie { get; set; } + + public bool IsMovie { get; set; } public string ToJson() => this.AsJson(); - - public bool IsMovie => !TvRegexes.Any(regex => regex.IsMatch(RawTitle)) && Season?.Count == 0 && Episode?.Count == 0; - - private static List TvRegexes { get; set; } = - [ - new(@"[se]\d\d", RegexOptions.IgnoreCase), - new(@"\b(tv|complete)\b", RegexOptions.IgnoreCase), - new(@"\b(saisons?|stages?|seasons?).?\d", RegexOptions.IgnoreCase), - new(@"[a-z]\s?\-\s?\d{2,4}\b", RegexOptions.IgnoreCase), - new(@"\d{2,4}\s?\-\s?\d{2,4}\b", RegexOptions.IgnoreCase), - ]; } \ No newline at end of file diff --git a/src/torrent-consumer/Extensions/ServiceCollectionExtensions.cs b/src/torrent-consumer/Extensions/ServiceCollectionExtensions.cs index e02b14f..aa9a623 100644 --- a/src/torrent-consumer/Extensions/ServiceCollectionExtensions.cs +++ b/src/torrent-consumer/Extensions/ServiceCollectionExtensions.cs @@ -82,11 +82,4 @@ public static class ServiceCollectionExtensions x.AddConsumer(); } - - internal static IServiceCollection AddServiceConfiguration(this IServiceCollection services) - { - services.AddSingleton(); - - return services; - } } diff --git a/src/torrent-consumer/GlobalUsings.cs b/src/torrent-consumer/GlobalUsings.cs index a1c6b79..e44308f 100644 --- a/src/torrent-consumer/GlobalUsings.cs +++ b/src/torrent-consumer/GlobalUsings.cs @@ -5,7 +5,6 @@ global using MassTransit; global using MassTransit.Mediator; global using Microsoft.AspNetCore.Builder; global using Microsoft.Extensions.DependencyInjection; -global using PromKnight.ParseTorrentTitle; global using SharedContracts.Configuration; global using SharedContracts.Dapper; global using SharedContracts.Extensions; diff --git a/src/torrent-consumer/Program.cs b/src/torrent-consumer/Program.cs index 8f5b124..262ac62 100644 --- a/src/torrent-consumer/Program.cs +++ b/src/torrent-consumer/Program.cs @@ -10,7 +10,6 @@ builder.Host builder.Services .RegisterMassTransit() - .AddServiceConfiguration() .AddDatabase(); var app = builder.Build(); diff --git a/src/torrent-consumer/TorrentConsumer.csproj b/src/torrent-consumer/TorrentConsumer.csproj index d6df32a..19bdcd1 100644 --- a/src/torrent-consumer/TorrentConsumer.csproj +++ b/src/torrent-consumer/TorrentConsumer.csproj @@ -16,7 +16,6 @@ -