helm chart [wip]

* Add housekeeping service to clean stale torrents

* version bump

.github/workflows/build_torrent_ingester.yaml

@@ -0,0 +1,15 @@
+name: Build and Push Torrent Ingestor Service
+
+on:
+  push:
+    paths:
+      - 'src/torrent-ingestor/**'
+
+jobs:
+  process:
+    uses: ./.github/workflows/base_image_workflow.yaml
+    secrets: inherit
+    with:
+      CONTEXT: ./src/torrent-ingestor
+      DOCKERFILE: ./src/torrent-ingestor/Dockerfile
+      IMAGE_NAME: knightcrawler-torrent-ingestor

.gitignore

@@ -611,3 +611,4 @@ fabric.properties
 
 # Mac directory indexes
 .DS_Store
+deployment/docker/stack.env

README.md

@@ -51,11 +51,11 @@ Download and install [Docker Compose](https://docs.docker.com/compose/install/),
 
 ### Environment Setup
 
-Before running the project, you need to set up the environment variables. Copy the `.env.example` file to `.env`:
+Before running the project, you need to set up the environment variables. Edit the values in `stack.env`:
 
 ```sh
 cd deployment/docker
-cp .env.example .env
+code stack.env
 ```
 
 Then set any of the values you wouldd like to customize.
@@ -67,9 +67,6 @@ Then set any of the values you wouldd like to customize.
 
 By default, Knight Crawler is configured to be *relatively* conservative in its resource usage. If running on a decent machine (16GB RAM, i5+ or equivalent), you can increase some settings to increase consumer throughput. This is especially helpful if you have a large backlog from [importing databases](#importing-external-dumps).
 
-In your `.env` file, under the `# Consumer` section increase `CONSUMER_REPLICAS` from `3` to `15`.
-You can also increase `JOB_CONCURRENCY` from `5` to `10`.
-
 ### DebridMediaManager setup (optional)
 
 There are some optional steps you should take to maximise the number of movies/tv shows we can find.
@@ -90,9 +87,9 @@ We can search DebridMediaManager hash lists which are hosted on GitHub. This all
         (checked) Public Repositories (read-only)
    ```
 4. Click `Generate token`
-5. Take the new token and add it to the bottom of the [.env](deployment/docker/.env) file
+5. Take the new token and add it to the bottom of the [stack.env](deployment/docker/stack.env) file
    ```
-   GithubSettings__PAT=<YOUR TOKEN HERE>
+   GITHUB_PAT=<YOUR TOKEN HERE>
    ```
 ### Configure external access
 
@@ -143,7 +140,7 @@ Remove or comment out the port for the addon, and connect it to Caddy:
 addon:
   <<: *knightcrawler-app
   env_file:
-    - .env
+    - stack.env
   hostname: knightcrawler-addon
   image: gabisonfire/knightcrawler-addon:latest
   labels:

deployment/docker/compose.yaml

@@ -1,7 +0,0 @@
-version: "3.9"
-name: "knightcrawler"
-include:
-  - components/network.yaml
-  - components/volumes.yaml
-  - components/infrastructure.yaml
-  - components/knightcrawler.yaml

deployment/docker/config/qbit/qbittorrent.conf

@@ -12,8 +12,11 @@ enabled=false
 program=
 
 [BitTorrent]
+Session\AnonymousModeEnabled=true
+Session\BTProtocol=TCP
 Session\DefaultSavePath=/downloads/
 Session\ExcludedFileNames=
+Session\MaxActiveCheckingTorrents=5
 Session\MaxActiveDownloads=10
 Session\MaxActiveTorrents=50
 Session\MaxActiveUploads=50
@@ -50,6 +53,7 @@ MailNotification\req_auth=true
 WebUI\Address=*
 WebUI\AuthSubnetWhitelist=0.0.0.0/0
 WebUI\AuthSubnetWhitelistEnabled=true
+WebUI\HostHeaderValidation=false
 WebUI\LocalHostAuth=false
 WebUI\ServerDomains=*
 

deployment/docker/docker-compose.yaml

@@ -0,0 +1,244 @@
+version: "3.9"
+
+name: knightcrawler
+
+networks:
+  knightcrawler-network:
+    name: knightcrawler-network
+    driver: bridge
+
+volumes:
+  postgres:
+  lavinmq:
+  redis:
+
+services:
+  ## Postgres is the database that is used by the services.
+  ## All downloaded metadata is stored in this database.
+  postgres:
+    env_file: stack.env
+    healthcheck:
+      test: [ "CMD", "sh", "-c", "pg_isready -h localhost -U $$POSTGRES_USER" ]
+      timeout: 10s
+      interval: 10s
+      retries: 3
+      start_period: 10s
+    image: postgres:latest
+    # # If you need the database to be accessible from outside, please open the below port.
+    # # Furthermore, please, please, please, change the username and password in the stack.env file.
+    # # If you want to enhance your security even more, create a new user for the database with a strong password.
+    # ports:
+    #   - "5432:5432"
+    networks:
+      - knightcrawler-network
+    restart: unless-stopped
+    volumes:
+      - postgres:/var/lib/postgresql/data
+
+  ## Redis is used as a cache for the services.
+  ## It is used to store the infohashes that are currently being processed in sagas, as well as intrim data.
+  redis:
+    env_file: stack.env
+    healthcheck:
+      test: ["CMD-SHELL", "redis-cli ping"]
+      timeout: 10s
+      interval: 10s
+      retries: 3
+      start_period: 10s
+    image: redis/redis-stack:latest
+    # # If you need redis to be accessible from outside, please open the below port.
+    # ports:
+    #   - "6379:6379"
+    networks:
+      - knightcrawler-network
+    restart: unless-stopped
+    volumes:
+      - redis:/data
+
+  ## LavinMQ is used as a message broker for the services.
+  ## It is a high performance drop in replacement for RabbitMQ.
+  ## It is used to communicate between the services.
+  lavinmq:
+    env_file: stack.env
+    # # If you need the database to be accessible from outside, please open the below port.
+    # # Furthermore, please, please, please, look at the documentation for lavinmq / rabbitmq on how to secure the service.
+    # ports:
+    #   - "5672:5672"
+    #   - "15672:15672"
+    #   - "15692:15692"
+    image: cloudamqp/lavinmq:latest
+    healthcheck:
+      test: ["CMD-SHELL", "lavinmqctl status"]
+      timeout: 10s
+      interval: 10s
+      retries: 3
+      start_period: 10s
+    restart: unless-stopped
+    networks:
+      - knightcrawler-network
+    volumes:
+      - lavinmq:/var/lib/lavinmq/
+
+  ## The addon. This is what is used in stremio
+  addon:
+    depends_on:
+      metadata:
+        condition: service_completed_successfully
+      migrator:
+        condition: service_completed_successfully
+      postgres:
+        condition: service_healthy
+      lavinmq:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    env_file: stack.env
+    hostname: knightcrawler-addon
+    image: gabisonfire/knightcrawler-addon:2.0.17
+    labels:
+      logging: promtail
+    networks:
+      - knightcrawler-network
+    ports:
+      - "7000:7000"
+    restart: unless-stopped
+
+  ## The consumer is responsible for consuming infohashes and orchestrating download of metadata.
+  consumer:
+    depends_on:
+      metadata:
+        condition: service_completed_successfully
+      migrator:
+        condition: service_completed_successfully
+      postgres:
+        condition: service_healthy
+      lavinmq:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    env_file: stack.env
+    image: gabisonfire/knightcrawler-consumer:2.0.17
+    labels:
+      logging: promtail
+    networks:
+      - knightcrawler-network
+    restart: unless-stopped
+
+  ## The debrid collector is responsible for downloading metadata from debrid services. (Currently only RealDebrid is supported)
+  debridcollector:
+    depends_on:
+      metadata:
+        condition: service_completed_successfully
+      migrator:
+        condition: service_completed_successfully
+      postgres:
+        condition: service_healthy
+      lavinmq:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    env_file: stack.env
+    image: gabisonfire/knightcrawler-debrid-collector:2.0.17
+    labels:
+      logging: promtail
+    networks:
+      - knightcrawler-network
+    restart: unless-stopped
+
+  ## The metadata service is responsible for downloading imdb publically available datasets.
+  ## This is used to enrich the metadata during production of ingested infohashes.
+  metadata:
+    depends_on:
+      migrator:
+        condition: service_completed_successfully
+    env_file: stack.env
+    image: gabisonfire/knightcrawler-metadata:2.0.17
+    networks:
+      - knightcrawler-network
+    restart: "no"
+
+  ## The migrator is responsible for migrating the database schema.
+  migrator:
+    depends_on:
+      postgres:
+        condition: service_healthy
+    env_file: stack.env
+    image: gabisonfire/knightcrawler-migrator:2.0.17
+    networks:
+      - knightcrawler-network
+    restart: "no"
+
+  ## The producer is responsible for producing infohashes by acquiring for various sites, including DMM.
+  producer:
+    depends_on:
+      metadata:
+        condition: service_completed_successfully
+      migrator:
+        condition: service_completed_successfully
+      postgres:
+        condition: service_healthy
+      lavinmq:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    env_file: stack.env
+    image: gabisonfire/knightcrawler-producer:2.0.17
+    labels:
+      logging: promtail
+    networks:
+      - knightcrawler-network
+    restart: unless-stopped
+
+  ## QBit collector utilizes QBitTorrent to download metadata.
+  qbitcollector:
+    depends_on:
+      metadata:
+        condition: service_completed_successfully
+      migrator:
+        condition: service_completed_successfully
+      postgres:
+        condition: service_healthy
+      lavinmq:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      qbittorrent:
+        condition: service_healthy
+    deploy:
+      replicas: ${QBIT_REPLICAS:-0}
+    env_file: stack.env
+    image: gabisonfire/knightcrawler-qbit-collector:2.0.17
+    labels:
+      logging: promtail
+    networks:
+      - knightcrawler-network
+    restart: unless-stopped
+
+  ## QBitTorrent is a torrent client that can be used to download torrents. In this case its used to download metadata.
+  ## The QBit collector requires this.
+  qbittorrent:
+    deploy:
+      replicas: ${QBIT_REPLICAS:-0}
+    env_file: stack.env
+    environment:
+      PGID: "1000"
+      PUID: "1000"
+      TORRENTING_PORT: "6881"
+      WEBUI_PORT: "8080"
+    healthcheck:
+      test: ["CMD-SHELL", "curl --fail http://localhost:8080"]
+      timeout: 10s
+      interval: 10s
+      retries: 3
+      start_period: 10s
+    image: lscr.io/linuxserver/qbittorrent:latest
+    networks:
+      - knightcrawler-network
+    ports:
+      - "6881:6881/tcp"
+      - "6881:6881/udp"
+    # if you want to expose the webui, uncomment the following line
+      # - "8001:8080"
+    restart: unless-stopped
+    volumes:
+      - ./config/qbit/qbittorrent.conf:/config/qBittorrent/qBittorrent.conf

deployment/docker/optional-services/grafana-metrics/config/prometheus/config.yml

@@ -16,7 +16,7 @@ rule_files:
 scrape_configs:
   - job_name: "rabbitmq"
     static_configs:
-      - targets: ["rabbitmq:15692"]
+      - targets: ["lavinmq:15692"]
   - job_name: "postgres-exporter"
     static_configs:
       - targets: ["postgres-exporter:9187"]

deployment/docker/src/components/infrastructure.yaml

@@ -4,8 +4,8 @@ x-basehealth: &base-health
   retries: 3
   start_period: 10s
 
-x-rabbithealth: &rabbitmq-health
-  test: rabbitmq-diagnostics -q ping
+x-lavinhealth: &lavinmq-health
+  test: [ "CMD-SHELL", "lavinmqctl status" ]
   <<: *base-health
 
 x-redishealth: &redis-health
@@ -13,7 +13,7 @@ x-redishealth: &redis-health
   <<: *base-health
 
 x-postgreshealth: &postgresdb-health
-  test: pg_isready
+  test: [ "CMD", "sh", "-c", "pg_isready -h localhost -U $$POSTGRES_USER" ]
   <<: *base-health
 
 x-qbit: &qbit-health
@@ -35,7 +35,7 @@ services:
       - postgres:/var/lib/postgresql/data
     healthcheck: *postgresdb-health
     restart: unless-stopped
-    env_file: ../.env
+    env_file: ../../.env
     networks:
       - knightcrawler-network
 
@@ -48,25 +48,23 @@ services:
       - redis:/data
     restart: unless-stopped
     healthcheck: *redis-health
-    env_file: ../.env
+    env_file: ../../.env
     networks:
       - knightcrawler-network
 
-  rabbitmq:
-    image: rabbitmq:3-management
+  lavinmq:
+    env_file: stack.env
     # # If you need the database to be accessible from outside, please open the below port.
-    # # Furthermore, please, please, please, look at the documentation for rabbit on how to secure the service.
+    # # Furthermore, please, please, please, look at the documentation for lavinmq / rabbitmq on how to secure the service.
     # ports:
     #   - "5672:5672"
     #   - "15672:15672"
     #   - "15692:15692"
-    volumes:
-      - rabbitmq:/var/lib/rabbitmq
+    image: cloudamqp/lavinmq:latest
+    healthcheck: *lavinmq-health
     restart: unless-stopped
-    healthcheck: *rabbitmq-health
-    env_file: ../.env
-    networks:
-      - knightcrawler-network
+    volumes:
+      - lavinmq:/var/lib/lavinmq/
 
   ## QBitTorrent is a torrent client that can be used to download torrents. In this case its used to download metadata.
   ## The QBit collector requires this.
@@ -80,10 +78,10 @@ services:
     ports:
       - 6881:6881
       - 6881:6881/udp
-    env_file: ../.env
+    env_file: ../../.env
     networks:
       - knightcrawler-network
     restart: unless-stopped
     healthcheck: *qbit-health
     volumes:
-      - ./config/qbit/qbittorrent.conf:/config/qBittorrent/qBittorrent.conf
+      - ../../config/qbit/qbittorrent.conf:/config/qBittorrent/qBittorrent.conf

deployment/docker/src/components/knightcrawler.yaml

@@ -1,7 +1,7 @@
 x-apps: &knightcrawler-app
   labels:
     logging: "promtail"
-  env_file: ../.env
+  env_file: ../../.env
   networks:
     - knightcrawler-network
 
@@ -11,7 +11,7 @@ x-depends: &knightcrawler-app-depends
       condition: service_healthy
     postgres:
       condition: service_healthy
-    rabbitmq:
+    lavinmq:
       condition: service_healthy
     migrator:
       condition: service_completed_successfully
@@ -20,8 +20,8 @@ x-depends: &knightcrawler-app-depends
 
 services:
   metadata:
-    image: gabisonfire/knightcrawler-metadata:2.0.6
-    env_file: ../.env
+    image: gabisonfire/knightcrawler-metadata:2.0.17
+    env_file: ../../.env
     networks:
       - knightcrawler-network
     restart: no
@@ -30,8 +30,8 @@ services:
         condition: service_completed_successfully
 
   migrator:
-    image: gabisonfire/knightcrawler-migrator:2.0.6
-    env_file: ../.env
+    image: gabisonfire/knightcrawler-migrator:2.0.17
+    env_file: ../../.env
     networks:
       - knightcrawler-network
     restart: no
@@ -40,7 +40,7 @@ services:
         condition: service_healthy
 
   addon:
-    image: gabisonfire/knightcrawler-addon:2.0.6
+    image: gabisonfire/knightcrawler-addon:2.0.17
     <<: [*knightcrawler-app, *knightcrawler-app-depends]
     restart: unless-stopped
     hostname: knightcrawler-addon
@@ -48,22 +48,22 @@ services:
       - "7000:7000"
 
   consumer:
-    image: gabisonfire/knightcrawler-consumer:2.0.6
+    image: gabisonfire/knightcrawler-consumer:2.0.17
     <<: [*knightcrawler-app, *knightcrawler-app-depends]
     restart: unless-stopped
 
   debridcollector:
-    image: gabisonfire/knightcrawler-debrid-collector:2.0.6
+    image: gabisonfire/knightcrawler-debrid-collector:2.0.17
     <<: [*knightcrawler-app, *knightcrawler-app-depends]
     restart: unless-stopped
         
   producer:
-    image: gabisonfire/knightcrawler-producer:2.0.6
+    image: gabisonfire/knightcrawler-producer:2.0.17
     <<: [*knightcrawler-app, *knightcrawler-app-depends]
     restart: unless-stopped
 
   qbitcollector:
-    image: gabisonfire/knightcrawler-qbit-collector:2.0.6
+    image: gabisonfire/knightcrawler-qbit-collector:2.0.17
     <<: [*knightcrawler-app, *knightcrawler-app-depends]
     restart: unless-stopped
     depends_on:

deployment/docker/src/components/volumes.yaml

@@ -1,4 +1,4 @@
 volumes:
   postgres:
   redis:
-  rabbitmq:
+  lavinmq:

deployment/docker/src/compose.yaml

@@ -0,0 +1,7 @@
+version: "3.9"
+name: "knightcrawler"
+include:
+  - ./components/network.yaml
+  - ./components/volumes.yaml
+  - ./components/infrastructure.yaml
+  - ./components/knightcrawler.yaml

deployment/docker/stack.env

@@ -13,8 +13,8 @@ REDIS_HOST=redis
 REDIS_PORT=6379
 REDIS_EXTRA=abortConnect=false,allowAdmin=true
 
-# RabbitMQ
-RABBITMQ_HOST=rabbitmq
+# AMQP
+RABBITMQ_HOST=lavinmq
 RABBITMQ_USER=guest
 RABBITMQ_PASSWORD=guest
 RABBITMQ_CONSUMER_QUEUE_NAME=ingested
@@ -31,6 +31,10 @@ COLLECTOR_QBIT_ENABLED=false
 COLLECTOR_DEBRID_ENABLED=true
 COLLECTOR_REAL_DEBRID_API_KEY=
 QBIT_HOST=http://qbittorrent:8080
+QBIT_TRACKERS_URL=https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_http.txt
+
+# Number of replicas for the qBittorrent collector and qBitTorrent client. Should be 0 or 1.
+QBIT_REPLICAS=0
 
 # Addon
 DEBUG_MODE=false

deployment/k8s/Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v2
+appVersion: 2.0.17
+description: A helm chart for Knightcrawler
+name: knightcrawler
+type: application
+version: 0.1.0

deployment/k8s/templates/NOTES.txt

@@ -0,0 +1,6 @@
+
+Congratulations,
+
+Knightcrawler is now deployed. This may take a while to be up and responding.
+
+

deployment/k8s/templates/config/knightcrawler.configmap.yaml

@@ -0,0 +1,27 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: '{{ .Release.Name }}-config'
+  labels:
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+data:
+  COLLECTOR_DEBRID_ENABLED: '{{ .Values.knightcrawler.debridcollector.enabled }}'
+  COLLECTOR_QBIT_ENABLED: '{{ .Values.knightcrawler.qbitcollector.enabled }}'
+  DEBUG_MODE: '{{ .Values.knightcrawler.debug }}'
+  METADATA_INSERT_BATCH_SIZE: '{{ .Values.environment.metadata.insertBatchSize }}'
+  POSTGRES_DB: '{{ .Values.environment.postgres.dbName }}'
+  POSTGRES_HOST: '{{ if .Values.environment.postgres.external }}{{ .Values.environment.postgres.host }}{{ else }}{{ .Release.Name }}-postgres{{ end }}'
+  POSTGRES_PORT: '{{ .Values.environment.postgres.port }}'
+  QBIT_HOST: '{{ .Values.environment.qbitcollector.qbitHost }}'
+  QBIT_TRACKERS_URL: '{{ .Values.environment.qbitcollector.trackersUrl }}'
+  RABBITMQ_CONSUMER_QUEUE_NAME: '{{ .Values.environment.producer.queueName }}'
+  RABBITMQ_DURABLE: '{{ .Values.environment.producer.durable }}'
+  RABBITMQ_HOST: '{{ if .Values.environment.lavinmq.external }}{{ .Values.environment.lavinmq.host }}{{ else }}{{ .Release.Name }}-lavinmq{{ end }}'
+  RABBITMQ_MAX_PUBLISH_BATCH_SIZE: '{{ .Values.environment.producer.maxPublishBatchSize }}'
+  RABBITMQ_MAX_QUEUE_SIZE: '{{ .Values.environment.producer.maxQueueSize }}'
+  RABBITMQ_PUBLISH_INTERVAL_IN_SECONDS: '{{ .Values.environment.producer.publishIntervalSeconds }}'
+  REDIS_EXTRA: '{{ .Values.environment.redis.extra }}'
+  REDIS_HOST: '{{ if .Values.environment.redis.external }}{{ .Values.environment.redis.host }}{{ else }}{{ .Release.Name }}-redis{{ end }}'
+  REDIS_PORT: '{{ .Values.environment.redis.port }}'
+  TZ: '{{ .Values.shared.timezone }}'

deployment/k8s/templates/config/knightcrawler.secrets.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: '{{ .Release.Name }}-secrets'
+  labels:
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+type: Opaque
+data:
+  GITHUB_PAT: '{{ .Values.environment.producer.githubPat | b64enc }}'
+  COLLECTOR_REAL_DEBRID_API_KEY: '{{ .Values.environment.debridcollector.realDebridApiKey | b64enc }}'
+  POSTGRES_USER: '{{ .Values.environment.postgres.user | b64enc }}'
+  POSTGRES_PASSWORD: '{{ .Values.environment.postgres.password | b64enc }}'
+  RABBITMQ_PASSWORD: '{{ .Values.environment.lavinmq.password | b64enc }}'
+  RABBITMQ_USER: '{{ .Values.environment.lavinmq.user | b64enc }}'

deployment/k8s/templates/infrastructure/lavinmq/lavinmq.service.yaml

@@ -0,0 +1,25 @@
+{{ if .Values.infrastructure.lavinmq.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: '{{ .Release.Name }}-lavinmq'
+  labels:
+    component: lavinmq
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+spec:
+  selector:
+    component: lavinmq
+    release: '{{ .Release.Name }}'
+  type: ClusterIP
+  ports:
+    - protocol: TCP
+      port: 5672
+      targetPort: 5672
+    - protocol: TCP
+      port: 15672
+      targetPort: 15672
+    - protocol: TCP
+      port: 15692
+      targetPort: 15692
+{{- end -}}

deployment/k8s/templates/infrastructure/lavinmq/lavinmq.statefulset.yaml

@@ -0,0 +1,60 @@
+{{ if .Values.infrastructure.lavinmq.enabled }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: '{{ .Release.Name }}-lavinmq'
+  labels:
+    component: lavinmq
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "0"
+spec:
+  serviceName: '{{ .Release.Name }}-lavinmq'
+  replicas: 1
+  selector:
+    matchLabels:
+      component: lavinmq
+      release: '{{ .Release.Name }}'
+  template:
+    metadata:
+      labels:
+        component: lavinmq
+        release: '{{ .Release.Name }}'
+    spec:
+      containers:
+        - name: lavinmq
+          image: '{{ .Values.infrastructure.lavinmq.image }}:{{ .Values.infrastructure.lavinmq.tag }}'
+          ports:
+            - name: lavinmq
+              containerPort: 5672
+            - name: lavinmq-15672
+              containerPort: 15672
+            - name: lavinmq-15692
+              containerPort: 15692
+          envFrom:
+            - configMapRef:
+                name: '{{ .Release.Name }}-config'
+            - secretRef:
+                name: '{{ .Release.Name }}-secrets'
+          volumeMounts:
+            - mountPath: /var/lib/lavinmq
+              name: lavinmq
+          livenessProbe:
+            exec:
+              command:
+                - lavinmqctl status
+            periodSeconds: 10
+            initialDelaySeconds: 10
+            successThreshold: 1
+            failureThreshold: 3
+  volumeClaimTemplates:
+    - metadata:
+        name: lavinmq
+      spec:
+        accessModes: [ "ReadWriteOnce" ]
+        resources:
+          requests:
+            storage: '{{ .Values.persistence.lavinmq.capacity }}'
+{{- end -}}

deployment/k8s/templates/infrastructure/postgres/postgres.service.yaml

@@ -0,0 +1,19 @@
+{{ if .Values.infrastructure.postgres.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: '{{ .Release.Name }}-postgres'
+  labels:
+    component: postgres
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+spec:
+  selector:
+    component: postgres
+    release: '{{ .Release.Name }}'
+  type: ClusterIP
+  ports:
+    - protocol: TCP
+      port: 5432
+      targetPort: 5432
+{{- end -}}

deployment/k8s/templates/infrastructure/postgres/postgres.statefulset.yaml

@@ -0,0 +1,58 @@
+{{ if .Values.infrastructure.postgres.enabled }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: '{{ .Release.Name }}-postgres'
+  labels:
+    component: postgres
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "0"
+spec:
+  serviceName: '{{ .Release.Name }}-postgres'
+  replicas: 1
+  selector:
+    matchLabels:
+      component: postgres
+      release: '{{ .Release.Name }}'
+  template:
+    metadata:
+      labels:
+        component: postgres
+        release: '{{ .Release.Name }}'
+    spec:
+      containers:
+        - name: postgres
+          image: '{{ .Values.infrastructure.postgres.image }}:{{ .Values.infrastructure.postgres.tag }}'
+          ports:
+            - name: postgres
+              containerPort: 5432
+          envFrom:
+            - configMapRef:
+                name: '{{ .Release.Name }}-config'
+            - secretRef:
+                name: '{{ .Release.Name }}-secrets'
+          volumeMounts:
+            - mountPath: /var/lib/postgresql/data
+              name: postgres
+          livenessProbe:
+            exec:
+              command:
+                - sh
+                - -c
+                - pg_isready -h localhost -U $POSTGRES_USER
+            periodSeconds: 10
+            initialDelaySeconds: 10
+            successThreshold: 1
+            failureThreshold: 3
+  volumeClaimTemplates:
+    - metadata:
+        name: postgres
+      spec:
+        accessModes: [ "ReadWriteOnce" ]
+        resources:
+          requests:
+            storage: '{{ .Values.persistence.postgres.capacity }}'
+{{- end -}}

deployment/k8s/templates/infrastructure/qbittorrent/qbittorrent.deployment.yaml

@@ -0,0 +1,57 @@
+{{ if .Values.knightcrawler.qbitcollector.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: '{{ .Release.Name }}-qbittorrent'
+  labels:
+    component: qbittorrent
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "0"
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      component: qbittorrent
+      release: '{{ .Release.Name }}'
+  template:
+    metadata:
+      labels:
+        component: qbittorrent
+        release: '{{ .Release.Name }}'
+    spec:
+      containers:
+        - name: qbittorrent
+          image: '{{ .Values.infrastructure.qbittorrent.image }}:{{ .Values.infrastructure.qbittorrent.tag }}'
+          ports:
+            - name: qbittorrent
+              containerPort: 6881
+            - name: qbittorrent-6881
+              containerPort: 6881
+            - name: qbittorrent-8080
+              containerPort: 8080
+          env:
+            - name: PUID
+              value: '{{ .Values.environment.qbittorrent.puid }}'
+            - name: PGID
+              value: '{{ .Values.environment.qbittorrent.pgid }}'
+            - name: TORRENTING_PORT
+              value: '{{ .Values.environment.qbittorrent.torrentingPort }}'
+            - name: WEBUI_PORT
+              value: '{{ .Values.environment.qbittorrent.webuiPort }}'
+          envFrom:
+            - configMapRef:
+                name: '{{ .Release.Name }}-config'
+            - secretRef:
+                name: '{{ .Release.Name }}-secrets'
+          livenessProbe:
+            exec:
+              command:
+                - curl --fail http://localhost:8080
+            periodSeconds: 10
+            initialDelaySeconds: 10
+            successThreshold: 1
+            failureThreshold: 3
+{{- end -}}

deployment/k8s/templates/infrastructure/qbittorrent/qbittorrent.service.yaml

@@ -0,0 +1,25 @@
+{{ if .Values.knightcrawler.qbitcollector.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: '{{ .Release.Name }}-qbittorrent'
+  labels:
+    component: qbittorrent
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+spec:
+  selector:
+    component: qbittorrent
+    release: '{{ .Release.Name }}'
+  type: ClusterIP
+  ports:
+    - protocol: TCP
+      port: 6881
+      targetPort: 6881
+    - protocol: TCP
+      port: 6881
+      targetPort: 6881
+    - protocol: TCP
+      port: 8080
+      targetPort: 8080
+{{- end -}}

deployment/k8s/templates/infrastructure/redis/redis.service.yaml

@@ -0,0 +1,19 @@
+{{ if .Values.infrastructure.redis.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: '{{ .Release.Name }}-redis'
+  labels:
+    component: redis
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+spec:
+  selector:
+    component: redis
+    release: '{{ .Release.Name }}'
+  type: ClusterIP
+  ports:
+    - protocol: TCP
+      port: 6379
+      targetPort: 6379
+{{- end -}}

deployment/k8s/templates/infrastructure/redis/redis.statefulset.yaml

@@ -0,0 +1,56 @@
+{{ if .Values.infrastructure.redis.enabled }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: '{{ .Release.Name }}-redis'
+  labels:
+    component: redis
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "0"
+spec:
+  serviceName: '{{ .Release.Name }}-redis'
+  replicas: 1
+  selector:
+    matchLabels:
+      component: redis
+      release: '{{ .Release.Name }}'
+  template:
+    metadata:
+      labels:
+        component: redis
+        release: '{{ .Release.Name }}'
+    spec:
+      containers:
+        - name: redis
+          image: '{{ .Values.infrastructure.redis.image }}:{{ .Values.infrastructure.redis.tag }}'
+          ports:
+            - name: redis
+              containerPort: 6379
+          envFrom:
+            - configMapRef:
+                name: '{{ .Release.Name }}-config'
+            - secretRef:
+                name: '{{ .Release.Name }}-secrets'
+          volumeMounts:
+            - mountPath: /data
+              name: redis
+          livenessProbe:
+            exec:
+              command:
+                - redis-cli ping
+            periodSeconds: 10
+            initialDelaySeconds: 10
+            successThreshold: 1
+            failureThreshold: 3
+  volumeClaimTemplates:
+    - metadata:
+        name: redis
+      spec:
+        accessModes: [ "ReadWriteOnce" ]
+        resources:
+          requests:
+            storage: '{{ .Values.persistence.redis.capacity }}'
+{{- end -}}

deployment/k8s/templates/jobs/metadata.job.yaml

@@ -0,0 +1,28 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: '{{ .Release.Name }}-metadata'
+  labels:
+    component: metadata
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "2"
+    "helm.sh/hook-delete-policy": hook-succeeded
+spec:
+  template:
+    metadata:
+      labels:
+        component: metadata
+        release: '{{ .Release.Name }}'
+    spec:
+      restartPolicy: OnFailure
+      containers:
+        - name: metadata
+          image: '{{ .Values.knightcrawler.metadata.image }}{{ if ne .Values.knightcrawler.globalImageTagOverride "" }}:{{ .Values.knightcrawler.globalImageTagOverride }}{{else}}:{{ .Values.knightcrawler.metadata.tag}}{{ end }}'
+          envFrom:
+            - configMapRef:
+                name: '{{ .Release.Name }}-config'
+            - secretRef:
+                name: '{{ .Release.Name }}-secrets'

deployment/k8s/templates/jobs/migrator.job.yaml

@@ -0,0 +1,28 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: '{{ .Release.Name }}-migrator'
+  labels:
+    component: migrator
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "1"
+    "helm.sh/hook-delete-policy": hook-succeeded
+spec:
+  template:
+    metadata:
+      labels:
+        component: migrator
+        release: '{{ .Release.Name }}'
+    spec:
+      restartPolicy: OnFailure
+      containers:
+        - name: migrator
+          image: '{{ .Values.knightcrawler.migrator.image }}{{ if ne .Values.knightcrawler.globalImageTagOverride "" }}:{{ .Values.knightcrawler.globalImageTagOverride }}{{else}}:{{ .Values.knightcrawler.migrator.tag}}{{ end }}'
+          envFrom:
+            - configMapRef:
+                name: '{{ .Release.Name }}-config'
+            - secretRef:
+                name: '{{ .Release.Name }}-secrets'

deployment/k8s/templates/kc/deployments/addon.deployment.yaml

@@ -0,0 +1,35 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: '{{ .Release.Name }}-addon'
+  labels:
+    component: addon
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "4"
+spec:
+  replicas: {{ .Values.knightcrawler.addon.replicas }}
+  selector:
+    matchLabels:
+      component: addon
+      release: '{{ .Release.Name }}'
+  template:
+    metadata:
+      labels:
+        component: addon
+        release: '{{ .Release.Name }}'
+    spec:
+      containers:
+        - name: addon
+          image: '{{ .Values.knightcrawler.addon.image }}{{ if ne .Values.knightcrawler.globalImageTagOverride "" }}:{{ .Values.knightcrawler.globalImageTagOverride }}{{else}}:{{ .Values.knightcrawler.addon.tag}}{{ end }}'
+          ports:
+            - name: addon
+              containerPort: 7000
+          envFrom:
+            - configMapRef:
+                name: '{{ .Release.Name }}-config'
+            - secretRef:
+                name: '{{ .Release.Name }}-secrets'
+

deployment/k8s/templates/kc/deployments/consumer.deployment.yaml

@@ -0,0 +1,32 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: '{{ .Release.Name }}-consumer'
+  labels:
+    component: consumer
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "4"
+spec:
+  replicas: {{ .Values.knightcrawler.consumer.replicas }}
+  selector:
+    matchLabels:
+      component: consumer
+      release: '{{ .Release.Name }}'
+  template:
+    metadata:
+      labels:
+        component: consumer
+        release: '{{ .Release.Name }}'
+    spec:
+      containers:
+        - name: consumer
+          image: '{{ .Values.knightcrawler.consumer.image }}{{ if ne .Values.knightcrawler.globalImageTagOverride "" }}:{{ .Values.knightcrawler.globalImageTagOverride }}{{else}}:{{ .Values.knightcrawler.consumer.tag}}{{ end }}'
+          envFrom:
+            - configMapRef:
+                name: '{{ .Release.Name }}-config'
+            - secretRef:
+                name: '{{ .Release.Name }}-secrets'
+

deployment/k8s/templates/kc/deployments/debridcollector.deployment.yaml

@@ -0,0 +1,31 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: '{{ .Release.Name }}-debridcollector'
+  labels:
+    component: debridcollector
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "4"
+spec:
+  replicas: {{ .Values.knightcrawler.debridcollector.replicas }}
+  selector:
+    matchLabels:
+      component: debridcollector
+      release: '{{ .Release.Name }}'
+  template:
+    metadata:
+      labels:
+        component: debridcollector
+        release: '{{ .Release.Name }}'
+    spec:
+      containers:
+        - name: debridcollector
+          image: '{{ .Values.knightcrawler.debridcollector.image }}{{ if ne .Values.knightcrawler.globalImageTagOverride "" }}:{{ .Values.knightcrawler.globalImageTagOverride }}{{else}}:{{ .Values.knightcrawler.debridcollector.tag}}{{ end }}'
+          envFrom:
+            - configMapRef:
+                name: '{{ .Release.Name }}-config'
+            - secretRef:
+                name: '{{ .Release.Name }}-secrets'

deployment/k8s/templates/kc/deployments/producer.deployment.yaml

@@ -0,0 +1,31 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: '{{ .Release.Name }}-producer'
+  labels:
+    component: producer
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "4"
+spec:
+  replicas: {{ .Values.knightcrawler.producer.replicas }}
+  selector:
+    matchLabels:
+      component: producer
+      release: '{{ .Release.Name }}'
+  template:
+    metadata:
+      labels:
+        component: producer
+        release: '{{ .Release.Name }}'
+    spec:
+      containers:
+        - name: producer
+          image: '{{ .Values.knightcrawler.producer.image }}{{ if ne .Values.knightcrawler.globalImageTagOverride "" }}:{{ .Values.knightcrawler.globalImageTagOverride }}{{else}}:{{ .Values.knightcrawler.producer.tag}}{{ end }}'
+          envFrom:
+            - configMapRef:
+                name: '{{ .Release.Name }}-config'
+            - secretRef:
+                name: '{{ .Release.Name }}-secrets'

deployment/k8s/templates/kc/deployments/qbitcollector.deployment.yaml

@@ -0,0 +1,33 @@
+{{ if .Values.knightcrawler.qbitcollector.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: '{{ .Release.Name }}-qbitcollector'
+  labels:
+    component: qbitcollector
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "4"
+spec:
+  replicas: {{ .Values.knightcrawler.qbitcollector.replicas }}
+  selector:
+    matchLabels:
+      component: qbitcollector
+      release: '{{ .Release.Name }}'
+  template:
+    metadata:
+      labels:
+        component: qbitcollector
+        release: '{{ .Release.Name }}'
+    spec:
+      containers:
+        - name: qbitcollector
+          image: '{{ .Values.knightcrawler.qbitcollector.image }}{{ if ne .Values.knightcrawler.globalImageTagOverride "" }}:{{ .Values.knightcrawler.globalImageTagOverride }}{{else}}:{{ .Values.knightcrawler.qbitcollector.tag}}{{ end }}'
+          envFrom:
+            - configMapRef:
+                name: '{{ .Release.Name }}-config'
+            - secretRef:
+                name: '{{ .Release.Name }}-secrets'
+{{- end -}}

deployment/k8s/templates/kc/services/addon.service.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: '{{ .Release.Name }}-addon'
+  labels:
+    component: addon
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+spec:
+  selector:
+    component: addon
+    release: '{{ .Release.Name }}'
+  type: ClusterIP
+  ports:
+    - protocol: TCP
+      port: 7000
+      targetPort: 7000

deployment/k8s/values.yaml

@@ -0,0 +1,100 @@
+knightcrawler:
+  debug: false
+  globalImageTagOverride: ""
+  addon:
+    image: "gabisonfire/knightcrawler-addon"
+    tag: "2.0.17"
+    replicas: 1
+  consumer:
+    image: "gabisonfire/knightcrawler-consumer"
+    tag: "2.0.17"
+    replicas: 1
+  metadata:
+    image: "gabisonfire/knightcrawler-metadata"
+    tag: "2.0.17"
+    replicas: 1
+  migrator:
+    image: "gabisonfire/knightcrawler-migrator"
+    tag: "2.0.17"
+    replicas: 1
+  debridcollector:
+    image: "gabisonfire/knightcrawler-debrid-collector"
+    tag: "2.0.17"
+    enabled: true
+    replicas: 1
+  qbitcollector:
+    image: "gabisonfire/knightcrawler-qbit-collector"
+    tag: "2.0.17"
+    enabled: false
+    replicas: 1
+  producer:
+    image: "gabisonfire/knightcrawler-producer"
+    tag: "2.0.17"
+    replicas: 1
+    
+infrastructure:
+  lavinmq:
+    image: "cloudamqp/lavinmq"
+    tag: "latest"
+    enabled: true
+  postgres:
+    image: "postgres"
+    tag: "latest"
+    enabled: true
+  redis:
+    image: "redis/redis-stack-server"
+    tag: "latest"
+    enabled: true
+  qbittorrent:
+    image: "lscr.io/linuxserver/qbittorrent"
+    tag: "latest"
+      
+environment:
+  redis:
+    external: false
+    host: ""
+    port: "6379"
+    extra: "abortConnect=false,allowAdmin=true"
+  postgres:
+    external: false
+    host: ""
+    port: "5432"
+    dbName: "knightcrawler"
+    user: "postgres"
+    password: "postgres"
+  lavinmq:
+    external: false
+    host: ""
+    user: "guest"
+    password: "guest"
+  qbitcollector:
+    qbitHost: "http://qbittorrent:8080"
+    trackersUrl: "https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_http.txt"
+  debridcollector:
+    realDebridApiKey: ""
+  producer:
+    githubPat: ""
+    queueName: "ingested"
+    durable: true
+    maxPublishBatchSize: 500
+    maxQueueSize: 0
+    publishIntervalSeconds: 10
+  metadata:
+    insertBatchSize: 50000
+  qbittorrent:
+    pgid: "1000"
+    puid: "1000"
+    torrentingPort: "6881"
+    webuiPort: "8080"
+      
+persistence:
+  storageClassName: ""
+  redis:
+    capacity: 1Gi
+  postgres:
+    capacity: 1Gi
+  lavinmq:
+    capacity: 1Gi
+    
+shared:
+  timezone: "London/Europe"

src/addon/src/lib/repository.js

@@ -14,13 +14,12 @@ const Torrent = database.define('torrent',
     {
         infoHash: { type: Sequelize.STRING(64), primaryKey: true },
         provider: { type: Sequelize.STRING(32), allowNull: false },
-        torrentId: { type: Sequelize.STRING(128) },
+        ingestedTorrentId: { type: Sequelize.BIGINT, allowNull: false },
         title: { type: Sequelize.STRING(256), allowNull: false },
         size: { type: Sequelize.BIGINT },
         type: { type: Sequelize.STRING(16), allowNull: false },
         uploadDate: { type: Sequelize.DATE, allowNull: false },
         seeders: { type: Sequelize.SMALLINT },
-        trackers: { type: Sequelize.STRING(4096) },
         languages: { type: Sequelize.STRING(4096) },
         resolution: { type: Sequelize.STRING(16) }
     }

src/debrid-collector/Features/Worker/Requests.cs

@@ -1,21 +1,21 @@
 namespace DebridCollector.Features.Worker;
 
-[EntityName("perform-metadata-request")]
+[EntityName("perform-metadata-request-debrid-collector")]
 public record PerformMetadataRequest(Guid CorrelationId, string InfoHash) : CorrelatedBy<Guid>;
 
-[EntityName("torrent-metadata-response")]
+[EntityName("torrent-metadata-response-debrid-collector")]
 public record GotMetadata(TorrentMetadataResponse Metadata) : CorrelatedBy<Guid>
 {
     public Guid CorrelationId { get; init; } = Metadata.CorrelationId;
 }
 
-[EntityName("write-metadata")]
+[EntityName("write-metadata-debrid-collector")]
 public record WriteMetadata(Torrent Torrent, TorrentMetadataResponse Metadata, string ImdbId) : CorrelatedBy<Guid>
 {
     public Guid CorrelationId { get; init; } = Metadata.CorrelationId;
 }
 
-[EntityName("metadata-written")]
+[EntityName("metadata-written-debrid-colloctor")]
 public record MetadataWritten(TorrentMetadataResponse Metadata) : CorrelatedBy<Guid>
 {
     public Guid CorrelationId { get; init; } = Metadata.CorrelationId;

src/metadata/Features/Files/BasicsFile.cs

@@ -72,7 +72,7 @@ public class BasicsFile(ILogger<BasicsFile> logger, ImdbDbService dbService): IF
                 Category = csv.GetField(1),
                 Title = csv.GetField(2),
                 Adult = isAdultSet && adult == 1,
-                Year = csv.GetField(5),
+                Year = csv.GetField(5) == @"\N" ? 0 : int.Parse(csv.GetField(5)),
             };
 
             if (cancellationToken.IsCancellationRequested)

src/metadata/Features/Files/ImdbBasicEntry.cs

@@ -6,5 +6,5 @@ public class ImdbBasicEntry
     public string? Category { get; set; }
     public string? Title { get; set; }
     public bool Adult { get; set; }
-    public string? Year { get; set; }
+    public int Year { get; set; }
 }

src/metadata/Features/ImportImdbData/ImdbDbService.cs

@@ -17,7 +17,7 @@ public class ImdbDbService(PostgresConfiguration configuration, ILogger<ImdbDbSe
                         await writer.WriteAsync(entry.ImdbId, NpgsqlDbType.Text);
                         await writer.WriteAsync(entry.Category, NpgsqlDbType.Text);
                         await writer.WriteAsync(entry.Title, NpgsqlDbType.Text);
-                        await writer.WriteAsync(entry.Year, NpgsqlDbType.Text);
+                        await writer.WriteAsync(entry.Year, NpgsqlDbType.Integer);
                         await writer.WriteAsync(entry.Adult, NpgsqlDbType.Boolean);
                     }
                     catch (Npgsql.PostgresException e)
@@ -116,7 +116,7 @@ public class ImdbDbService(PostgresConfiguration configuration, ILogger<ImdbDbSe
         ExecuteCommandAsync(
             async connection =>
             {
-                await using var command = new NpgsqlCommand($"CREATE INDEX title_gist ON {TableNames.MetadataTable} USING gist(title gist_trgm_ops)", connection);
+                await using var command = new NpgsqlCommand($"CREATE INDEX title_gin ON {TableNames.MetadataTable} USING gin(title gin_trgm_ops)", connection);
                 await command.ExecuteNonQueryAsync();
             }, "Error while creating index on imdb_metadata table");
 
@@ -125,7 +125,7 @@ public class ImdbDbService(PostgresConfiguration configuration, ILogger<ImdbDbSe
             async connection =>
             {
                 logger.LogInformation("Dropping Trigrams index if it exists already");
-                await using var dropCommand = new NpgsqlCommand("DROP INDEX if exists title_gist", connection);
+                await using var dropCommand = new NpgsqlCommand("DROP INDEX if exists title_gin", connection);
                 await dropCommand.ExecuteNonQueryAsync();
             }, $"Error while dropping index on {TableNames.MetadataTable} table");
 

src/migrator/migrations/009_imdb_year_column_int.sql

@@ -0,0 +1,35 @@
+-- Purpose: Change the year column to integer and add a search function that allows for searching by year.
+ALTER TABLE imdb_metadata
+ALTER COLUMN year TYPE integer USING (CASE WHEN year = '\N' THEN 0 ELSE year::integer END);
+
+-- Remove the old search function
+DROP FUNCTION IF EXISTS search_imdb_meta(TEXT, TEXT, TEXT, INT);
+
+-- Add the new search function that allows for searching by year with a plus/minus one year range
+CREATE OR REPLACE FUNCTION search_imdb_meta(search_term TEXT, category_param TEXT DEFAULT NULL, year_param INT DEFAULT NULL, limit_param INT DEFAULT 10)
+    RETURNS TABLE(imdb_id character varying(16), title character varying(1000),category character varying(50),year INT, score REAL) AS $$
+BEGIN
+    SET pg_trgm.similarity_threshold = 0.9;
+    RETURN QUERY
+        SELECT imdb_metadata.imdb_id, imdb_metadata.title, imdb_metadata.category, imdb_metadata.year, similarity(imdb_metadata.title, search_term) as score
+        FROM imdb_metadata
+        WHERE (imdb_metadata.title % search_term)
+          AND (imdb_metadata.adult = FALSE)
+          AND (category_param IS NULL OR imdb_metadata.category = category_param)
+          AND (year_param IS NULL OR imdb_metadata.year BETWEEN year_param - 1 AND year_param + 1)
+        ORDER BY score DESC
+        LIMIT limit_param;
+END; $$
+    LANGUAGE plpgsql;
+
+-- Drop the old indexes
+DROP INDEX IF EXISTS idx_imdb_metadata_adult;
+DROP INDEX IF EXISTS idx_imdb_metadata_category;
+DROP INDEX IF EXISTS idx_imdb_metadata_year;
+DROP INDEX IF EXISTS title_gist;
+
+-- Add indexes for the new columns
+CREATE INDEX idx_imdb_metadata_adult ON imdb_metadata(adult);
+CREATE INDEX idx_imdb_metadata_category ON imdb_metadata(category);
+CREATE INDEX idx_imdb_metadata_year ON imdb_metadata(year);
+CREATE INDEX title_gin ON imdb_metadata USING gin(title gin_trgm_ops);

src/migrator/migrations/010_add_rtn_response_to_ingested_and_torrent.sql

@@ -0,0 +1,40 @@
+-- Purpose: Add the jsonb column to the ingested_torrents table to store the response from RTN
+ALTER TABLE ingested_torrents
+ADD COLUMN IF NOT EXISTS rtn_response jsonb;
+
+-- Purpose: Drop torrentId column from torrents table
+ALTER TABLE torrents
+DROP COLUMN IF EXISTS "torrentId";
+
+-- Purpose: Drop Trackers column from torrents table
+ALTER TABLE torrents
+DROP COLUMN IF EXISTS "trackers";
+
+-- Purpose: Create a foreign key relationsship if it does not already exist between torrents and the source table ingested_torrents, but do not cascade on delete.
+ALTER TABLE torrents
+ADD COLUMN IF NOT EXISTS "ingestedTorrentId" bigint;
+
+DO $$
+BEGIN
+    IF EXISTS (
+        SELECT 1
+        FROM   information_schema.table_constraints 
+        WHERE  constraint_name = 'fk_torrents_info_hash'
+    )
+    THEN
+        ALTER TABLE torrents
+        DROP CONSTRAINT fk_torrents_info_hash;
+    END IF;
+END $$;
+
+ALTER TABLE torrents
+ADD CONSTRAINT fk_torrents_info_hash
+FOREIGN KEY ("ingestedTorrentId")
+REFERENCES ingested_torrents("id")
+ON DELETE NO ACTION;
+
+UPDATE torrents
+SET "ingestedTorrentId" = ingested_torrents."id"
+FROM ingested_torrents
+WHERE torrents."infoHash" = ingested_torrents."info_hash"
+AND torrents."provider" = ingested_torrents."source";

src/migrator/migrations/011_housekeeping_reconciliation_imdbIds_dmm.sql

@@ -0,0 +1,55 @@
+DROP FUNCTION IF EXISTS kc_maintenance_reconcile_dmm_imdb_ids();
+CREATE OR REPLACE FUNCTION kc_maintenance_reconcile_dmm_imdb_ids()
+RETURNS INTEGER AS $$
+DECLARE
+    rec RECORD;
+    imdb_rec RECORD;
+    rows_affected INTEGER := 0;
+BEGIN
+    RAISE NOTICE 'Starting Reconciliation of DMM IMDB Ids...';
+    FOR rec IN
+        SELECT 
+            it."id" as "ingestion_id",
+            t."infoHash",
+            it."category" as "ingestion_category",
+            f."id" as "file_Id",
+            f."title" as "file_Title",
+            (rtn_response->>'raw_title')::text as "raw_title",
+            (rtn_response->>'parsed_title')::text as "parsed_title",
+            (rtn_response->>'year')::int as "year"
+        FROM torrents t
+        JOIN ingested_torrents it ON t."ingestedTorrentId" = it."id"
+        JOIN files f ON t."infoHash" = f."infoHash"
+        WHERE t."provider" = 'DMM'
+    LOOP
+        RAISE NOTICE 'Processing record with file_Id: %', rec."file_Id";
+        FOR imdb_rec IN
+            SELECT * FROM search_imdb_meta(
+                rec."parsed_title",
+                    CASE 
+                        WHEN rec."ingestion_category" = 'tv' THEN 'tvSeries'
+                        WHEN rec."ingestion_category" = 'movies' THEN 'movie' 
+                    END, 
+                    CASE 
+                        WHEN rec."year" = 0 THEN NULL 
+                        ELSE rec."year" END,
+                1)
+        LOOP
+            IF imdb_rec IS NOT NULL THEN
+                RAISE NOTICE 'Updating file_Id: % with imdbId: %, parsed title: %, imdb title: %', rec."file_Id", imdb_rec."imdb_id", rec."parsed_title", imdb_rec."title";
+                UPDATE "files"
+                SET "imdbId" = imdb_rec."imdb_id"
+                WHERE "id" = rec."file_Id";
+                rows_affected := rows_affected + 1;
+            ELSE
+                RAISE NOTICE 'No IMDB ID found for file_Id: %, parsed title: %, imdb title: %, setting imdbId to NULL', rec."file_Id", rec."parsed_title", imdb_rec."title";
+                UPDATE "files"
+                SET "imdbId" = NULL
+                WHERE "id" = rec."file_Id";
+            END IF;
+        END LOOP;
+    END LOOP;
+    RAISE NOTICE 'Finished reconciliation. Total rows affected: %', rows_affected;
+    RETURN rows_affected;
+END;
+$$ LANGUAGE plpgsql;

src/migrator/migrations/012_imdb_change_ratio.sql

@@ -0,0 +1,19 @@
+-- Remove the old search function
+DROP FUNCTION IF EXISTS search_imdb_meta(TEXT, TEXT, INT, INT);
+
+-- Add the new search function that allows for searching by year with a plus/minus one year range
+CREATE OR REPLACE FUNCTION search_imdb_meta(search_term TEXT, category_param TEXT DEFAULT NULL, year_param INT DEFAULT NULL, limit_param INT DEFAULT 10, similarity_threshold REAL DEFAULT 0.95)
+    RETURNS TABLE(imdb_id character varying(16), title character varying(1000),category character varying(50),year INT, score REAL) AS $$
+BEGIN
+    SET pg_trgm.similarity_threshold = similarity_threshold;
+    RETURN QUERY
+        SELECT imdb_metadata.imdb_id, imdb_metadata.title, imdb_metadata.category, imdb_metadata.year, similarity(imdb_metadata.title, search_term) as score
+        FROM imdb_metadata
+        WHERE (imdb_metadata.title % search_term)
+          AND (imdb_metadata.adult = FALSE)
+          AND (category_param IS NULL OR imdb_metadata.category = category_param)
+          AND (year_param IS NULL OR imdb_metadata.year BETWEEN year_param - 1 AND year_param + 1)
+        ORDER BY score DESC
+        LIMIT limit_param;
+END; $$
+    LANGUAGE plpgsql;

src/migrator/migrations/013_imdb_change_ratio_fix.sql

@@ -0,0 +1,19 @@
+-- Remove the old search function
+DROP FUNCTION IF EXISTS search_imdb_meta(TEXT, TEXT, INT, INT);
+
+-- Add the new search function that allows for searching by year with a plus/minus one year range
+CREATE OR REPLACE FUNCTION search_imdb_meta(search_term TEXT, category_param TEXT DEFAULT NULL, year_param INT DEFAULT NULL, limit_param INT DEFAULT 10, similarity_threshold REAL DEFAULT 0.95)
+    RETURNS TABLE(imdb_id character varying(16), title character varying(1000),category character varying(50),year INT, score REAL) AS $$
+BEGIN
+    EXECUTE format('SET pg_trgm.similarity_threshold = %L', similarity_threshold);
+    RETURN QUERY
+        SELECT imdb_metadata.imdb_id, imdb_metadata.title, imdb_metadata.category, imdb_metadata.year, similarity(imdb_metadata.title, search_term) as score
+        FROM imdb_metadata
+        WHERE (imdb_metadata.title % search_term)
+          AND (imdb_metadata.adult = FALSE)
+          AND (category_param IS NULL OR imdb_metadata.category = category_param)
+          AND (year_param IS NULL OR imdb_metadata.year BETWEEN year_param - 1 AND year_param + 1)
+        ORDER BY score DESC
+        LIMIT limit_param;
+END; $$
+    LANGUAGE plpgsql;

src/producer/.dockerignore

@@ -0,0 +1,2 @@
+**/python/
+.idea/

src/producer/Producer.sln

@@ -6,6 +6,12 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SharedContracts", "..\share
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "shared", "shared", "{FF5CA857-51E8-4446-8840-2A1D24ED3952}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "eng", "eng", "{1AE7F597-24C4-4575-B59F-67A625D95C1E}"
+	ProjectSection(SolutionItems) = preProject
+		eng\install-python-reqs.ps1 = eng\install-python-reqs.ps1
+		eng\install-python-reqs.sh = eng\install-python-reqs.sh
+	EndProjectSection
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU

src/producer/eng/install-python-reqs.ps1

@@ -0,0 +1,3 @@
+remove-item -recurse -force ../src/python
+mkdir -p ../src/python
+pip install -r ../src/requirements.txt -t ../src/python/

src/producer/eng/install-python-reqs.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+rm -rf ../src/python
+mkdir -p ../src/python
+python3 -m pip install -r ../src/requirements.txt -t ../src/python/

src/producer/src/.dockerignore

@@ -0,0 +1,2 @@
+**/python/
+.idea/

src/producer/src/Dockerfile

@@ -8,13 +8,27 @@ WORKDIR /src/producer/src
 RUN dotnet restore -a $TARGETARCH
 RUN dotnet publish -c Release --no-restore -o /src/out -a $TARGETARCH
 
-
-FROM mcr.microsoft.com/dotnet/aspnet:8.0-alpine
+FROM mcr.microsoft.com/dotnet/aspnet:8.0-alpine3.19
 
 WORKDIR /app
+
+ENV PYTHONUNBUFFERED=1
+
+RUN apk add --update --no-cache python3=~3.11.8-r0 py3-pip && ln -sf python3 /usr/bin/python
+
 COPY --from=build /src/out .
+
+RUN rm -rf /app/python && mkdir -p /app/python
+
+RUN pip3 install -r /app/requirements.txt -t /app/python
+
 RUN addgroup -S producer && adduser -S -G producer producer
+
 USER producer
+
 HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 \
   CMD pgrep -f dotnet || exit 1
+
+ENV PYTHONNET_PYDLL=/usr/lib/libpython3.11.so.1.0
+
 ENTRYPOINT ["dotnet", "Producer.dll"]

src/producer/src/Features/Crawlers/Dmm/DebridMediaManagerCrawler.cs

@@ -5,7 +5,7 @@ public partial class DebridMediaManagerCrawler(
     ILogger<DebridMediaManagerCrawler> logger,
     IDataStorage storage,
     GithubConfiguration githubConfiguration,
-    IParseTorrentTitle parseTorrentTitle,
+    IRankTorrentName rankTorrentName,
     IDistributedCache cache) : BaseCrawler(logger, storage)
 {
     [GeneratedRegex("""<iframe src="https:\/\/debridmediamanager.com\/hashlist#(.*)"></iframe>""")]
@@ -108,99 +108,68 @@ public partial class DebridMediaManagerCrawler(
             return null;
         }
         
-        var parsedTorrent = parseTorrentTitle.Parse(torrentTitle.CleanTorrentTitleForImdb());
+        var parsedTorrent = rankTorrentName.Parse(torrentTitle);
         
-        var (cached, cachedResult) = await CheckIfInCacheAndReturn(parsedTorrent.Title);
+        if (!parsedTorrent.Success)
+        {
+            return null;
+        }
+        
+        var torrentType = parsedTorrent.Response.IsMovie ? "movie" : "tvSeries";
+        
+        var cacheKey = GetCacheKey(torrentType, parsedTorrent.Response.ParsedTitle, parsedTorrent.Response.Year);
+        
+        var (cached, cachedResult) = await CheckIfInCacheAndReturn(cacheKey);
         
         if (cached)
         {
-            logger.LogInformation("[{ImdbId}] Found cached imdb result for {Title}", cachedResult.ImdbId, parsedTorrent.Title);
-            return new()
-            {
-                Source = Source,
-                Name = cachedResult.Title,
-                Imdb = cachedResult.ImdbId,
-                Size = bytesElement.GetInt64().ToString(),
-                InfoHash = hashElement.ToString(),
-                Seeders = 0,
-                Leechers = 0,
-                Category = parsedTorrent.TorrentType switch
-                {
-                    TorrentType.Movie => "movies",
-                    TorrentType.Tv => "tv",
-                    _ => "unknown",
-                },
-            };
+            logger.LogInformation("[{ImdbId}] Found cached imdb result for {Title}", cachedResult.ImdbId, parsedTorrent.Response.ParsedTitle);
+            return MapToTorrent(cachedResult, bytesElement, hashElement, parsedTorrent);
         }
 
-        var imdbEntry = await Storage.FindImdbMetadata(parsedTorrent.Title, parsedTorrent.TorrentType, parsedTorrent.Year);
+        int? year = parsedTorrent.Response.Year != 0 ? parsedTorrent.Response.Year : null;
+        var imdbEntry = await Storage.FindImdbMetadata(parsedTorrent.Response.ParsedTitle, torrentType, year);
 
-        if (imdbEntry.Count == 0)
+        if (imdbEntry is null)
         {
             return null;
         }
         
-        var scoredTitles = await ScoreTitles(parsedTorrent, imdbEntry);
+        await AddToCache(cacheKey, imdbEntry);
         
-        if (!scoredTitles.Success)
-        {
-            return null;
+        logger.LogInformation("[{ImdbId}] Found best match for {Title}: {BestMatch} with score {Score}", imdbEntry.ImdbId, parsedTorrent.Response.ParsedTitle, imdbEntry.Title, imdbEntry.Score);
+
+        return MapToTorrent(imdbEntry, bytesElement, hashElement, parsedTorrent);
     }
 
-        logger.LogInformation("[{ImdbId}] Found best match for {Title}: {BestMatch} with score {Score}", scoredTitles.BestMatch.Value.ImdbId, parsedTorrent.Title, scoredTitles.BestMatch.Value.Title, scoredTitles.BestMatch.Score);
-
-        var torrent = new IngestedTorrent
+    private IngestedTorrent MapToTorrent(ImdbEntry result, JsonElement bytesElement, JsonElement hashElement, ParseTorrentTitleResponse parsedTorrent) =>
+        new()
         {
             Source = Source,
-            Name = scoredTitles.BestMatch.Value.Title,
-            Imdb = scoredTitles.BestMatch.Value.ImdbId,
+            Name = result.Title,
+            Imdb = result.ImdbId,
             Size = bytesElement.GetInt64().ToString(),
             InfoHash = hashElement.ToString(),
             Seeders = 0,
             Leechers = 0,
-            Category = parsedTorrent.TorrentType switch
-            {
-                TorrentType.Movie => "movies",
-                TorrentType.Tv => "tv",
-                _ => "unknown",
-            },
+            Category = AssignCategory(result),
+            RtnResponse = parsedTorrent.Response.ToJson(),
         };
     
-        return torrent;
-    }
 
-    private async Task<(bool Success, ExtractedResult<ImdbEntry>? BestMatch)> ScoreTitles(TorrentMetadata parsedTorrent, List<ImdbEntry> imdbEntries)
-    {
-        var lowerCaseTitle = parsedTorrent.Title.ToLowerInvariant();
-       
-        // Scoring directly operates on the List<ImdbEntry>, no need for lookup table.
-        var scoredResults = Process.ExtractAll(new(){Title = lowerCaseTitle}, imdbEntries, x => x.Title?.ToLowerInvariant(), scorer: new DefaultRatioScorer(), cutoff: 90);
-
-        var best = scoredResults.MaxBy(x => x.Score);
-
-        if (best is null)
-        {
-            return (false, null);
-        }
-        
-        await AddToCache(lowerCaseTitle, best);
-        
-        return (true, best);
-    }
-
-    private Task AddToCache(string lowerCaseTitle, ExtractedResult<ImdbEntry> best)
+    private Task AddToCache(string cacheKey, ImdbEntry best)
     {
         var cacheOptions = new DistributedCacheEntryOptions
         {
-            AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(15),
+            AbsoluteExpirationRelativeToNow = TimeSpan.FromDays(1),
         };
         
-        return cache.SetStringAsync(lowerCaseTitle, JsonSerializer.Serialize(best.Value), cacheOptions);
+        return cache.SetStringAsync(cacheKey, JsonSerializer.Serialize(best), cacheOptions);
     }
 
-    private async Task<(bool Success, ImdbEntry? Entry)> CheckIfInCacheAndReturn(string title)
+    private async Task<(bool Success, ImdbEntry? Entry)> CheckIfInCacheAndReturn(string cacheKey)
     {
-        var cachedImdbId = await cache.GetStringAsync(title.ToLowerInvariant());
+        var cachedImdbId = await cache.GetStringAsync(cacheKey);
         
         if (!string.IsNullOrEmpty(cachedImdbId))
         {
@@ -240,4 +209,14 @@ public partial class DebridMediaManagerCrawler(
 
         return (pageIngested, name);
     }
+    
+    private static string AssignCategory(ImdbEntry entry) =>
+        entry.Category.ToLower() switch
+        {
+            var category when string.Equals(category, "movie", StringComparison.OrdinalIgnoreCase) => "movies",
+            var category when string.Equals(category, "tvSeries", StringComparison.OrdinalIgnoreCase) => "tv",
+            _ => "unknown",
+        };
+    
+    private static string GetCacheKey(string category, string title, int year) => $"{category.ToLowerInvariant()}:{year}:{title.ToLowerInvariant()}";
 }

src/producer/src/Features/DataProcessing/LengthAwareRatioScorer.cs

@@ -0,0 +1,24 @@
+namespace Producer.Features.DataProcessing
+{
+    public class LengthAwareRatioScorer : IRatioScorer
+    {
+        private readonly IRatioScorer _defaultScorer = new DefaultRatioScorer();
+
+        public int Score(string input1, string input2)
+        {
+            var score = _defaultScorer.Score(input1, input2);
+            var lengthRatio = (double)Math.Min(input1.Length, input2.Length) / Math.Max(input1.Length, input2.Length);
+            var result = (int)(score * lengthRatio);
+            return result > 100 ? 100 : result;
+        }
+
+        public int Score(string input1, string input2, PreprocessMode preprocessMode)
+        {
+            var score = _defaultScorer.Score(input1, input2, preprocessMode);
+            var lengthRatio = (double)Math.Min(input1.Length, input2.Length) / Math.Max(input1.Length, input2.Length);
+            var result = (int)(score * lengthRatio);
+
+            return result > 100 ? 100 : result;
+        }
+    }
+}

src/producer/src/Features/DataProcessing/ServiceCollectionExtensions.cs

@@ -9,7 +9,8 @@ internal static class ServiceCollectionExtensions
        
         services.AddTransient<IDataStorage, DapperDataStorage>();
         services.AddTransient<IMessagePublisher, TorrentPublisher>();
-        services.AddSingleton<IParseTorrentTitle, ParseTorrentTitle>();
+        services.RegisterPythonEngine();
+        services.AddSingleton<IRankTorrentName, RankTorrentName>();
         services.AddStackExchangeRedisCache(options =>
         {
             options.Configuration = redisConfiguration.ConnectionString;

src/producer/src/GlobalUsings.cs

@@ -7,6 +7,8 @@ global using System.Text.RegularExpressions;
 global using System.Xml.Linq;
 global using FuzzySharp;
 global using FuzzySharp.Extractor;
+global using FuzzySharp.PreProcess;
+global using FuzzySharp.SimilarityRatio.Scorer;
 global using FuzzySharp.SimilarityRatio.Scorer.StrategySensitive;
 global using LZStringCSharp;
 global using MassTransit;
@@ -23,11 +25,10 @@ global using Producer.Features.Crawlers.Torrentio;
 global using Producer.Features.CrawlerSupport;
 global using Producer.Features.DataProcessing;
 global using Producer.Features.JobSupport;
-global using PromKnight.ParseTorrentTitle;
-global using Serilog;
 global using SharedContracts.Configuration;
 global using SharedContracts.Dapper;
 global using SharedContracts.Extensions;
 global using SharedContracts.Models;
+global using SharedContracts.Python;
+global using SharedContracts.Python.RTN;
 global using SharedContracts.Requests;
-global using StackExchange.Redis;

src/producer/src/Producer.csproj

@@ -19,6 +19,7 @@
       <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
       <PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
       <PackageReference Include="Polly" Version="8.3.0" />
+      <PackageReference Include="pythonnet" Version="3.0.3" />
       <PackageReference Include="Quartz.Extensions.DependencyInjection" Version="3.8.0" />
       <PackageReference Include="Quartz.Extensions.Hosting" Version="3.8.0" />
       <PackageReference Include="Serilog" Version="3.1.1" />
@@ -32,11 +33,14 @@
       <None Include="Configuration\*.json">
         <CopyToOutputDirectory>Always</CopyToOutputDirectory>
       </None>
+      <None Update="requirements.txt">
+        <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+      </None>
     </ItemGroup>
 
-    <ItemGroup>
-        <Content Remove="Data\**" />
-        <None Include="Data\**">
+    <ItemGroup Condition="'$(Configuration)' == 'Debug'">
+        <Content Remove="python\**" />
+        <None Include="python\**">
             <CopyToOutputDirectory>Always</CopyToOutputDirectory>
         </None>
     </ItemGroup>

src/producer/src/requirements.txt

@@ -0,0 +1 @@
+rank-torrent-name==0.1.8

src/qbit-collector/Extensions/ServiceCollectionExtensions.cs

@@ -18,6 +18,7 @@ public static class ServiceCollectionExtensions
         services.AddHttpClient();
         services.AddSingleton<ITrackersService, TrackersService>();
         services.AddHostedService<TrackersBackgroundService>();
+        services.AddHostedService<HousekeepingBackgroundService>();
 
         return services;
     }

src/qbit-collector/Features/Qbit/HousekeepingBackgroundService.cs

@@ -0,0 +1,52 @@
+namespace QBitCollector.Features.Qbit;
+
+public class HousekeepingBackgroundService(IQBittorrentClient client, ILogger<HousekeepingBackgroundService> logger) : BackgroundService
+{
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        logger.LogInformation("Service is Running.");
+
+        await DoWork();
+
+        using PeriodicTimer timer = new(TimeSpan.FromMinutes(2));
+
+        try
+        {
+            while (await timer.WaitForNextTickAsync(stoppingToken))
+            {
+                await DoWork();
+            }
+        }
+        catch (OperationCanceledException)
+        {
+            logger.LogInformation("Service stopping.");
+        }
+    }
+
+    private async Task DoWork()
+    {
+        try
+        {
+            logger.LogInformation("Cleaning Stale Entries in Qbit...");
+
+            var torrents = await client.GetTorrentListAsync();
+
+            foreach (var torrentInfo in torrents)
+            {
+                if (!(torrentInfo.AddedOn < DateTimeOffset.UtcNow.AddMinutes(-1)))
+                {
+                    continue;
+                }
+                
+                logger.LogInformation("Torrent [{InfoHash}] Identified as stale because was added at {AddedOn}", torrentInfo.Hash, torrentInfo.AddedOn);
+
+                await client.DeleteAsync(new[] {torrentInfo.Hash}, deleteDownloadedData: true);
+                logger.LogInformation("Cleaned up stale torrent: [{InfoHash}]", torrentInfo.Hash);
+            }
+        }
+        catch (Exception e)
+        {
+            logger.LogError(e, "Error cleaning up stale torrents this interval.");
+        }
+    }
+}

src/qbit-collector/Features/Qbit/QbitConfiguration.cs

@@ -3,7 +3,9 @@ namespace QBitCollector.Features.Qbit;
 public class QbitConfiguration
 {
     private const string Prefix = "QBIT";
-    private const string ConnectionStringVariable = "HOST";
+    private const string HOST_VARIABLE = "HOST";
+    private const string TRACKERS_URL_VARIABLE = "TRACKERS_URL";
     
-    public string? Host { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(ConnectionStringVariable);
+    public string? Host { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(HOST_VARIABLE);
+    public string? TrackersUrl { get; init; } = Prefix.GetRequiredEnvironmentVariableAsString(TRACKERS_URL_VARIABLE);
 }

src/qbit-collector/Features/Trackers/TrackersService.cs

@@ -1,8 +1,7 @@
 namespace QBitCollector.Features.Trackers;
 
-public class TrackersService(IDistributedCache cache, HttpClient client, IMemoryCache memoryCache) : ITrackersService
+public class TrackersService(IDistributedCache cache, HttpClient client, IMemoryCache memoryCache, QbitConfiguration configuration) : ITrackersService
 {
-    private const string TrackersListUrl = "https://ngosang.github.io/trackerslist/trackers_all.txt";
     private const string CacheKey = "trackers";
 
     public async Task<List<string>> GetTrackers()
@@ -42,7 +41,7 @@ public class TrackersService(IDistributedCache cache, HttpClient client, IMemory
 
     private async Task<List<string>> GetTrackersAsync()
     {
-        var response = await client.GetStringAsync(TrackersListUrl);
+        var response = await client.GetStringAsync(configuration.TrackersUrl);
 
         var lines = response.Split(["\r\n", "\r", "\n"], StringSplitOptions.None);
 

src/qbit-collector/Features/Worker/Requests.cs

@@ -1,22 +1,24 @@
 namespace QBitCollector.Features.Worker;
 
-[EntityName("perform-metadata-request")]
+[EntityName("perform-metadata-request-qbit-collector")]
 public record PerformQbitMetadataRequest(Guid CorrelationId, string InfoHash) : CorrelatedBy<Guid>;
 
-[EntityName("torrent-metadata-response")]
+[EntityName("torrent-metadata-response-qbit-collector")]
 public record GotQbitMetadata(QBitMetadataResponse Metadata) : CorrelatedBy<Guid>
 {
     public Guid CorrelationId { get; init; } = Metadata.CorrelationId;
 }
 
-[EntityName("write-metadata")]
+[EntityName("write-metadata-qbit-collector")]
 public record WriteQbitMetadata(Torrent Torrent, QBitMetadataResponse Metadata, string ImdbId) : CorrelatedBy<Guid>
 {
     public Guid CorrelationId { get; init; } = Metadata.CorrelationId;
 }
 
-[EntityName("metadata-written")]
+[EntityName("metadata-written-qbit-collector")]
 public record QbitMetadataWritten(QBitMetadataResponse Metadata) : CorrelatedBy<Guid>
 {
     public Guid CorrelationId { get; init; } = Metadata.CorrelationId;
+
+    public QBitMetadataResponse Metadata { get; init; } = Metadata;
 }

src/shared/Dapper/DapperDataStorage.cs

@@ -9,9 +9,9 @@ public class DapperDataStorage(PostgresConfiguration configuration, RabbitMqConf
             const string query = 
                 """
                 INSERT INTO ingested_torrents
-                    ("name", "source", "category", "info_hash", "size", "seeders", "leechers", "imdb", "processed", "createdAt", "updatedAt")
+                    ("name", "source", "category", "info_hash", "size", "seeders", "leechers", "imdb", "processed", "createdAt", "updatedAt", "rtn_response")
                 VALUES
-                    (@Name, @Source, @Category, @InfoHash, @Size, @Seeders, @Leechers, @Imdb, @Processed, @CreatedAt, @UpdatedAt)
+                    (@Name, @Source, @Category, @InfoHash, @Size, @Seeders, @Leechers, @Imdb, @Processed, @CreatedAt, @UpdatedAt, @RtnResponse::jsonb)
                 ON CONFLICT (source, info_hash) DO NOTHING
                 """;
             
@@ -110,21 +110,21 @@ public class DapperDataStorage(PostgresConfiguration configuration, RabbitMqConf
     public async Task<List<ImdbEntry>> GetImdbEntriesForRequests(int year, int batchSize, string? stateLastProcessedImdbId, CancellationToken cancellationToken = default) =>
         await ExecuteCommandAsync(async connection =>
         {
-            const string query = @"SELECT imdb_id AS ImdbId, title as Title, category as Category, year as Year, adult as Adult FROM imdb_metadata WHERE CAST(NULLIF(Year, '\N') AS INTEGER) <= @Year AND imdb_id > @LastProcessedImdbId ORDER BY ImdbId LIMIT @BatchSize";
+            const string query = @"SELECT imdb_id AS ImdbId, title as Title, category as Category, year as Year, adult as Adult FROM imdb_metadata WHERE Year <= @Year AND imdb_id > @LastProcessedImdbId ORDER BY ImdbId LIMIT @BatchSize";
             var result = await connection.QueryAsync<ImdbEntry>(query, new { Year = year, LastProcessedImdbId = stateLastProcessedImdbId, BatchSize = batchSize });
             return result.ToList();
         }, "Error getting imdb metadata.", cancellationToken);
 
-    public async Task<List<ImdbEntry>> FindImdbMetadata(string? parsedTorrentTitle, TorrentType torrentType, string? year, CancellationToken cancellationToken = default) =>
+    public async Task<ImdbEntry?> FindImdbMetadata(string? parsedTorrentTitle, string torrentType, int? year, CancellationToken cancellationToken = default) =>
         await ExecuteCommandAsync(async connection =>
         {
-            var query = $"select \"imdb_id\" as \"ImdbId\", \"title\" as \"Title\", \"year\" as \"Year\" from search_imdb_meta('{parsedTorrentTitle.Replace("'", "").Replace("\"", "")}', '{(torrentType == TorrentType.Movie ? "movie" : "tvSeries")}'";
-            query += year is not null ? $", '{year}'" : ", NULL";
-            query += ", 15)";
+            var query = $"select \"imdb_id\" as \"ImdbId\", \"title\" as \"Title\", \"year\" as \"Year\", \"score\" as Score, \"category\" as Category from search_imdb_meta('{parsedTorrentTitle.Replace("'", "").Replace("\"", "")}', '{torrentType}'";
+            query += year is not null ? $", {year}" : ", NULL";
+            query += ", 1)";
             
             var result = await connection.QueryAsync<ImdbEntry>(query);
-
-            return result.ToList();
+            var results = result.ToList();
+            return results.FirstOrDefault();
         }, "Error finding imdb metadata.", cancellationToken);
     
     public Task InsertTorrent(Torrent torrent, CancellationToken cancellationToken = default) =>
@@ -134,9 +134,9 @@ public class DapperDataStorage(PostgresConfiguration configuration, RabbitMqConf
                 const string query = 
                     """
                     INSERT INTO "torrents"
-                        ("infoHash", "provider", "torrentId", "title", "size", "type", "uploadDate", "seeders", "trackers", "languages", "resolution", "reviewed", "opened", "createdAt", "updatedAt")
+                        ("infoHash", "ingestedTorrentId", "provider", "title", "size", "type", "uploadDate", "seeders", "languages", "resolution", "reviewed", "opened", "createdAt", "updatedAt")
                     VALUES
-                        (@InfoHash, @Provider, @TorrentId, @Title, 0, @Type, NOW(), @Seeders, NULL, NULL, NULL, false, false, NOW(), NOW())
+                        (@InfoHash, @IngestedTorrentId, @Provider, @Title, 0, @Type, NOW(), @Seeders, NULL, NULL, false, false, NOW(), NOW())
                     ON CONFLICT ("infoHash") DO NOTHING
                     """;
                 

src/shared/Dapper/IDataStorage.cs

@@ -9,7 +9,7 @@ public interface IDataStorage
     Task<DapperResult<PageIngestedResult, PageIngestedResult>> MarkPageAsIngested(string pageId, CancellationToken cancellationToken = default);
     Task<DapperResult<int, int>> GetRowCountImdbMetadata(CancellationToken cancellationToken = default);
     Task<List<ImdbEntry>> GetImdbEntriesForRequests(int year, int batchSize, string? stateLastProcessedImdbId, CancellationToken cancellationToken = default);
-    Task<List<ImdbEntry>> FindImdbMetadata(string? parsedTorrentTitle, TorrentType parsedTorrentTorrentType, string? parsedTorrentYear, CancellationToken cancellationToken = default);
+    Task<ImdbEntry?> FindImdbMetadata(string? parsedTorrentTitle, string parsedTorrentTorrentType, int? parsedTorrentYear, CancellationToken cancellationToken = default);
     Task InsertTorrent(Torrent torrent, CancellationToken cancellationToken = default);
     Task InsertFiles(IEnumerable<TorrentFile> files, CancellationToken cancellationToken = default);
     Task InsertSubtitles(IEnumerable<SubtitleFile> subtitles, CancellationToken cancellationToken = default);

src/shared/Extensions/ConfigurationExtensions.cs

@@ -1,4 +1,3 @@
-using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.DependencyInjection.Extensions;
 
 namespace SharedContracts.Extensions;

src/shared/Extensions/JsonExtensions.cs

@@ -0,0 +1,14 @@
+namespace SharedContracts.Extensions;
+
+public static class JsonExtensions
+{
+    private static readonly JsonSerializerOptions JsonSerializerOptions = new()
+    {
+        PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
+        WriteIndented = false,
+        ReferenceHandler = ReferenceHandler.IgnoreCycles,
+        NumberHandling = JsonNumberHandling.Strict,
+    };
+    
+    public static string AsJson<T>(this T obj) => JsonSerializer.Serialize(obj, JsonSerializerOptions);
+}

src/shared/Extensions/StringExtensions.cs

@@ -1,5 +1,3 @@
-using System.Text.RegularExpressions;
-
 namespace SharedContracts.Extensions;
 
 public static partial class StringExtensions

src/shared/GlobalUsings.cs

@@ -1,15 +1,18 @@
 // Global using directives
 
 global using System.Text.Json;
+global using System.Text.Json.Serialization;
+global using System.Text.RegularExpressions;
 global using Dapper;
 global using MassTransit;
 global using Microsoft.AspNetCore.Builder;
 global using Microsoft.AspNetCore.Hosting;
 global using Microsoft.Extensions.Configuration;
+global using Microsoft.Extensions.DependencyInjection;
 global using Microsoft.Extensions.Hosting;
 global using Microsoft.Extensions.Logging;
 global using Npgsql;
-global using PromKnight.ParseTorrentTitle;
+global using Python.Runtime;
 global using Serilog;
 global using SharedContracts.Configuration;
 global using SharedContracts.Extensions;

src/shared/Models/ImdbEntry.cs

@@ -7,4 +7,5 @@ public class ImdbEntry
     public string? Category { get; set; }
     public string? Year { get; set; }
     public bool? Adult { get; set; }
+    public decimal? Score { get; set; }
 }

src/shared/Models/IngestedTorrent.cs

@@ -12,7 +12,9 @@ public class IngestedTorrent
     public int Leechers { get; set; }
     public string? Imdb { get; set; }
 
-    public bool Processed { get; set; } = false;
+    public bool Processed { get; set; }
     public DateTime CreatedAt { get; set; } = DateTime.UtcNow;
     public DateTime UpdatedAt { get; set; } = DateTime.UtcNow;
+    
+    public string? RtnResponse { get; set; }
 }

src/shared/Models/Torrent.cs

@@ -3,6 +3,7 @@ namespace SharedContracts.Models;
 public class Torrent
 {
     public string? InfoHash { get; set; }
+    public long? IngestedTorrentId { get; set; }
     public string? Provider { get; set; }
     public string? TorrentId { get; set; }
     public string? Title { get; set; }

src/shared/Python/IPythonEngineService.cs

@@ -0,0 +1,13 @@
+namespace SharedContracts.Python;
+
+public interface IPythonEngineService
+{
+    ILogger<PythonEngineService> Logger { get; }
+
+    Task InitializePythonEngine(CancellationToken cancellationToken);
+    T ExecuteCommandOrScript<T>(string command, PyModule module, bool throwOnErrors);
+    T ExecutePythonOperation<T>(Func<T> operation, string operationName, bool throwOnErrors);
+    T ExecutePythonOperationWithDefault<T>(Func<T> operation, T? defaultValue, string operationName, bool throwOnErrors, bool logErrors);
+    Task StopPythonEngine(CancellationToken cancellationToken);
+    dynamic? Sys { get; }
+}

src/shared/Python/PythonEngineManager.cs

@@ -0,0 +1,8 @@
+namespace SharedContracts.Python;
+
+public class PythonEngineManager(IPythonEngineService pythonEngineService) : IHostedService
+{
+    public Task StartAsync(CancellationToken cancellationToken) => pythonEngineService.InitializePythonEngine(cancellationToken);
+
+    public Task StopAsync(CancellationToken cancellationToken) => pythonEngineService.StopPythonEngine(cancellationToken);
+}

src/shared/Python/PythonEngineService.cs

@@ -0,0 +1,124 @@
+namespace SharedContracts.Python;
+
+public class PythonEngineService(ILogger<PythonEngineService> logger) : IPythonEngineService
+{
+    private IntPtr _mainThreadState;
+    private bool _isInitialized;
+
+    public ILogger<PythonEngineService> Logger { get; } = logger;
+
+    public dynamic? Sys { get; private set; }
+
+    public Task InitializePythonEngine(CancellationToken cancellationToken)
+    {
+        if (_isInitialized)
+        {
+            return Task.CompletedTask;
+        }
+
+        try
+        {
+            var pythonDllEnv = Environment.GetEnvironmentVariable("PYTHONNET_PYDLL");
+
+            if (string.IsNullOrWhiteSpace(pythonDllEnv))
+            {
+                Logger.LogWarning("PYTHONNET_PYDLL env is not set. Exiting Application");
+                Environment.Exit(1);
+                return Task.CompletedTask;
+            }
+
+            Runtime.PythonDLL = pythonDllEnv;
+            PythonEngine.Initialize();
+            _mainThreadState = PythonEngine.BeginAllowThreads();
+
+            _isInitialized = true;
+            Logger.LogInformation("Python engine initialized");
+        }
+        catch (Exception e)
+        {
+            Logger.LogError(e, $"Failed to initialize Python engine: {e.Message}");
+            Environment.Exit(1);
+        }
+
+        return Task.CompletedTask;
+    }
+
+    public T ExecuteCommandOrScript<T>(string command, PyModule module, bool throwOnErrors) =>
+        ExecutePythonOperation(
+            () =>
+            {
+                var pyCompile = PythonEngine.Compile(command);
+                var nativeResult = module.Execute(pyCompile);
+                return nativeResult.As<T>();
+            }, nameof(ExecuteCommandOrScript), throwOnErrors);
+
+    public T ExecutePythonOperation<T>(Func<T> operation, string operationName, bool throwOnErrors) =>
+        ExecutePythonOperationWithDefault(operation, default, operationName, throwOnErrors, true);
+
+    public T ExecutePythonOperationWithDefault<T>(Func<T> operation, T? defaultValue, string operationName, bool throwOnErrors, bool logErrors) =>
+        ExecutePythonOperationInternal(operation, defaultValue, operationName, throwOnErrors, logErrors);
+
+    public void ExecuteOnGIL(Action act, bool throwOnErrors)
+    {
+        Sys ??= LoadSys();
+
+        try
+        {
+            using var gil = Py.GIL();
+            act();
+        }
+        catch (Exception ex)
+        {
+            Logger.LogError(ex, "Python Error: {Message} ({OperationName})", ex.Message, nameof(ExecuteOnGIL));
+
+            if (throwOnErrors)
+            {
+                throw;
+            }
+        }
+    }
+
+    public Task StopPythonEngine(CancellationToken cancellationToken)
+    {
+        PythonEngine.EndAllowThreads(_mainThreadState);
+        PythonEngine.Shutdown();
+
+        return Task.CompletedTask;
+    }
+
+    private static dynamic LoadSys()
+    {
+        using var gil = Py.GIL();
+        var sys = Py.Import("sys");
+        
+        return sys;
+    }
+
+    // ReSharper disable once EntityNameCapturedOnly.Local
+    private T ExecutePythonOperationInternal<T>(Func<T> operation, T? defaultValue, string operationName, bool throwOnErrors, bool logErrors)
+    {
+        Sys ??= LoadSys();
+
+        var result = defaultValue;
+
+        try
+        {
+            using var gil = Py.GIL();
+            result = operation();
+        }
+        catch (Exception ex)
+        {
+            if (logErrors)
+            {
+                Logger.LogError(ex, "Python Error: {Message} ({OperationName})", ex.Message, nameof(operationName));
+            }
+
+            if (throwOnErrors)
+            {
+                throw;
+            }
+        }
+
+        return result;
+    }
+}

src/shared/Python/RTN/IRankTorrentName.cs

@@ -0,0 +1,6 @@
+namespace SharedContracts.Python.RTN;
+
+public interface IRankTorrentName
+{
+    ParseTorrentTitleResponse Parse(string title);
+}

src/shared/Python/RTN/ParseTorrentTitleResponse.cs

@@ -0,0 +1,3 @@
+namespace SharedContracts.Python.RTN;
+
+public record ParseTorrentTitleResponse(bool Success, RtnResponse? Response);

src/shared/Python/RTN/RankTorrentName.cs

@@ -0,0 +1,50 @@
+namespace SharedContracts.Python.RTN;
+
+public class RankTorrentName : IRankTorrentName
+{
+    private readonly IPythonEngineService _pythonEngineService;
+    private const string RtnModuleName = "RTN";
+
+    private dynamic? _rtn;
+
+    public RankTorrentName(IPythonEngineService pythonEngineService)
+    {
+        _pythonEngineService = pythonEngineService;
+        InitModules();
+    }
+   
+    public ParseTorrentTitleResponse Parse(string title) =>
+        _pythonEngineService.ExecutePythonOperationWithDefault(
+            () =>
+            {
+                var result = _rtn?.parse(title);
+                return ParseResult(result);
+            }, new ParseTorrentTitleResponse(false, null), nameof(Parse), throwOnErrors: false, logErrors: false);
+    
+    private static ParseTorrentTitleResponse ParseResult(dynamic result)
+    {
+        if (result == null)
+        {
+            return new(false, null);
+        }
+
+        var json = result.model_dump_json()?.As<string?>();
+
+        if (json is null || string.IsNullOrEmpty(json))
+        {
+            return new(false, null);
+        }
+        
+        var response = JsonSerializer.Deserialize<RtnResponse>(json);
+
+        return new(true, response);
+    }
+
+    private void InitModules() => 
+        _rtn = 
+            _pythonEngineService.ExecutePythonOperation(() =>
+        {
+            _pythonEngineService.Sys.path.append(Path.Combine(AppContext.BaseDirectory, "python"));
+            return Py.Import(RtnModuleName);
+        }, nameof(InitModules), throwOnErrors: false);
+}

src/shared/Python/RTN/RtnResponse.cs

@@ -0,0 +1,98 @@
+namespace SharedContracts.Python.RTN;
+
+public class RtnResponse
+{
+    [JsonPropertyName("raw_title")]
+    public string? RawTitle { get; set; }
+    
+    [JsonPropertyName("parsed_title")]
+    public string? ParsedTitle { get; set; }
+    
+    [JsonPropertyName("fetch")]
+    public bool Fetch { get; set; }
+    
+    [JsonPropertyName("is_4k")]
+    public bool Is4K { get; set; }
+    
+    [JsonPropertyName("is_multi_audio")]
+    public bool IsMultiAudio { get; set; }
+    
+    [JsonPropertyName("is_multi_subtitle")]
+    public bool IsMultiSubtitle { get; set; }
+    
+    [JsonPropertyName("is_complete")]
+    public bool IsComplete { get; set; }
+    
+    [JsonPropertyName("year")]
+    public int Year { get; set; }
+    
+    [JsonPropertyName("resolution")]
+    public List<string>? Resolution { get; set; }
+    
+    [JsonPropertyName("quality")]
+    public List<string>? Quality { get; set; }
+    
+    [JsonPropertyName("season")]
+    public List<int>? Season { get; set; }
+    
+    [JsonPropertyName("episode")]
+    public List<int>? Episode { get; set; }
+    
+    [JsonPropertyName("codec")]
+    public List<string>? Codec { get; set; }
+    
+    [JsonPropertyName("audio")]
+    public List<string>? Audio { get; set; }
+    
+    [JsonPropertyName("subtitles")]
+    public List<string>? Subtitles { get; set; }
+    
+    [JsonPropertyName("language")]
+    public List<string>? Language { get; set; }
+    
+    [JsonPropertyName("bit_depth")]
+    public List<int>? BitDepth { get; set; }
+    
+    [JsonPropertyName("hdr")]
+    public string? Hdr { get; set; }
+    
+    [JsonPropertyName("proper")]
+    public bool Proper { get; set; }
+    
+    [JsonPropertyName("repack")]
+    public bool Repack { get; set; }
+    
+    [JsonPropertyName("remux")]
+    public bool Remux { get; set; }
+    
+    [JsonPropertyName("upscaled")]
+    public bool Upscaled { get; set; }
+    
+    [JsonPropertyName("remastered")]
+    public bool Remastered { get; set; }
+    
+    [JsonPropertyName("directors_cut")]
+    public bool DirectorsCut { get; set; }
+    
+    [JsonPropertyName("extended")]
+    public bool Extended { get; set; }
+    
+    // [JsonPropertyName("is_show")]
+    // public bool IsTvShow { get; set; }
+    //
+    // [JsonPropertyName("is_movie")]
+    // public bool IsMovie { get; set; }
+    
+    public string ToJson() => this.AsJson();
+    
+    public bool IsMovie => !TvRegexes.Any(regex => regex.IsMatch(RawTitle)) && Season?.Count == 0 && Episode?.Count == 0;
+    
+    private static List<Regex> TvRegexes { get; set; } =
+    [
+        new(@"[se]\d\d", RegexOptions.IgnoreCase),
+        new(@"\b(tv|complete)\b", RegexOptions.IgnoreCase),
+        new(@"\b(saisons?|stages?|seasons?).?\d", RegexOptions.IgnoreCase),
+        new(@"[a-z]\s?\-\s?\d{2,4}\b", RegexOptions.IgnoreCase),
+        new(@"\d{2,4}\s?\-\s?\d{2,4}\b", RegexOptions.IgnoreCase),
+    ];
+}

src/shared/Python/ServiceCollectionExtensions.cs

@@ -0,0 +1,12 @@
+namespace SharedContracts.Python;
+
+public static class ServiceCollectionExtensions
+{
+    public static IServiceCollection RegisterPythonEngine(this IServiceCollection services)
+    {
+        services.AddSingleton<IPythonEngineService, PythonEngineService>();
+        services.AddHostedService<PythonEngineManager>();
+
+        return services;
+    }
+}

src/shared/SharedContracts.csproj

@@ -16,7 +16,7 @@
   <PackageReference Include="MassTransit.Abstractions" Version="8.2.0" />
   <PackageReference Include="MassTransit.RabbitMQ" Version="8.2.0" />
   <PackageReference Include="Npgsql" Version="8.0.2" />
-  <PackageReference Include="PromKnight.ParseTorrentTitle" Version="1.0.4" />
+  <PackageReference Include="pythonnet" Version="3.0.3" />
   <PackageReference Include="Serilog" Version="3.1.1" />
   <PackageReference Include="Serilog.Extensions.Hosting" Version="8.0.0" />
   <PackageReference Include="Serilog.Settings.Configuration" Version="8.0.0" />

src/torrent-consumer/Features/TorrentIngestion/PerformIngestionConsumer.cs

@@ -11,6 +11,7 @@ public class PerformIngestionConsumer(IDataStorage dataStorage, ILogger<PerformI
         var torrent = new Torrent
         {
             InfoHash = request.IngestedTorrent.InfoHash.ToLowerInvariant(),
+            IngestedTorrentId = request.IngestedTorrent.Id,
             Provider = request.IngestedTorrent.Source,
             Title = request.IngestedTorrent.Name,
             Type = request.IngestedTorrent.Category,

src/torrent-ingestor/.dockerignore

@@ -0,0 +1 @@
+Dockerfile

src/torrent-ingestor/.envrc

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+layout python
+
+if ! has poetry; then
+    pip install poetry
+fi

src/torrent-ingestor/.python-version

@@ -0,0 +1 @@
+3.11

src/torrent-ingestor/Dockerfile

@@ -0,0 +1,17 @@
+FROM docker.io/python:3.11-slim
+
+WORKDIR /app
+
+RUN pip install --no-cache-dir poetry
+RUN poetry config virtualenvs.create false
+
+COPY . /app
+RUN poetry install --no-dev
+
+RUN groupadd ingestor && useradd -g ingestor ingestor
+USER ingestor
+
+HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 \
+  CMD pgrep -f python || exit 1
+  
+CMD ["python", "/app/torrent_ingestor/main.py"]

src/torrent-ingestor/README.md

@@ -0,0 +1,15 @@
+## Torrent Processor
+
+This project subscribes to the Annatar Redis pubsub event `events:v1:torrent:added` and writes the results to `ingested_torrents` table.
+
+Why? The Annatar event occurs when Annatar identifies a torrent from Jackett. This adds another source of torrents to the KC backed.
+
+## Run
+
+```bash
+POSTGRES_URL=postgresql://USERNAME:PASSWORD@127.0.0.1/knightcrawler \
+	REDIS_URL=redis://localhost \
+	python torrent_ingestor/main.py
+```
+
+You can run multiple instances as it uses a SETNX to acquire a lock for each info hash.

src/torrent-ingestor/poetry.lock

@@ -0,0 +1,579 @@
+# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand.
+
+[[package]]
+name = "annotated-types"
+version = "0.6.0"
+description = "Reusable constraint types to use with typing.Annotated"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "annotated_types-0.6.0-py3-none-any.whl", hash = "sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43"},
+    {file = "annotated_types-0.6.0.tar.gz", hash = "sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d"},
+]
+
+[[package]]
+name = "async-timeout"
+version = "4.0.3"
+description = "Timeout context manager for asyncio programs"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"},
+    {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"},
+]
+
+[[package]]
+name = "asyncpg"
+version = "0.29.0"
+description = "An asyncio PostgreSQL driver"
+optional = false
+python-versions = ">=3.8.0"
+files = [
+    {file = "asyncpg-0.29.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72fd0ef9f00aeed37179c62282a3d14262dbbafb74ec0ba16e1b1864d8a12169"},
+    {file = "asyncpg-0.29.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:52e8f8f9ff6e21f9b39ca9f8e3e33a5fcdceaf5667a8c5c32bee158e313be385"},
+    {file = "asyncpg-0.29.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a9e6823a7012be8b68301342ba33b4740e5a166f6bbda0aee32bc01638491a22"},
+    {file = "asyncpg-0.29.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:746e80d83ad5d5464cfbf94315eb6744222ab00aa4e522b704322fb182b83610"},
+    {file = "asyncpg-0.29.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:ff8e8109cd6a46ff852a5e6bab8b0a047d7ea42fcb7ca5ae6eaae97d8eacf397"},
+    {file = "asyncpg-0.29.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:97eb024685b1d7e72b1972863de527c11ff87960837919dac6e34754768098eb"},
+    {file = "asyncpg-0.29.0-cp310-cp310-win32.whl", hash = "sha256:5bbb7f2cafd8d1fa3e65431833de2642f4b2124be61a449fa064e1a08d27e449"},
+    {file = "asyncpg-0.29.0-cp310-cp310-win_amd64.whl", hash = "sha256:76c3ac6530904838a4b650b2880f8e7af938ee049e769ec2fba7cd66469d7772"},
+    {file = "asyncpg-0.29.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4900ee08e85af01adb207519bb4e14b1cae8fd21e0ccf80fac6aa60b6da37b4"},
+    {file = "asyncpg-0.29.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a65c1dcd820d5aea7c7d82a3fdcb70e096f8f70d1a8bf93eb458e49bfad036ac"},
+    {file = "asyncpg-0.29.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5b52e46f165585fd6af4863f268566668407c76b2c72d366bb8b522fa66f1870"},
+    {file = "asyncpg-0.29.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc600ee8ef3dd38b8d67421359779f8ccec30b463e7aec7ed481c8346decf99f"},
+    {file = "asyncpg-0.29.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:039a261af4f38f949095e1e780bae84a25ffe3e370175193174eb08d3cecab23"},
+    {file = "asyncpg-0.29.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6feaf2d8f9138d190e5ec4390c1715c3e87b37715cd69b2c3dfca616134efd2b"},
+    {file = "asyncpg-0.29.0-cp311-cp311-win32.whl", hash = "sha256:1e186427c88225ef730555f5fdda6c1812daa884064bfe6bc462fd3a71c4b675"},
+    {file = "asyncpg-0.29.0-cp311-cp311-win_amd64.whl", hash = "sha256:cfe73ffae35f518cfd6e4e5f5abb2618ceb5ef02a2365ce64f132601000587d3"},
+    {file = "asyncpg-0.29.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:6011b0dc29886ab424dc042bf9eeb507670a3b40aece3439944006aafe023178"},
+    {file = "asyncpg-0.29.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b544ffc66b039d5ec5a7454667f855f7fec08e0dfaf5a5490dfafbb7abbd2cfb"},
+    {file = "asyncpg-0.29.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d84156d5fb530b06c493f9e7635aa18f518fa1d1395ef240d211cb563c4e2364"},
+    {file = "asyncpg-0.29.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:54858bc25b49d1114178d65a88e48ad50cb2b6f3e475caa0f0c092d5f527c106"},
+    {file = "asyncpg-0.29.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:bde17a1861cf10d5afce80a36fca736a86769ab3579532c03e45f83ba8a09c59"},
+    {file = "asyncpg-0.29.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:37a2ec1b9ff88d8773d3eb6d3784dc7e3fee7756a5317b67f923172a4748a175"},
+    {file = "asyncpg-0.29.0-cp312-cp312-win32.whl", hash = "sha256:bb1292d9fad43112a85e98ecdc2e051602bce97c199920586be83254d9dafc02"},
+    {file = "asyncpg-0.29.0-cp312-cp312-win_amd64.whl", hash = "sha256:2245be8ec5047a605e0b454c894e54bf2ec787ac04b1cb7e0d3c67aa1e32f0fe"},
+    {file = "asyncpg-0.29.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0009a300cae37b8c525e5b449233d59cd9868fd35431abc470a3e364d2b85cb9"},
+    {file = "asyncpg-0.29.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5cad1324dbb33f3ca0cd2074d5114354ed3be2b94d48ddfd88af75ebda7c43cc"},
+    {file = "asyncpg-0.29.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:012d01df61e009015944ac7543d6ee30c2dc1eb2f6b10b62a3f598beb6531548"},
+    {file = "asyncpg-0.29.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:000c996c53c04770798053e1730d34e30cb645ad95a63265aec82da9093d88e7"},
+    {file = "asyncpg-0.29.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:e0bfe9c4d3429706cf70d3249089de14d6a01192d617e9093a8e941fea8ee775"},
+    {file = "asyncpg-0.29.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:642a36eb41b6313ffa328e8a5c5c2b5bea6ee138546c9c3cf1bffaad8ee36dd9"},
+    {file = "asyncpg-0.29.0-cp38-cp38-win32.whl", hash = "sha256:a921372bbd0aa3a5822dd0409da61b4cd50df89ae85150149f8c119f23e8c408"},
+    {file = "asyncpg-0.29.0-cp38-cp38-win_amd64.whl", hash = "sha256:103aad2b92d1506700cbf51cd8bb5441e7e72e87a7b3a2ca4e32c840f051a6a3"},
+    {file = "asyncpg-0.29.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5340dd515d7e52f4c11ada32171d87c05570479dc01dc66d03ee3e150fb695da"},
+    {file = "asyncpg-0.29.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e17b52c6cf83e170d3d865571ba574577ab8e533e7361a2b8ce6157d02c665d3"},
+    {file = "asyncpg-0.29.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f100d23f273555f4b19b74a96840aa27b85e99ba4b1f18d4ebff0734e78dc090"},
+    {file = "asyncpg-0.29.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:48e7c58b516057126b363cec8ca02b804644fd012ef8e6c7e23386b7d5e6ce83"},
+    {file = "asyncpg-0.29.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f9ea3f24eb4c49a615573724d88a48bd1b7821c890c2effe04f05382ed9e8810"},
+    {file = "asyncpg-0.29.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8d36c7f14a22ec9e928f15f92a48207546ffe68bc412f3be718eedccdf10dc5c"},
+    {file = "asyncpg-0.29.0-cp39-cp39-win32.whl", hash = "sha256:797ab8123ebaed304a1fad4d7576d5376c3a006a4100380fb9d517f0b59c1ab2"},
+    {file = "asyncpg-0.29.0-cp39-cp39-win_amd64.whl", hash = "sha256:cce08a178858b426ae1aa8409b5cc171def45d4293626e7aa6510696d46decd8"},
+    {file = "asyncpg-0.29.0.tar.gz", hash = "sha256:d1c49e1f44fffafd9a55e1a9b101590859d881d639ea2922516f5d9c512d354e"},
+]
+
+[package.dependencies]
+async-timeout = {version = ">=4.0.3", markers = "python_version < \"3.12.0\""}
+
+[package.extras]
+docs = ["Sphinx (>=5.3.0,<5.4.0)", "sphinx-rtd-theme (>=1.2.2)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)"]
+test = ["flake8 (>=6.1,<7.0)", "uvloop (>=0.15.3)"]
+
+[[package]]
+name = "databases"
+version = "0.9.0"
+description = "Async database support for Python."
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "databases-0.9.0-py3-none-any.whl", hash = "sha256:9ee657c9863b34f8d3a06c06eafbe1bda68af2a434b56996312edf1f1c0b6297"},
+    {file = "databases-0.9.0.tar.gz", hash = "sha256:d2f259677609bf187737644c95fa41701072e995dfeb8d2882f335795c5b61b0"},
+]
+
+[package.dependencies]
+asyncpg = {version = "*", optional = true, markers = "extra == \"postgresql\""}
+sqlalchemy = ">=2.0.7"
+
+[package.extras]
+aiomysql = ["aiomysql"]
+aiopg = ["aiopg"]
+aiosqlite = ["aiosqlite"]
+asyncmy = ["asyncmy"]
+asyncpg = ["asyncpg"]
+mysql = ["aiomysql"]
+postgresql = ["asyncpg"]
+sqlite = ["aiosqlite"]
+
+[[package]]
+name = "greenlet"
+version = "3.0.3"
+description = "Lightweight in-process concurrent programming"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "greenlet-3.0.3-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:9da2bd29ed9e4f15955dd1595ad7bc9320308a3b766ef7f837e23ad4b4aac31a"},
+    {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d353cadd6083fdb056bb46ed07e4340b0869c305c8ca54ef9da3421acbdf6881"},
+    {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dca1e2f3ca00b84a396bc1bce13dd21f680f035314d2379c4160c98153b2059b"},
+    {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3ed7fb269f15dc662787f4119ec300ad0702fa1b19d2135a37c2c4de6fadfd4a"},
+    {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd4f49ae60e10adbc94b45c0b5e6a179acc1736cf7a90160b404076ee283cf83"},
+    {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:73a411ef564e0e097dbe7e866bb2dda0f027e072b04da387282b02c308807405"},
+    {file = "greenlet-3.0.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:7f362975f2d179f9e26928c5b517524e89dd48530a0202570d55ad6ca5d8a56f"},
+    {file = "greenlet-3.0.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:649dde7de1a5eceb258f9cb00bdf50e978c9db1b996964cd80703614c86495eb"},
+    {file = "greenlet-3.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:68834da854554926fbedd38c76e60c4a2e3198c6fbed520b106a8986445caaf9"},
+    {file = "greenlet-3.0.3-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:b1b5667cced97081bf57b8fa1d6bfca67814b0afd38208d52538316e9422fc61"},
+    {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:52f59dd9c96ad2fc0d5724107444f76eb20aaccb675bf825df6435acb7703559"},
+    {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:afaff6cf5200befd5cec055b07d1c0a5a06c040fe5ad148abcd11ba6ab9b114e"},
+    {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fe754d231288e1e64323cfad462fcee8f0288654c10bdf4f603a39ed923bef33"},
+    {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2797aa5aedac23af156bbb5a6aa2cd3427ada2972c828244eb7d1b9255846379"},
+    {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b7f009caad047246ed379e1c4dbcb8b020f0a390667ea74d2387be2998f58a22"},
+    {file = "greenlet-3.0.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c5e1536de2aad7bf62e27baf79225d0d64360d4168cf2e6becb91baf1ed074f3"},
+    {file = "greenlet-3.0.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:894393ce10ceac937e56ec00bb71c4c2f8209ad516e96033e4b3b1de270e200d"},
+    {file = "greenlet-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:1ea188d4f49089fc6fb283845ab18a2518d279c7cd9da1065d7a84e991748728"},
+    {file = "greenlet-3.0.3-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:70fb482fdf2c707765ab5f0b6655e9cfcf3780d8d87355a063547b41177599be"},
+    {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4d1ac74f5c0c0524e4a24335350edad7e5f03b9532da7ea4d3c54d527784f2e"},
+    {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:149e94a2dd82d19838fe4b2259f1b6b9957d5ba1b25640d2380bea9c5df37676"},
+    {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:15d79dd26056573940fcb8c7413d84118086f2ec1a8acdfa854631084393efcc"},
+    {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:881b7db1ebff4ba09aaaeae6aa491daeb226c8150fc20e836ad00041bcb11230"},
+    {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fcd2469d6a2cf298f198f0487e0a5b1a47a42ca0fa4dfd1b6862c999f018ebbf"},
+    {file = "greenlet-3.0.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:1f672519db1796ca0d8753f9e78ec02355e862d0998193038c7073045899f305"},
+    {file = "greenlet-3.0.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2516a9957eed41dd8f1ec0c604f1cdc86758b587d964668b5b196a9db5bfcde6"},
+    {file = "greenlet-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:bba5387a6975598857d86de9eac14210a49d554a77eb8261cc68b7d082f78ce2"},
+    {file = "greenlet-3.0.3-cp37-cp37m-macosx_11_0_universal2.whl", hash = "sha256:5b51e85cb5ceda94e79d019ed36b35386e8c37d22f07d6a751cb659b180d5274"},
+    {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:daf3cb43b7cf2ba96d614252ce1684c1bccee6b2183a01328c98d36fcd7d5cb0"},
+    {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99bf650dc5d69546e076f413a87481ee1d2d09aaaaaca058c9251b6d8c14783f"},
+    {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2dd6e660effd852586b6a8478a1d244b8dc90ab5b1321751d2ea15deb49ed414"},
+    {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e3391d1e16e2a5a1507d83e4a8b100f4ee626e8eca43cf2cadb543de69827c4c"},
+    {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e1f145462f1fa6e4a4ae3c0f782e580ce44d57c8f2c7aae1b6fa88c0b2efdb41"},
+    {file = "greenlet-3.0.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1a7191e42732df52cb5f39d3527217e7ab73cae2cb3694d241e18f53d84ea9a7"},
+    {file = "greenlet-3.0.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0448abc479fab28b00cb472d278828b3ccca164531daab4e970a0458786055d6"},
+    {file = "greenlet-3.0.3-cp37-cp37m-win32.whl", hash = "sha256:b542be2440edc2d48547b5923c408cbe0fc94afb9f18741faa6ae970dbcb9b6d"},
+    {file = "greenlet-3.0.3-cp37-cp37m-win_amd64.whl", hash = "sha256:01bc7ea167cf943b4c802068e178bbf70ae2e8c080467070d01bfa02f337ee67"},
+    {file = "greenlet-3.0.3-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:1996cb9306c8595335bb157d133daf5cf9f693ef413e7673cb07e3e5871379ca"},
+    {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ddc0f794e6ad661e321caa8d2f0a55ce01213c74722587256fb6566049a8b04"},
+    {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c9db1c18f0eaad2f804728c67d6c610778456e3e1cc4ab4bbd5eeb8e6053c6fc"},
+    {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7170375bcc99f1a2fbd9c306f5be8764eaf3ac6b5cb968862cad4c7057756506"},
+    {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b66c9c1e7ccabad3a7d037b2bcb740122a7b17a53734b7d72a344ce39882a1b"},
+    {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:098d86f528c855ead3479afe84b49242e174ed262456c342d70fc7f972bc13c4"},
+    {file = "greenlet-3.0.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:81bb9c6d52e8321f09c3d165b2a78c680506d9af285bfccbad9fb7ad5a5da3e5"},
+    {file = "greenlet-3.0.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fd096eb7ffef17c456cfa587523c5f92321ae02427ff955bebe9e3c63bc9f0da"},
+    {file = "greenlet-3.0.3-cp38-cp38-win32.whl", hash = "sha256:d46677c85c5ba00a9cb6f7a00b2bfa6f812192d2c9f7d9c4f6a55b60216712f3"},
+    {file = "greenlet-3.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:419b386f84949bf0e7c73e6032e3457b82a787c1ab4a0e43732898a761cc9dbf"},
+    {file = "greenlet-3.0.3-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:da70d4d51c8b306bb7a031d5cff6cc25ad253affe89b70352af5f1cb68e74b53"},
+    {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:086152f8fbc5955df88382e8a75984e2bb1c892ad2e3c80a2508954e52295257"},
+    {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d73a9fe764d77f87f8ec26a0c85144d6a951a6c438dfe50487df5595c6373eac"},
+    {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7dcbe92cc99f08c8dd11f930de4d99ef756c3591a5377d1d9cd7dd5e896da71"},
+    {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1551a8195c0d4a68fac7a4325efac0d541b48def35feb49d803674ac32582f61"},
+    {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64d7675ad83578e3fc149b617a444fab8efdafc9385471f868eb5ff83e446b8b"},
+    {file = "greenlet-3.0.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b37eef18ea55f2ffd8f00ff8fe7c8d3818abd3e25fb73fae2ca3b672e333a7a6"},
+    {file = "greenlet-3.0.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:77457465d89b8263bca14759d7c1684df840b6811b2499838cc5b040a8b5b113"},
+    {file = "greenlet-3.0.3-cp39-cp39-win32.whl", hash = "sha256:57e8974f23e47dac22b83436bdcf23080ade568ce77df33159e019d161ce1d1e"},
+    {file = "greenlet-3.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:c5ee858cfe08f34712f548c3c363e807e7186f03ad7a5039ebadb29e8c6be067"},
+    {file = "greenlet-3.0.3.tar.gz", hash = "sha256:43374442353259554ce33599da8b692d5aa96f8976d567d4badf263371fbe491"},
+]
+
+[package.extras]
+docs = ["Sphinx", "furo"]
+test = ["objgraph", "psutil"]
+
+[[package]]
+name = "nodeenv"
+version = "1.8.0"
+description = "Node.js virtual environment builder"
+optional = false
+python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*"
+files = [
+    {file = "nodeenv-1.8.0-py2.py3-none-any.whl", hash = "sha256:df865724bb3c3adc86b3876fa209771517b0cfe596beff01a92700e0e8be4cec"},
+    {file = "nodeenv-1.8.0.tar.gz", hash = "sha256:d51e0c37e64fbf47d017feac3145cdbb58836d7eee8c6f6d3b6880c5456227d2"},
+]
+
+[package.dependencies]
+setuptools = "*"
+
+[[package]]
+name = "psycopg2-binary"
+version = "2.9.9"
+description = "psycopg2 - Python-PostgreSQL Database Adapter"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "psycopg2-binary-2.9.9.tar.gz", hash = "sha256:7f01846810177d829c7692f1f5ada8096762d9172af1b1a28d4ab5b77c923c1c"},
+    {file = "psycopg2_binary-2.9.9-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c2470da5418b76232f02a2fcd2229537bb2d5a7096674ce61859c3229f2eb202"},
+    {file = "psycopg2_binary-2.9.9-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c6af2a6d4b7ee9615cbb162b0738f6e1fd1f5c3eda7e5da17861eacf4c717ea7"},
+    {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:75723c3c0fbbf34350b46a3199eb50638ab22a0228f93fb472ef4d9becc2382b"},
+    {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:83791a65b51ad6ee6cf0845634859d69a038ea9b03d7b26e703f94c7e93dbcf9"},
+    {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0ef4854e82c09e84cc63084a9e4ccd6d9b154f1dbdd283efb92ecd0b5e2b8c84"},
+    {file = "psycopg2_binary-2.9.9-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed1184ab8f113e8d660ce49a56390ca181f2981066acc27cf637d5c1e10ce46e"},
+    {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d2997c458c690ec2bc6b0b7ecbafd02b029b7b4283078d3b32a852a7ce3ddd98"},
+    {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:b58b4710c7f4161b5e9dcbe73bb7c62d65670a87df7bcce9e1faaad43e715245"},
+    {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:0c009475ee389757e6e34611d75f6e4f05f0cf5ebb76c6037508318e1a1e0d7e"},
+    {file = "psycopg2_binary-2.9.9-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8dbf6d1bc73f1d04ec1734bae3b4fb0ee3cb2a493d35ede9badbeb901fb40f6f"},
+    {file = "psycopg2_binary-2.9.9-cp310-cp310-win32.whl", hash = "sha256:3f78fd71c4f43a13d342be74ebbc0666fe1f555b8837eb113cb7416856c79682"},
+    {file = "psycopg2_binary-2.9.9-cp310-cp310-win_amd64.whl", hash = "sha256:876801744b0dee379e4e3c38b76fc89f88834bb15bf92ee07d94acd06ec890a0"},
+    {file = "psycopg2_binary-2.9.9-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ee825e70b1a209475622f7f7b776785bd68f34af6e7a46e2e42f27b659b5bc26"},
+    {file = "psycopg2_binary-2.9.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1ea665f8ce695bcc37a90ee52de7a7980be5161375d42a0b6c6abedbf0d81f0f"},
+    {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:143072318f793f53819048fdfe30c321890af0c3ec7cb1dfc9cc87aa88241de2"},
+    {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c332c8d69fb64979ebf76613c66b985414927a40f8defa16cf1bc028b7b0a7b0"},
+    {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f7fc5a5acafb7d6ccca13bfa8c90f8c51f13d8fb87d95656d3950f0158d3ce53"},
+    {file = "psycopg2_binary-2.9.9-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:977646e05232579d2e7b9c59e21dbe5261f403a88417f6a6512e70d3f8a046be"},
+    {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b6356793b84728d9d50ead16ab43c187673831e9d4019013f1402c41b1db9b27"},
+    {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:bc7bb56d04601d443f24094e9e31ae6deec9ccb23581f75343feebaf30423359"},
+    {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:77853062a2c45be16fd6b8d6de2a99278ee1d985a7bd8b103e97e41c034006d2"},
+    {file = "psycopg2_binary-2.9.9-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:78151aa3ec21dccd5cdef6c74c3e73386dcdfaf19bced944169697d7ac7482fc"},
+    {file = "psycopg2_binary-2.9.9-cp311-cp311-win32.whl", hash = "sha256:dc4926288b2a3e9fd7b50dc6a1909a13bbdadfc67d93f3374d984e56f885579d"},
+    {file = "psycopg2_binary-2.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:b76bedd166805480ab069612119ea636f5ab8f8771e640ae103e05a4aae3e417"},
+    {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:8532fd6e6e2dc57bcb3bc90b079c60de896d2128c5d9d6f24a63875a95a088cf"},
+    {file = "psycopg2_binary-2.9.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0605eaed3eb239e87df0d5e3c6489daae3f7388d455d0c0b4df899519c6a38d"},
+    {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f8544b092a29a6ddd72f3556a9fcf249ec412e10ad28be6a0c0d948924f2212"},
+    {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2d423c8d8a3c82d08fe8af900ad5b613ce3632a1249fd6a223941d0735fce493"},
+    {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2e5afae772c00980525f6d6ecf7cbca55676296b580c0e6abb407f15f3706996"},
+    {file = "psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e6f98446430fdf41bd36d4faa6cb409f5140c1c2cf58ce0bbdaf16af7d3f119"},
+    {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c77e3d1862452565875eb31bdb45ac62502feabbd53429fdc39a1cc341d681ba"},
+    {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:cb16c65dcb648d0a43a2521f2f0a2300f40639f6f8c1ecbc662141e4e3e1ee07"},
+    {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:911dda9c487075abd54e644ccdf5e5c16773470a6a5d3826fda76699410066fb"},
+    {file = "psycopg2_binary-2.9.9-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:57fede879f08d23c85140a360c6a77709113efd1c993923c59fde17aa27599fe"},
+    {file = "psycopg2_binary-2.9.9-cp312-cp312-win32.whl", hash = "sha256:64cf30263844fa208851ebb13b0732ce674d8ec6a0c86a4e160495d299ba3c93"},
+    {file = "psycopg2_binary-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:81ff62668af011f9a48787564ab7eded4e9fb17a4a6a74af5ffa6a457400d2ab"},
+    {file = "psycopg2_binary-2.9.9-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2293b001e319ab0d869d660a704942c9e2cce19745262a8aba2115ef41a0a42a"},
+    {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:03ef7df18daf2c4c07e2695e8cfd5ee7f748a1d54d802330985a78d2a5a6dca9"},
+    {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a602ea5aff39bb9fac6308e9c9d82b9a35c2bf288e184a816002c9fae930b77"},
+    {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8359bf4791968c5a78c56103702000105501adb557f3cf772b2c207284273984"},
+    {file = "psycopg2_binary-2.9.9-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:275ff571376626195ab95a746e6a04c7df8ea34638b99fc11160de91f2fef503"},
+    {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:f9b5571d33660d5009a8b3c25dc1db560206e2d2f89d3df1cb32d72c0d117d52"},
+    {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:420f9bbf47a02616e8554e825208cb947969451978dceb77f95ad09c37791dae"},
+    {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:4154ad09dac630a0f13f37b583eae260c6aa885d67dfbccb5b02c33f31a6d420"},
+    {file = "psycopg2_binary-2.9.9-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a148c5d507bb9b4f2030a2025c545fccb0e1ef317393eaba42e7eabd28eb6041"},
+    {file = "psycopg2_binary-2.9.9-cp37-cp37m-win32.whl", hash = "sha256:68fc1f1ba168724771e38bee37d940d2865cb0f562380a1fb1ffb428b75cb692"},
+    {file = "psycopg2_binary-2.9.9-cp37-cp37m-win_amd64.whl", hash = "sha256:281309265596e388ef483250db3640e5f414168c5a67e9c665cafce9492eda2f"},
+    {file = "psycopg2_binary-2.9.9-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:60989127da422b74a04345096c10d416c2b41bd7bf2a380eb541059e4e999980"},
+    {file = "psycopg2_binary-2.9.9-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:246b123cc54bb5361588acc54218c8c9fb73068bf227a4a531d8ed56fa3ca7d6"},
+    {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34eccd14566f8fe14b2b95bb13b11572f7c7d5c36da61caf414d23b91fcc5d94"},
+    {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:18d0ef97766055fec15b5de2c06dd8e7654705ce3e5e5eed3b6651a1d2a9a152"},
+    {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d3f82c171b4ccd83bbaf35aa05e44e690113bd4f3b7b6cc54d2219b132f3ae55"},
+    {file = "psycopg2_binary-2.9.9-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ead20f7913a9c1e894aebe47cccf9dc834e1618b7aa96155d2091a626e59c972"},
+    {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:ca49a8119c6cbd77375ae303b0cfd8c11f011abbbd64601167ecca18a87e7cdd"},
+    {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:323ba25b92454adb36fa425dc5cf6f8f19f78948cbad2e7bc6cdf7b0d7982e59"},
+    {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:1236ed0952fbd919c100bc839eaa4a39ebc397ed1c08a97fc45fee2a595aa1b3"},
+    {file = "psycopg2_binary-2.9.9-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:729177eaf0aefca0994ce4cffe96ad3c75e377c7b6f4efa59ebf003b6d398716"},
+    {file = "psycopg2_binary-2.9.9-cp38-cp38-win32.whl", hash = "sha256:804d99b24ad523a1fe18cc707bf741670332f7c7412e9d49cb5eab67e886b9b5"},
+    {file = "psycopg2_binary-2.9.9-cp38-cp38-win_amd64.whl", hash = "sha256:a6cdcc3ede532f4a4b96000b6362099591ab4a3e913d70bcbac2b56c872446f7"},
+    {file = "psycopg2_binary-2.9.9-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:72dffbd8b4194858d0941062a9766f8297e8868e1dd07a7b36212aaa90f49472"},
+    {file = "psycopg2_binary-2.9.9-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:30dcc86377618a4c8f3b72418df92e77be4254d8f89f14b8e8f57d6d43603c0f"},
+    {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:31a34c508c003a4347d389a9e6fcc2307cc2150eb516462a7a17512130de109e"},
+    {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:15208be1c50b99203fe88d15695f22a5bed95ab3f84354c494bcb1d08557df67"},
+    {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1873aade94b74715be2246321c8650cabf5a0d098a95bab81145ffffa4c13876"},
+    {file = "psycopg2_binary-2.9.9-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a58c98a7e9c021f357348867f537017057c2ed7f77337fd914d0bedb35dace7"},
+    {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:4686818798f9194d03c9129a4d9a702d9e113a89cb03bffe08c6cf799e053291"},
+    {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:ebdc36bea43063116f0486869652cb2ed7032dbc59fbcb4445c4862b5c1ecf7f"},
+    {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:ca08decd2697fdea0aea364b370b1249d47336aec935f87b8bbfd7da5b2ee9c1"},
+    {file = "psycopg2_binary-2.9.9-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ac05fb791acf5e1a3e39402641827780fe44d27e72567a000412c648a85ba860"},
+    {file = "psycopg2_binary-2.9.9-cp39-cp39-win32.whl", hash = "sha256:9dba73be7305b399924709b91682299794887cbbd88e38226ed9f6712eabee90"},
+    {file = "psycopg2_binary-2.9.9-cp39-cp39-win_amd64.whl", hash = "sha256:f7ae5d65ccfbebdfa761585228eb4d0df3a8b15cfb53bd953e713e09fbb12957"},
+]
+
+[[package]]
+name = "pydantic"
+version = "2.6.4"
+description = "Data validation using Python type hints"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "pydantic-2.6.4-py3-none-any.whl", hash = "sha256:cc46fce86607580867bdc3361ad462bab9c222ef042d3da86f2fb333e1d916c5"},
+    {file = "pydantic-2.6.4.tar.gz", hash = "sha256:b1704e0847db01817624a6b86766967f552dd9dbf3afba4004409f908dcc84e6"},
+]
+
+[package.dependencies]
+annotated-types = ">=0.4.0"
+pydantic-core = "2.16.3"
+typing-extensions = ">=4.6.1"
+
+[package.extras]
+email = ["email-validator (>=2.0.0)"]
+
+[[package]]
+name = "pydantic-core"
+version = "2.16.3"
+description = ""
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "pydantic_core-2.16.3-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:75b81e678d1c1ede0785c7f46690621e4c6e63ccd9192af1f0bd9d504bbb6bf4"},
+    {file = "pydantic_core-2.16.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9c865a7ee6f93783bd5d781af5a4c43dadc37053a5b42f7d18dc019f8c9d2bd1"},
+    {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:162e498303d2b1c036b957a1278fa0899d02b2842f1ff901b6395104c5554a45"},
+    {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2f583bd01bbfbff4eaee0868e6fc607efdfcc2b03c1c766b06a707abbc856187"},
+    {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b926dd38db1519ed3043a4de50214e0d600d404099c3392f098a7f9d75029ff8"},
+    {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:716b542728d4c742353448765aa7cdaa519a7b82f9564130e2b3f6766018c9ec"},
+    {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc4ad7f7ee1a13d9cb49d8198cd7d7e3aa93e425f371a68235f784e99741561f"},
+    {file = "pydantic_core-2.16.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bd87f48924f360e5d1c5f770d6155ce0e7d83f7b4e10c2f9ec001c73cf475c99"},
+    {file = "pydantic_core-2.16.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0df446663464884297c793874573549229f9eca73b59360878f382a0fc085979"},
+    {file = "pydantic_core-2.16.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4df8a199d9f6afc5ae9a65f8f95ee52cae389a8c6b20163762bde0426275b7db"},
+    {file = "pydantic_core-2.16.3-cp310-none-win32.whl", hash = "sha256:456855f57b413f077dff513a5a28ed838dbbb15082ba00f80750377eed23d132"},
+    {file = "pydantic_core-2.16.3-cp310-none-win_amd64.whl", hash = "sha256:732da3243e1b8d3eab8c6ae23ae6a58548849d2e4a4e03a1924c8ddf71a387cb"},
+    {file = "pydantic_core-2.16.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:519ae0312616026bf4cedc0fe459e982734f3ca82ee8c7246c19b650b60a5ee4"},
+    {file = "pydantic_core-2.16.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b3992a322a5617ded0a9f23fd06dbc1e4bd7cf39bc4ccf344b10f80af58beacd"},
+    {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d62da299c6ecb04df729e4b5c52dc0d53f4f8430b4492b93aa8de1f541c4aac"},
+    {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2acca2be4bb2f2147ada8cac612f8a98fc09f41c89f87add7256ad27332c2fda"},
+    {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1b662180108c55dfbf1280d865b2d116633d436cfc0bba82323554873967b340"},
+    {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e7c6ed0dc9d8e65f24f5824291550139fe6f37fac03788d4580da0d33bc00c97"},
+    {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a6b1bb0827f56654b4437955555dc3aeeebeddc47c2d7ed575477f082622c49e"},
+    {file = "pydantic_core-2.16.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e56f8186d6210ac7ece503193ec84104da7ceb98f68ce18c07282fcc2452e76f"},
+    {file = "pydantic_core-2.16.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:936e5db01dd49476fa8f4383c259b8b1303d5dd5fb34c97de194560698cc2c5e"},
+    {file = "pydantic_core-2.16.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:33809aebac276089b78db106ee692bdc9044710e26f24a9a2eaa35a0f9fa70ba"},
+    {file = "pydantic_core-2.16.3-cp311-none-win32.whl", hash = "sha256:ded1c35f15c9dea16ead9bffcde9bb5c7c031bff076355dc58dcb1cb436c4721"},
+    {file = "pydantic_core-2.16.3-cp311-none-win_amd64.whl", hash = "sha256:d89ca19cdd0dd5f31606a9329e309d4fcbb3df860960acec32630297d61820df"},
+    {file = "pydantic_core-2.16.3-cp311-none-win_arm64.whl", hash = "sha256:6162f8d2dc27ba21027f261e4fa26f8bcb3cf9784b7f9499466a311ac284b5b9"},
+    {file = "pydantic_core-2.16.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:0f56ae86b60ea987ae8bcd6654a887238fd53d1384f9b222ac457070b7ac4cff"},
+    {file = "pydantic_core-2.16.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c9bd22a2a639e26171068f8ebb5400ce2c1bc7d17959f60a3b753ae13c632975"},
+    {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4204e773b4b408062960e65468d5346bdfe139247ee5f1ca2a378983e11388a2"},
+    {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f651dd19363c632f4abe3480a7c87a9773be27cfe1341aef06e8759599454120"},
+    {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aaf09e615a0bf98d406657e0008e4a8701b11481840be7d31755dc9f97c44053"},
+    {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8e47755d8152c1ab5b55928ab422a76e2e7b22b5ed8e90a7d584268dd49e9c6b"},
+    {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:500960cb3a0543a724a81ba859da816e8cf01b0e6aaeedf2c3775d12ee49cade"},
+    {file = "pydantic_core-2.16.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:cf6204fe865da605285c34cf1172879d0314ff267b1c35ff59de7154f35fdc2e"},
+    {file = "pydantic_core-2.16.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d33dd21f572545649f90c38c227cc8631268ba25c460b5569abebdd0ec5974ca"},
+    {file = "pydantic_core-2.16.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:49d5d58abd4b83fb8ce763be7794d09b2f50f10aa65c0f0c1696c677edeb7cbf"},
+    {file = "pydantic_core-2.16.3-cp312-none-win32.whl", hash = "sha256:f53aace168a2a10582e570b7736cc5bef12cae9cf21775e3eafac597e8551fbe"},
+    {file = "pydantic_core-2.16.3-cp312-none-win_amd64.whl", hash = "sha256:0d32576b1de5a30d9a97f300cc6a3f4694c428d956adbc7e6e2f9cad279e45ed"},
+    {file = "pydantic_core-2.16.3-cp312-none-win_arm64.whl", hash = "sha256:ec08be75bb268473677edb83ba71e7e74b43c008e4a7b1907c6d57e940bf34b6"},
+    {file = "pydantic_core-2.16.3-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:b1f6f5938d63c6139860f044e2538baeee6f0b251a1816e7adb6cbce106a1f01"},
+    {file = "pydantic_core-2.16.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2a1ef6a36fdbf71538142ed604ad19b82f67b05749512e47f247a6ddd06afdc7"},
+    {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:704d35ecc7e9c31d48926150afada60401c55efa3b46cd1ded5a01bdffaf1d48"},
+    {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d937653a696465677ed583124b94a4b2d79f5e30b2c46115a68e482c6a591c8a"},
+    {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c9803edf8e29bd825f43481f19c37f50d2b01899448273b3a7758441b512acf8"},
+    {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:72282ad4892a9fb2da25defeac8c2e84352c108705c972db82ab121d15f14e6d"},
+    {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f752826b5b8361193df55afcdf8ca6a57d0232653494ba473630a83ba50d8c9"},
+    {file = "pydantic_core-2.16.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4384a8f68ddb31a0b0c3deae88765f5868a1b9148939c3f4121233314ad5532c"},
+    {file = "pydantic_core-2.16.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:a4b2bf78342c40b3dc830880106f54328928ff03e357935ad26c7128bbd66ce8"},
+    {file = "pydantic_core-2.16.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:13dcc4802961b5f843a9385fc821a0b0135e8c07fc3d9949fd49627c1a5e6ae5"},
+    {file = "pydantic_core-2.16.3-cp38-none-win32.whl", hash = "sha256:e3e70c94a0c3841e6aa831edab1619ad5c511199be94d0c11ba75fe06efe107a"},
+    {file = "pydantic_core-2.16.3-cp38-none-win_amd64.whl", hash = "sha256:ecdf6bf5f578615f2e985a5e1f6572e23aa632c4bd1dc67f8f406d445ac115ed"},
+    {file = "pydantic_core-2.16.3-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:bda1ee3e08252b8d41fa5537413ffdddd58fa73107171a126d3b9ff001b9b820"},
+    {file = "pydantic_core-2.16.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:21b888c973e4f26b7a96491c0965a8a312e13be108022ee510248fe379a5fa23"},
+    {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be0ec334369316fa73448cc8c982c01e5d2a81c95969d58b8f6e272884df0074"},
+    {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b5b6079cc452a7c53dd378c6f881ac528246b3ac9aae0f8eef98498a75657805"},
+    {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7ee8d5f878dccb6d499ba4d30d757111847b6849ae07acdd1205fffa1fc1253c"},
+    {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7233d65d9d651242a68801159763d09e9ec96e8a158dbf118dc090cd77a104c9"},
+    {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6119dc90483a5cb50a1306adb8d52c66e447da88ea44f323e0ae1a5fcb14256"},
+    {file = "pydantic_core-2.16.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:578114bc803a4c1ff9946d977c221e4376620a46cf78da267d946397dc9514a8"},
+    {file = "pydantic_core-2.16.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d8f99b147ff3fcf6b3cc60cb0c39ea443884d5559a30b1481e92495f2310ff2b"},
+    {file = "pydantic_core-2.16.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4ac6b4ce1e7283d715c4b729d8f9dab9627586dafce81d9eaa009dd7f25dd972"},
+    {file = "pydantic_core-2.16.3-cp39-none-win32.whl", hash = "sha256:e7774b570e61cb998490c5235740d475413a1f6de823169b4cf94e2fe9e9f6b2"},
+    {file = "pydantic_core-2.16.3-cp39-none-win_amd64.whl", hash = "sha256:9091632a25b8b87b9a605ec0e61f241c456e9248bfdcf7abdf344fdb169c81cf"},
+    {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:36fa178aacbc277bc6b62a2c3da95226520da4f4e9e206fdf076484363895d2c"},
+    {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:dcca5d2bf65c6fb591fff92da03f94cd4f315972f97c21975398bd4bd046854a"},
+    {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a72fb9963cba4cd5793854fd12f4cfee731e86df140f59ff52a49b3552db241"},
+    {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b60cc1a081f80a2105a59385b92d82278b15d80ebb3adb200542ae165cd7d183"},
+    {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:cbcc558401de90a746d02ef330c528f2e668c83350f045833543cd57ecead1ad"},
+    {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:fee427241c2d9fb7192b658190f9f5fd6dfe41e02f3c1489d2ec1e6a5ab1e04a"},
+    {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f4cb85f693044e0f71f394ff76c98ddc1bc0953e48c061725e540396d5c8a2e1"},
+    {file = "pydantic_core-2.16.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:b29eeb887aa931c2fcef5aa515d9d176d25006794610c264ddc114c053bf96fe"},
+    {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:a425479ee40ff021f8216c9d07a6a3b54b31c8267c6e17aa88b70d7ebd0e5e5b"},
+    {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:5c5cbc703168d1b7a838668998308018a2718c2130595e8e190220238addc96f"},
+    {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:99b6add4c0b39a513d323d3b93bc173dac663c27b99860dd5bf491b240d26137"},
+    {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75f76ee558751746d6a38f89d60b6228fa174e5172d143886af0f85aa306fd89"},
+    {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:00ee1c97b5364b84cb0bd82e9bbf645d5e2871fb8c58059d158412fee2d33d8a"},
+    {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:287073c66748f624be4cef893ef9174e3eb88fe0b8a78dc22e88eca4bc357ca6"},
+    {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:ed25e1835c00a332cb10c683cd39da96a719ab1dfc08427d476bce41b92531fc"},
+    {file = "pydantic_core-2.16.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:86b3d0033580bd6bbe07590152007275bd7af95f98eaa5bd36f3da219dcd93da"},
+    {file = "pydantic_core-2.16.3.tar.gz", hash = "sha256:1cac689f80a3abab2d3c0048b29eea5751114054f032a941a32de4c852c59cad"},
+]
+
+[package.dependencies]
+typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0"
+
+[[package]]
+name = "pyright"
+version = "1.1.354"
+description = "Command line wrapper for pyright"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "pyright-1.1.354-py3-none-any.whl", hash = "sha256:f28d61ae8ae035fc52ded1070e8d9e786051a26a4127bbd7a4ba0399b81b37b5"},
+    {file = "pyright-1.1.354.tar.gz", hash = "sha256:b1070dc774ff2e79eb0523fe87f4ba9a90550de7e4b030a2bc9e031864029a1f"},
+]
+
+[package.dependencies]
+nodeenv = ">=1.6.0"
+
+[package.extras]
+all = ["twine (>=3.4.1)"]
+dev = ["twine (>=3.4.1)"]
+
+[[package]]
+name = "redis"
+version = "5.0.3"
+description = "Python client for Redis database and key-value store"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "redis-5.0.3-py3-none-any.whl", hash = "sha256:5da9b8fe9e1254293756c16c008e8620b3d15fcc6dde6babde9541850e72a32d"},
+    {file = "redis-5.0.3.tar.gz", hash = "sha256:4973bae7444c0fbed64a06b87446f79361cb7e4ec1538c022d696ed7a5015580"},
+]
+
+[package.dependencies]
+async-timeout = {version = ">=4.0.3", markers = "python_full_version < \"3.11.3\""}
+
+[package.extras]
+hiredis = ["hiredis (>=1.0.0)"]
+ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==20.0.1)", "requests (>=2.26.0)"]
+
+[[package]]
+name = "ruff"
+version = "0.3.3"
+description = "An extremely fast Python linter and code formatter, written in Rust."
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "ruff-0.3.3-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:973a0e388b7bc2e9148c7f9be8b8c6ae7471b9be37e1cc732f8f44a6f6d7720d"},
+    {file = "ruff-0.3.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:cfa60d23269d6e2031129b053fdb4e5a7b0637fc6c9c0586737b962b2f834493"},
+    {file = "ruff-0.3.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1eca7ff7a47043cf6ce5c7f45f603b09121a7cc047447744b029d1b719278eb5"},
+    {file = "ruff-0.3.3-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e7d3f6762217c1da954de24b4a1a70515630d29f71e268ec5000afe81377642d"},
+    {file = "ruff-0.3.3-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b24c19e8598916d9c6f5a5437671f55ee93c212a2c4c569605dc3842b6820386"},
+    {file = "ruff-0.3.3-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:5a6cbf216b69c7090f0fe4669501a27326c34e119068c1494f35aaf4cc683778"},
+    {file = "ruff-0.3.3-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:352e95ead6964974b234e16ba8a66dad102ec7bf8ac064a23f95371d8b198aab"},
+    {file = "ruff-0.3.3-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d6ab88c81c4040a817aa432484e838aaddf8bfd7ca70e4e615482757acb64f8"},
+    {file = "ruff-0.3.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79bca3a03a759cc773fca69e0bdeac8abd1c13c31b798d5bb3c9da4a03144a9f"},
+    {file = "ruff-0.3.3-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:2700a804d5336bcffe063fd789ca2c7b02b552d2e323a336700abb8ae9e6a3f8"},
+    {file = "ruff-0.3.3-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:fd66469f1a18fdb9d32e22b79f486223052ddf057dc56dea0caaf1a47bdfaf4e"},
+    {file = "ruff-0.3.3-py3-none-musllinux_1_2_i686.whl", hash = "sha256:45817af234605525cdf6317005923bf532514e1ea3d9270acf61ca2440691376"},
+    {file = "ruff-0.3.3-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:0da458989ce0159555ef224d5b7c24d3d2e4bf4c300b85467b08c3261c6bc6a8"},
+    {file = "ruff-0.3.3-py3-none-win32.whl", hash = "sha256:f2831ec6a580a97f1ea82ea1eda0401c3cdf512cf2045fa3c85e8ef109e87de0"},
+    {file = "ruff-0.3.3-py3-none-win_amd64.whl", hash = "sha256:be90bcae57c24d9f9d023b12d627e958eb55f595428bafcb7fec0791ad25ddfc"},
+    {file = "ruff-0.3.3-py3-none-win_arm64.whl", hash = "sha256:0171aab5fecdc54383993389710a3d1227f2da124d76a2784a7098e818f92d61"},
+    {file = "ruff-0.3.3.tar.gz", hash = "sha256:38671be06f57a2f8aba957d9f701ea889aa5736be806f18c0cd03d6ff0cbca8d"},
+]
+
+[[package]]
+name = "setuptools"
+version = "69.2.0"
+description = "Easily download, build, install, upgrade, and uninstall Python packages"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "setuptools-69.2.0-py3-none-any.whl", hash = "sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c"},
+    {file = "setuptools-69.2.0.tar.gz", hash = "sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e"},
+]
+
+[package.extras]
+docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"]
+testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"]
+testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"]
+
+[[package]]
+name = "sqlalchemy"
+version = "2.0.28"
+description = "Database Abstraction Library"
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "SQLAlchemy-2.0.28-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e0b148ab0438f72ad21cb004ce3bdaafd28465c4276af66df3b9ecd2037bf252"},
+    {file = "SQLAlchemy-2.0.28-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:bbda76961eb8f27e6ad3c84d1dc56d5bc61ba8f02bd20fcf3450bd421c2fcc9c"},
+    {file = "SQLAlchemy-2.0.28-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:feea693c452d85ea0015ebe3bb9cd15b6f49acc1a31c28b3c50f4db0f8fb1e71"},
+    {file = "SQLAlchemy-2.0.28-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5da98815f82dce0cb31fd1e873a0cb30934971d15b74e0d78cf21f9e1b05953f"},
+    {file = "SQLAlchemy-2.0.28-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:4a5adf383c73f2d49ad15ff363a8748319ff84c371eed59ffd0127355d6ea1da"},
+    {file = "SQLAlchemy-2.0.28-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:56856b871146bfead25fbcaed098269d90b744eea5cb32a952df00d542cdd368"},
+    {file = "SQLAlchemy-2.0.28-cp310-cp310-win32.whl", hash = "sha256:943aa74a11f5806ab68278284a4ddd282d3fb348a0e96db9b42cb81bf731acdc"},
+    {file = "SQLAlchemy-2.0.28-cp310-cp310-win_amd64.whl", hash = "sha256:c6c4da4843e0dabde41b8f2e8147438330924114f541949e6318358a56d1875a"},
+    {file = "SQLAlchemy-2.0.28-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:46a3d4e7a472bfff2d28db838669fc437964e8af8df8ee1e4548e92710929adc"},
+    {file = "SQLAlchemy-2.0.28-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0d3dd67b5d69794cfe82862c002512683b3db038b99002171f624712fa71aeaa"},
+    {file = "SQLAlchemy-2.0.28-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c61e2e41656a673b777e2f0cbbe545323dbe0d32312f590b1bc09da1de6c2a02"},
+    {file = "SQLAlchemy-2.0.28-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0315d9125a38026227f559488fe7f7cee1bd2fbc19f9fd637739dc50bb6380b2"},
+    {file = "SQLAlchemy-2.0.28-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:af8ce2d31679006e7b747d30a89cd3ac1ec304c3d4c20973f0f4ad58e2d1c4c9"},
+    {file = "SQLAlchemy-2.0.28-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:81ba314a08c7ab701e621b7ad079c0c933c58cdef88593c59b90b996e8b58fa5"},
+    {file = "SQLAlchemy-2.0.28-cp311-cp311-win32.whl", hash = "sha256:1ee8bd6d68578e517943f5ebff3afbd93fc65f7ef8f23becab9fa8fb315afb1d"},
+    {file = "SQLAlchemy-2.0.28-cp311-cp311-win_amd64.whl", hash = "sha256:ad7acbe95bac70e4e687a4dc9ae3f7a2f467aa6597049eeb6d4a662ecd990bb6"},
+    {file = "SQLAlchemy-2.0.28-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:d3499008ddec83127ab286c6f6ec82a34f39c9817f020f75eca96155f9765097"},
+    {file = "SQLAlchemy-2.0.28-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9b66fcd38659cab5d29e8de5409cdf91e9986817703e1078b2fdaad731ea66f5"},
+    {file = "SQLAlchemy-2.0.28-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bea30da1e76cb1acc5b72e204a920a3a7678d9d52f688f087dc08e54e2754c67"},
+    {file = "SQLAlchemy-2.0.28-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:124202b4e0edea7f08a4db8c81cc7859012f90a0d14ba2bf07c099aff6e96462"},
+    {file = "SQLAlchemy-2.0.28-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:e23b88c69497a6322b5796c0781400692eca1ae5532821b39ce81a48c395aae9"},
+    {file = "SQLAlchemy-2.0.28-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4b6303bfd78fb3221847723104d152e5972c22367ff66edf09120fcde5ddc2e2"},
+    {file = "SQLAlchemy-2.0.28-cp312-cp312-win32.whl", hash = "sha256:a921002be69ac3ab2cf0c3017c4e6a3377f800f1fca7f254c13b5f1a2f10022c"},
+    {file = "SQLAlchemy-2.0.28-cp312-cp312-win_amd64.whl", hash = "sha256:b4a2cf92995635b64876dc141af0ef089c6eea7e05898d8d8865e71a326c0385"},
+    {file = "SQLAlchemy-2.0.28-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e91b5e341f8c7f1e5020db8e5602f3ed045a29f8e27f7f565e0bdee3338f2c7"},
+    {file = "SQLAlchemy-2.0.28-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45c7b78dfc7278329f27be02c44abc0d69fe235495bb8e16ec7ef1b1a17952db"},
+    {file = "SQLAlchemy-2.0.28-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3eba73ef2c30695cb7eabcdb33bb3d0b878595737479e152468f3ba97a9c22a4"},
+    {file = "SQLAlchemy-2.0.28-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:5df5d1dafb8eee89384fb7a1f79128118bc0ba50ce0db27a40750f6f91aa99d5"},
+    {file = "SQLAlchemy-2.0.28-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:2858bbab1681ee5406650202950dc8f00e83b06a198741b7c656e63818633526"},
+    {file = "SQLAlchemy-2.0.28-cp37-cp37m-win32.whl", hash = "sha256:9461802f2e965de5cff80c5a13bc945abea7edaa1d29360b485c3d2b56cdb075"},
+    {file = "SQLAlchemy-2.0.28-cp37-cp37m-win_amd64.whl", hash = "sha256:a6bec1c010a6d65b3ed88c863d56b9ea5eeefdf62b5e39cafd08c65f5ce5198b"},
+    {file = "SQLAlchemy-2.0.28-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:843a882cadebecc655a68bd9a5b8aa39b3c52f4a9a5572a3036fb1bb2ccdc197"},
+    {file = "SQLAlchemy-2.0.28-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:dbb990612c36163c6072723523d2be7c3eb1517bbdd63fe50449f56afafd1133"},
+    {file = "SQLAlchemy-2.0.28-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd7e4baf9161d076b9a7e432fce06217b9bd90cfb8f1d543d6e8c4595627edb9"},
+    {file = "SQLAlchemy-2.0.28-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e0a5354cb4de9b64bccb6ea33162cb83e03dbefa0d892db88a672f5aad638a75"},
+    {file = "SQLAlchemy-2.0.28-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:fffcc8edc508801ed2e6a4e7b0d150a62196fd28b4e16ab9f65192e8186102b6"},
+    {file = "SQLAlchemy-2.0.28-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:aca7b6d99a4541b2ebab4494f6c8c2f947e0df4ac859ced575238e1d6ca5716b"},
+    {file = "SQLAlchemy-2.0.28-cp38-cp38-win32.whl", hash = "sha256:8c7f10720fc34d14abad5b647bc8202202f4948498927d9f1b4df0fb1cf391b7"},
+    {file = "SQLAlchemy-2.0.28-cp38-cp38-win_amd64.whl", hash = "sha256:243feb6882b06a2af68ecf4bec8813d99452a1b62ba2be917ce6283852cf701b"},
+    {file = "SQLAlchemy-2.0.28-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:fc4974d3684f28b61b9a90fcb4c41fb340fd4b6a50c04365704a4da5a9603b05"},
+    {file = "SQLAlchemy-2.0.28-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:87724e7ed2a936fdda2c05dbd99d395c91ea3c96f029a033a4a20e008dd876bf"},
+    {file = "SQLAlchemy-2.0.28-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68722e6a550f5de2e3cfe9da6afb9a7dd15ef7032afa5651b0f0c6b3adb8815d"},
+    {file = "SQLAlchemy-2.0.28-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:328529f7c7f90adcd65aed06a161851f83f475c2f664a898af574893f55d9e53"},
+    {file = "SQLAlchemy-2.0.28-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:df40c16a7e8be7413b885c9bf900d402918cc848be08a59b022478804ea076b8"},
+    {file = "SQLAlchemy-2.0.28-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:426f2fa71331a64f5132369ede5171c52fd1df1bd9727ce621f38b5b24f48750"},
+    {file = "SQLAlchemy-2.0.28-cp39-cp39-win32.whl", hash = "sha256:33157920b233bc542ce497a81a2e1452e685a11834c5763933b440fedd1d8e2d"},
+    {file = "SQLAlchemy-2.0.28-cp39-cp39-win_amd64.whl", hash = "sha256:2f60843068e432311c886c5f03c4664acaef507cf716f6c60d5fde7265be9d7b"},
+    {file = "SQLAlchemy-2.0.28-py3-none-any.whl", hash = "sha256:78bb7e8da0183a8301352d569900d9d3594c48ac21dc1c2ec6b3121ed8b6c986"},
+    {file = "SQLAlchemy-2.0.28.tar.gz", hash = "sha256:dd53b6c4e6d960600fd6532b79ee28e2da489322fcf6648738134587faf767b6"},
+]
+
+[package.dependencies]
+greenlet = {version = "!=0.4.17", optional = true, markers = "platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\" or extra == \"asyncio\""}
+typing-extensions = ">=4.6.0"
+
+[package.extras]
+aiomysql = ["aiomysql (>=0.2.0)", "greenlet (!=0.4.17)"]
+aioodbc = ["aioodbc", "greenlet (!=0.4.17)"]
+aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing_extensions (!=3.10.0.1)"]
+asyncio = ["greenlet (!=0.4.17)"]
+asyncmy = ["asyncmy (>=0.2.3,!=0.2.4,!=0.2.6)", "greenlet (!=0.4.17)"]
+mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2,!=1.1.5)"]
+mssql = ["pyodbc"]
+mssql-pymssql = ["pymssql"]
+mssql-pyodbc = ["pyodbc"]
+mypy = ["mypy (>=0.910)"]
+mysql = ["mysqlclient (>=1.4.0)"]
+mysql-connector = ["mysql-connector-python"]
+oracle = ["cx_oracle (>=8)"]
+oracle-oracledb = ["oracledb (>=1.0.1)"]
+postgresql = ["psycopg2 (>=2.7)"]
+postgresql-asyncpg = ["asyncpg", "greenlet (!=0.4.17)"]
+postgresql-pg8000 = ["pg8000 (>=1.29.1)"]
+postgresql-psycopg = ["psycopg (>=3.0.7)"]
+postgresql-psycopg2binary = ["psycopg2-binary"]
+postgresql-psycopg2cffi = ["psycopg2cffi"]
+postgresql-psycopgbinary = ["psycopg[binary] (>=3.0.7)"]
+pymysql = ["pymysql"]
+sqlcipher = ["sqlcipher3_binary"]
+
+[[package]]
+name = "structlog"
+version = "24.1.0"
+description = "Structured Logging for Python"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "structlog-24.1.0-py3-none-any.whl", hash = "sha256:3f6efe7d25fab6e86f277713c218044669906537bb717c1807a09d46bca0714d"},
+    {file = "structlog-24.1.0.tar.gz", hash = "sha256:41a09886e4d55df25bdcb9b5c9674bccfab723ff43e0a86a1b7b236be8e57b16"},
+]
+
+[package.extras]
+dev = ["structlog[tests,typing]"]
+docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-mermaid", "sphinxext-opengraph", "twisted"]
+tests = ["freezegun (>=0.2.8)", "pretend", "pytest (>=6.0)", "pytest-asyncio (>=0.17)", "simplejson"]
+typing = ["mypy (>=1.4)", "rich", "twisted"]
+
+[[package]]
+name = "typing-extensions"
+version = "4.10.0"
+description = "Backported and Experimental Type Hints for Python 3.8+"
+optional = false
+python-versions = ">=3.8"
+files = [
+    {file = "typing_extensions-4.10.0-py3-none-any.whl", hash = "sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475"},
+    {file = "typing_extensions-4.10.0.tar.gz", hash = "sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb"},
+]
+
+[metadata]
+lock-version = "2.0"
+python-versions = "^3.11"
+content-hash = "390e2c493fedc2d4ab5e62a8ef220a07c4909fba4f7678fcb519e1dc9d344ad5"

src/torrent-ingestor/pyproject.toml

@@ -0,0 +1,60 @@
+[tool.poetry]
+name = "torrent-ingestor"
+version = "0.1.0"
+description = "Annatar Torrent Ingestor"
+authors = ["Brett <eruiluvatar@pnbx.xyz>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.11"
+redis = "^5.0.3"
+pydantic = "^2.6.4"
+sqlalchemy = {extras = ["asyncio"], version = "^2.0.28"}
+databases = {extras = ["postgresql"], version = "^0.9.0"}
+psycopg2-binary = "^2.9.9"
+structlog = "^24.1.0"
+
+
+[tool.poetry.group.dev.dependencies]
+ruff = "^0.3.3"
+pyright = "^1.1.354"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+
+[tool.ruff]
+# Exclude a variety of commonly ignored directories.
+line-length = 100
+exclude = [ ".direnv", ".git", ".pyenv", ".pytest_cache", ".pytype", ".ruff_cache", ".vscode", "__pypackages__", "_build", "build", "dist", "typings", "venv"]
+
+[tool.ruff.lint]
+# https://docs.astral.sh/ruff/rules/
+ignore = [
+  "PLR0913", # flask8-bugbear: Too many arguments for a method or function
+  "PLR0911", # flask8-bugbear: Too many return statements
+  "PLR2004", # flake8-bugbear: Magic value used in comparison
+  "S104", # flake8-bandit: Possible binding to all interfaces
+  "S108", # flake8-bandit: Probable insecure usage of temp file/directory
+  "S311", # flake8-bandit: Standard pseudo-random generators are not suitable for security/cryptographic purposes
+]
+extend-select = [
+  "I",   # isort
+  "C90", # mccabe complexity
+  "B",   # flake8-bugbear
+  "PL",  # pycodestyle
+  "S",   # flake8-bandit
+  "T10", # flake8-debugger
+  "PIE", # flake8-pie
+  "T20", # flake8-print
+  "Q",   # flake8-quotes
+  "RSE", # flake8-raise
+  "RET", # flake8-return
+  "SLF", # flake8-self
+  "SIM", # flake8-simplify
+  "ARG", # flake8-unused-arguments
+]
+
+[tool.ruff.lint.isort]
+case-sensitive = true

src/torrent-ingestor/torrent_ingestor/logger.py

@@ -0,0 +1,56 @@
+import inspect
+import logging
+import os
+import sys
+from typing import Any
+
+import structlog
+
+LOGGER = os.getenv("LOGGER") or "console"
+
+root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+LOG_LEVEL = os.getenv("LOG_LEVEL", "info")
+logging.basicConfig(
+    format="%(message)s",
+    stream=sys.stderr,
+    level=LOG_LEVEL.upper(),
+)
+
+
+def add_code_info(_: logging.Logger, __: str, event_dict: Any) -> dict[str, Any]:
+    frame = inspect.currentframe().f_back.f_back.f_back.f_back.f_back  # type: ignore
+    event_dict["code_func"] = frame.f_code.co_name  # type: ignore
+    # fname: str = frame.f_code.co_filename.replace(root_dir, "").lstrip("/")  # type: ignore
+    event_dict["code_line"] = frame.f_lineno  # type: ignore
+    # event_dict["code_file"] = f"{fname}:{frame.f_lineno}"  # type: ignore
+    return event_dict
+
+
+structlog.configure(
+    processors=[
+        structlog.stdlib.filter_by_level,
+        structlog.contextvars.merge_contextvars,
+        structlog.stdlib.add_log_level,
+        structlog.stdlib.add_logger_name,
+        add_code_info,
+        structlog.processors.TimeStamper(fmt="iso"),
+        structlog.processors.EventRenamer(to="msg"),
+        structlog.processors.StackInfoRenderer(),
+        structlog.processors.format_exc_info,
+        (
+            structlog.processors.JSONRenderer()
+            if LOGGER == "json"
+            else structlog.dev.ConsoleRenderer(event_key="msg")
+        ),
+    ],
+    context_class=dict,
+    logger_factory=structlog.stdlib.LoggerFactory(),
+    wrapper_class=structlog.stdlib.BoundLogger,
+    cache_logger_on_first_use=True,
+)
+
+
+def init():
+    # the work is done on load
+    return None

src/torrent-ingestor/torrent_ingestor/main.py

@@ -0,0 +1,148 @@
+import asyncio
+import contextlib
+from datetime import datetime, timedelta
+from os import environ
+
+import logger
+import structlog
+from pydantic import BaseModel
+from redis import asyncio as redis
+from sqlalchemy import Boolean, Column, DateTime, Integer, String, create_engine
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.orm import Session, declarative_base, sessionmaker
+
+logger.init()
+log = structlog.getLogger(__name__)
+
+Base = declarative_base()
+
+
+class TorrentAddedEvent(BaseModel):
+    info_hash: str
+    title: str
+    imdb: str
+    size: int
+    indexer: str
+    category: str
+    season: int | None = None
+    episode: int | None = None
+
+
+class IngestedTorrent(Base):
+    __tablename__ = "ingested_torrents"
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    name = Column(String)
+    source = Column(String, default="Annatar Pubsub")
+    category = Column(String)
+    info_hash = Column(String)
+    size = Column(Integer, default=0)
+    seeders = Column(Integer, default=0)
+    leechers = Column(Integer, default=0)
+    imdb = Column(String, nullable=False)
+    processed = Column(Boolean, default=False)
+    createdAt = Column(DateTime, default=datetime.utcnow)
+    updatedAt = Column(DateTime, default=datetime.utcnow)
+
+
+async def write_to_postgres(
+    session: sessionmaker[Session],
+    queue: asyncio.Queue[TorrentAddedEvent],
+) -> None:
+    while True:
+        sess = session()
+        try:
+            event: TorrentAddedEvent = await queue.get()
+            with contextlib.suppress(IntegrityError):
+                sess.add(
+                    IngestedTorrent(
+                        name=event.title,
+                        info_hash=event.info_hash,
+                        imdb=event.imdb,
+                        size=event.size,
+                        category=event.category,
+                        updatedAt=datetime.utcnow(),
+                    )
+                )
+                sess.commit()
+                log.info("saved torrent", torrent=event.info_hash)
+            queue.task_done()
+        except asyncio.CancelledError:
+            return
+        except Exception as e:
+            log.error("error saving torrent", exc_info=e)
+            await asyncio.sleep(1)
+        finally:
+            sess.close()
+
+
+def run_redis_subscriber(
+    redis_url: str, channel_name: str, queue: asyncio.Queue[TorrentAddedEvent]
+) -> None:
+    asyncio.run(redis_subscriber(redis_url, channel_name, queue))
+
+
+def start_queue_processor(SessionLocal: sessionmaker[Session], queue: asyncio.Queue):
+    asyncio.run(run_queue_processors(SessionLocal, queue, 10))
+
+
+async def redis_subscriber(
+    redis_url: str, channel_name: str, queue: asyncio.Queue[TorrentAddedEvent]
+) -> None:
+    db = redis.from_url(redis_url)
+    pubsub = db.pubsub()
+    await pubsub.subscribe(channel_name)
+    try:
+        async for message in pubsub.listen():
+            if message["type"] == "message":
+                log.debug("got message", message=message)
+                event_data = TorrentAddedEvent.model_validate_json(message["data"].decode())
+                # only process the event if we haven't seen it in the last 24 hours
+                # this is to prevent processing the same event multiple times
+                # to allow multiple consumers since redis pubsub is a broadcast
+                if await db.set(
+                    name=f"kc:torrents:seen:{event_data.info_hash}",
+                    value=1,
+                    nx=True,
+                    ex=timedelta(days=1),
+                ):
+                    await queue.put(event_data)
+    finally:
+        await pubsub.unsubscribe(channel_name)
+        await db.aclose()
+
+
+async def main():
+    REDIS_URL = environ.get("REDIS_URL", "redis://localhost")
+    if not REDIS_URL:
+        raise ValueError("REDIS_URL env var is required.")
+
+    # Database connection settings
+    DATABASE_URL = environ.get("POSTGRES_URL")
+    if not DATABASE_URL:
+        raise ValueError("POSTGRES_URL env var is required.")
+    engine = create_engine(
+        DATABASE_URL, pool_size=100, max_overflow=0, pool_timeout=5, pool_recycle=3600
+    )
+    SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+
+    Base.metadata.create_all(bind=engine, checkfirst=True)
+
+    queue = asyncio.Queue[TorrentAddedEvent](maxsize=100)
+
+    await asyncio.wait(
+        [
+            asyncio.create_task(redis_subscriber(REDIS_URL, "events:v1:torrent:added", queue)),
+            asyncio.create_task(run_queue_processors(SessionLocal, queue, 10)),
+        ]
+    )
+
+
+async def run_queue_processors(
+    session: sessionmaker[Session], queue: asyncio.Queue, n_processors: int
+):
+    tasks = [asyncio.create_task(write_to_postgres(session, queue)) for _ in range(n_processors)]
+    await asyncio.gather(*tasks)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())