helm chart [wip]

* Add housekeeping service to clean stale torrents

* version bump

.github/workflows/base_image_workflow.yaml

@@ -6,12 +6,16 @@ on:
       CONTEXT:
         required: true
         type: string
+      DOCKERFILE:
+        required: true
+        type: string
       IMAGE_NAME:
         required: true
         type: string
 
 env:
   CONTEXT: ${{ inputs.CONTEXT }}
+  DOCKERFILE: ${{ inputs.DOCKERFILE }}
   IMAGE_NAME: ${{ inputs.IMAGE_NAME }}
   PLATFORMS: linux/amd64,linux/arm64
 
@@ -21,11 +25,13 @@ jobs:
     steps:
       - name: Setting variables
         run: |
-          echo "CONTEXT=${{ env.CONTEXT }}
-          echo "IMAGE_NAME=${{ env.IMAGE_NAME }}
+          echo "CONTEXT=${{ env.CONTEXT }}"
+          echo "DOCKERFILE=${{ env.DOCKERFILE }}"
+          echo "IMAGE_NAME=${{ env.IMAGE_NAME }}"
           echo "PLATFORMS=${{ env.PLATFORMS }}"
     outputs:
       CONTEXT: ${{ env.CONTEXT }}
+      DOCKERFILE: ${{ env.DOCKERFILE }}
       IMAGE_NAME: ${{ env.IMAGE_NAME }}
       PLATFORMS: ${{ env.PLATFORMS }}
 
@@ -70,14 +76,17 @@ jobs:
           flavor: |
             latest=auto
           tags: |
-            type=edge,branch=master,commit=${{ github.sha }}
+            type=ref,event=tag
+            type=ref,event=pr
             type=sha,commit=${{ github.sha }}
+            type=semver,pattern={{version}}
             type=raw,value=latest,enable={{is_default_branch}}
 
       - name: Build image for scanning
         uses: docker/build-push-action@v5
         with:
           context: ${{ needs.set-vars.outputs.CONTEXT }}
+          file: ${{ needs.set-vars.outputs.DOCKERFILE }}
           push: true
           provenance: false
           tags: localhost:5000/dockle-examine-image:test
@@ -130,10 +139,11 @@ jobs:
           sarif_file: 'trivy-results-os.sarif'
 
       - name: Push Service Image to repo
-        if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master'
+        # if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master'
         uses: docker/build-push-action@v5
         with:
           context: ${{ needs.set-vars.outputs.CONTEXT }}
+          file: ${{ needs.set-vars.outputs.DOCKERFILE }}
           push: true
           provenance: false
           tags: ${{ steps.docker-metadata.outputs.tags }}

.github/workflows/build_addon.yaml

@@ -2,13 +2,17 @@ name: Build and Push Addon Service
 
 on:
   push:
+    tags:
+      - '**'
     paths:
-      - 'src/node/addon/**'
+      - 'src/addon/**'
+  workflow_dispatch:
 
 jobs:
   process:
     uses: ./.github/workflows/base_image_workflow.yaml
     secrets: inherit
     with:
-      CONTEXT: ./src/node/addon/
+      CONTEXT: ./src/addon/
+      DOCKERFILE: ./src/addon/Dockerfile
       IMAGE_NAME: knightcrawler-addon

.github/workflows/build_consumer.yaml

@@ -2,13 +2,17 @@ name: Build and Push Consumer Service
 
 on:
   push:
+    tags:
+      - '**'
     paths:
-      - 'src/node/consumer/**'
+      - 'src/torrent-consumer/**'
+  workflow_dispatch:
 
 jobs:
   process:
     uses: ./.github/workflows/base_image_workflow.yaml
     secrets: inherit
     with:
-      CONTEXT: ./src/node/consumer/
+      CONTEXT: ./src/
+      DOCKERFILE: ./src/torrent-consumer/Dockerfile
       IMAGE_NAME: knightcrawler-consumer

.github/workflows/build_debrid_collector.yaml

@@ -0,0 +1,18 @@
+name: Build and Push Debrid Collector Service
+
+on:
+  push:
+    tags:
+      - '**'
+    paths:
+      - 'src/debrid-collector/**'
+  workflow_dispatch:
+
+jobs:
+  process:
+    uses: ./.github/workflows/base_image_workflow.yaml
+    secrets: inherit
+    with:
+      CONTEXT: ./src/
+      DOCKERFILE: ./src/debrid-collector/Dockerfile
+      IMAGE_NAME: knightcrawler-debrid-collector

.github/workflows/build_jackett-addon.yaml

@@ -2,13 +2,17 @@ name: Build and Push Jackett Addon Service
 
 on:
   push:
+    tags:
+      - '**'
     paths:
-      - 'src/node/addon-jackett/**'
+      - 'src/addon-jackett/**'
+  workflow_dispatch:
 
 jobs:
   process:
     uses: ./.github/workflows/base_image_workflow.yaml
     secrets: inherit
     with:
-      CONTEXT: ./src/node/addon-jackett/
+      CONTEXT: ./src/addon-jackett/
+      DOCKERFILE: ./src/addon-jackett/Dockerfile
       IMAGE_NAME: knightcrawler-addon-jackett

.github/workflows/build_metadata.yaml

@@ -2,8 +2,11 @@ name: Build and Push Metadata Service
 
 on:
   push:
+    tags:
+      - '**'
     paths:
       - 'src/metadata/**'
+  workflow_dispatch:
 
 jobs:
   process:
@@ -11,4 +14,5 @@ jobs:
     secrets: inherit
     with:
       CONTEXT: ./src/metadata/
+      DOCKERFILE: ./src/metadata/Dockerfile
       IMAGE_NAME: knightcrawler-metadata

.github/workflows/build_migrator.yaml

@@ -2,8 +2,11 @@ name: Build and Push Migrator Service
 
 on:
   push:
+    tags:
+      - '**'
     paths:
       - 'src/migrator/**'
+  workflow_dispatch:
 
 jobs:
   process:
@@ -11,4 +14,5 @@ jobs:
     secrets: inherit
     with:
       CONTEXT: ./src/migrator/
+      DOCKERFILE: ./src/migrator/Dockerfile
       IMAGE_NAME: knightcrawler-migrator

.github/workflows/build_producer.yaml

@@ -2,13 +2,17 @@ name: Build and Push Producer Service
 
 on:
   push:
+    tags:
+      - '**'
     paths:
       - 'src/producer/**'
+  workflow_dispatch:
 
 jobs:
   process:
     uses: ./.github/workflows/base_image_workflow.yaml
     secrets: inherit
     with:
-      CONTEXT: ./src/producer/
+      CONTEXT: ./src/
+      DOCKERFILE: ./src/producer/src/Dockerfile
       IMAGE_NAME: knightcrawler-producer

.github/workflows/build_qbit_collector.yaml

@@ -0,0 +1,18 @@
+name: Build and Push Qbit Collector Service
+
+on:
+  push:
+    tags:
+      - '**'
+    paths:
+      - 'src/qbit-collector/**'
+  workflow_dispatch:
+
+jobs:
+  process:
+    uses: ./.github/workflows/base_image_workflow.yaml
+    secrets: inherit
+    with:
+      CONTEXT: ./src/
+      DOCKERFILE: ./src/qbit-collector/Dockerfile
+      IMAGE_NAME: knightcrawler-qbit-collector

.github/workflows/build_tissue.yaml

@@ -0,0 +1,18 @@
+name: Build and Push Tissue Service
+
+on:
+  push:
+    tags:
+      - '**'
+    paths:
+      - 'src/tissue/**'
+  workflow_dispatch:
+
+jobs:
+  process:
+    uses: ./.github/workflows/base_image_workflow.yaml
+    secrets: inherit
+    with:
+      CONTEXT: ./src/tissue/
+      DOCKERFILE: ./src/tissue/Dockerfile
+      IMAGE_NAME: knightcrawler-tissue

.github/workflows/build_torrent_ingester.yaml

@@ -0,0 +1,15 @@
+name: Build and Push Torrent Ingestor Service
+
+on:
+  push:
+    paths:
+      - 'src/torrent-ingestor/**'
+
+jobs:
+  process:
+    uses: ./.github/workflows/base_image_workflow.yaml
+    secrets: inherit
+    with:
+      CONTEXT: ./src/torrent-ingestor
+      DOCKERFILE: ./src/torrent-ingestor/Dockerfile
+      IMAGE_NAME: knightcrawler-torrent-ingestor

.gitignore

@@ -355,6 +355,9 @@ MigrationBackup/
 # Fody - auto-generated XML schema
 FodyWeavers.xsd
 
+#  Jetbrains ide's run profiles (Could contain sensative information)
+**/.run/
+
 # VS Code files for those working on multiple tools
 .vscode/*
 !.vscode/settings.json
@@ -392,8 +395,6 @@ dist/
 downloads/
 eggs/
 .eggs/
-lib/
-lib64/
 parts/
 sdist/
 var/
@@ -607,3 +608,7 @@ fabric.properties
 # Caddy logs
 !**/caddy/logs/.gitkeep
 **/caddy/logs/**
+
+# Mac directory indexes
+.DS_Store
+deployment/docker/stack.env

.pre-commit-config.yaml

@@ -3,6 +3,7 @@ repos:
     rev: v4.5.0
     hooks:
       - id: check-added-large-files
+        args: ['--maxkb=2500']
       - id: check-json
       - id: check-toml
       - id: check-xml
@@ -15,5 +16,6 @@ repos:
     rev: v2.2.6
     hooks:
       - id: codespell
-        exclude: ^src/node/consumer/test/
+        exclude: |
+            (?x)^(src/node/consumer/test/.*|src/producer/Data/.*|src/tissue/Data/.*)$
         args: ["-L", "strem,chage"]

README.md

@@ -7,9 +7,6 @@
 
 ## Contents
 
-> [!CAUTION]
-> Until we reach `v1.0.0`, please consider releases as alpha.
-
 > [!IMPORTANT]
 > The latest change renames the project and requires a [small migration](#selfhostio-to-knightcrawler-migration).
 - [Contents](#contents)
@@ -54,11 +51,11 @@ Download and install [Docker Compose](https://docs.docker.com/compose/install/),
 
 ### Environment Setup
 
-Before running the project, you need to set up the environment variables. Copy the `.env.example` file to `.env`:
+Before running the project, you need to set up the environment variables. Edit the values in `stack.env`:
 
 ```sh
 cd deployment/docker
-cp .env.example .env
+code stack.env
 ```
 
 Then set any of the values you wouldd like to customize.
@@ -70,9 +67,6 @@ Then set any of the values you wouldd like to customize.
 
 By default, Knight Crawler is configured to be *relatively* conservative in its resource usage. If running on a decent machine (16GB RAM, i5+ or equivalent), you can increase some settings to increase consumer throughput. This is especially helpful if you have a large backlog from [importing databases](#importing-external-dumps).
 
-In your `.env` file, under the `# Consumer` section increase `CONSUMER_REPLICAS` from `3` to `15`.
-You can also increase `JOB_CONCURRENCY` from `5` to `10`.
-
 ### DebridMediaManager setup (optional)
 
 There are some optional steps you should take to maximise the number of movies/tv shows we can find.
@@ -93,9 +87,9 @@ We can search DebridMediaManager hash lists which are hosted on GitHub. This all
         (checked) Public Repositories (read-only)
    ```
 4. Click `Generate token`
-5. Take the new token and add it to the bottom of the [.env](deployment/docker/.env) file
+5. Take the new token and add it to the bottom of the [stack.env](deployment/docker/stack.env) file
    ```
-   GithubSettings__PAT=<YOUR TOKEN HERE>
+   GITHUB_PAT=<YOUR TOKEN HERE>
    ```
 ### Configure external access
 
@@ -146,7 +140,7 @@ Remove or comment out the port for the addon, and connect it to Caddy:
 addon:
   <<: *knightcrawler-app
   env_file:
-    - .env
+    - stack.env
   hostname: knightcrawler-addon
   image: gabisonfire/knightcrawler-addon:latest
   labels:

deployment/docker/.env.example

@@ -1,55 +0,0 @@
-# General environment variables
-TZ=London/Europe
-
-# PostgreSQL
-POSTGRES_HOST=postgres
-POSTGRES_PORT=5432
-POSTGRES_USER=postgres
-POSTGRES_PASSWORD=postgres
-POSTGRES_DB=knightcrawler
-
-# MongoDB
-MONGODB_HOST=mongodb
-MONGODB_PORT=27017
-MONGODB_DB=knightcrawler
-MONGODB_USER=mongo
-MONGODB_PASSWORD=mongo
-
-# RabbitMQ
-RABBITMQ_HOST=rabbitmq
-RABBITMQ_USER=guest
-RABBITMQ_PASSWORD=guest
-RABBITMQ_QUEUE_NAME=ingested
-RABBITMQ_DURABLE=true
-RABBITMQ_MAX_QUEUE_SIZE=0
-RABBITMQ_MAX_PUBLISH_BATCH_SIZE=500
-RABBITMQ_PUBLISH_INTERVAL_IN_SECONDS=10
-
-# Metadata
-## Only used if DATA_ONCE is set to false. If true, the schedule is ignored
-METADATA_DOWNLOAD_IMDB_DATA_SCHEDULE="0 0 1 * *"
-## If true, the metadata will be downloaded once and then the schedule will be ignored
-METADATA_DOWNLOAD_IMDB_DATA_ONCE=true
-## Controls the amount of records processed in memory at any given time during import, higher values will consume more memory
-METADATA_INSERT_BATCH_SIZE=25000
-
-# Addon
-DEBUG_MODE=false
-
-# Consumer
-JOB_CONCURRENCY=5
-JOBS_ENABLED=true
-## can be debug for extra verbosity (a lot more verbosity - useful for development)
-LOG_LEVEL=info
-MAX_CONNECTIONS_PER_TORRENT=10
-MAX_CONNECTIONS_OVERALL=100
-TORRENT_TIMEOUT=30000
-UDP_TRACKERS_ENABLED=true
-CONSUMER_REPLICAS=3
-## Fix for #66 - toggle on for development
-AUTO_CREATE_AND_APPLY_MIGRATIONS=false
-## Allows control of the threshold for matching titles to the IMDB dataset. The closer to 0, the more strict the matching.
-TITLE_MATCH_THRESHOLD=0.25
-
-# Producer
-GITHUB_PAT=

deployment/docker/config/qbit/qbittorrent.conf

@@ -0,0 +1,62 @@
+[Application]
+FileLogger\Age=1
+FileLogger\AgeType=1
+FileLogger\Backup=true
+FileLogger\DeleteOld=true
+FileLogger\Enabled=true
+FileLogger\MaxSizeBytes=66560
+FileLogger\Path=/config/qBittorrent/logs
+
+[AutoRun]
+enabled=false
+program=
+
+[BitTorrent]
+Session\AnonymousModeEnabled=true
+Session\BTProtocol=TCP
+Session\DefaultSavePath=/downloads/
+Session\ExcludedFileNames=
+Session\MaxActiveCheckingTorrents=5
+Session\MaxActiveDownloads=10
+Session\MaxActiveTorrents=50
+Session\MaxActiveUploads=50
+Session\MaxConnections=2000
+Session\Port=6881
+Session\QueueingSystemEnabled=true
+Session\TempPath=/downloads/incomplete/
+Session\TorrentStopCondition=MetadataReceived
+
+[Core]
+AutoDeleteAddedTorrentFile=Never
+
+[LegalNotice]
+Accepted=true
+
+[Meta]
+MigrationVersion=6
+
+[Network]
+PortForwardingEnabled=true
+Proxy\HostnameLookupEnabled=false
+Proxy\Profiles\BitTorrent=true
+Proxy\Profiles\Misc=true
+Proxy\Profiles\RSS=true
+
+[Preferences]
+Connection\PortRangeMin=6881
+Connection\ResolvePeerCountries=false
+Connection\UPnP=false
+Downloads\SavePath=/downloads/
+Downloads\TempPath=/downloads/incomplete/
+General\Locale=en
+MailNotification\req_auth=true
+WebUI\Address=*
+WebUI\AuthSubnetWhitelist=0.0.0.0/0
+WebUI\AuthSubnetWhitelistEnabled=true
+WebUI\HostHeaderValidation=false
+WebUI\LocalHostAuth=false
+WebUI\ServerDomains=*
+
+[RSS]
+AutoDownloader\DownloadRepacks=true
+AutoDownloader\SmartEpisodeFilter=s(\\d+)e(\\d+), (\\d+)x(\\d+), "(\\d{4}[.\\-]\\d{1,2}[.\\-]\\d{1,2})", "(\\d{1,2}[.\\-]\\d{1,2}[.\\-]\\d{4})"

deployment/docker/docker-compose.yaml

@@ -1,139 +1,244 @@
+version: "3.9"
+
 name: knightcrawler
 
-x-restart: &restart-policy "unless-stopped"
+networks:
+  knightcrawler-network:
+    name: knightcrawler-network
+    driver: bridge
 
-x-basehealth: &base-health
-  interval: 10s
-  timeout: 10s
-  retries: 3
-  start_period: 10s
-
-x-rabbithealth: &rabbitmq-health
-  test: rabbitmq-diagnostics -q ping
-  <<: *base-health
-
-x-mongohealth: &mongodb-health
-  test: ["CMD", "mongosh", "--eval", "db.adminCommand('ping')"]
-  <<: *base-health
-
-x-postgreshealth: &postgresdb-health
-  test: pg_isready
-  <<: *base-health
-
-x-apps: &knightcrawler-app
-  depends_on:
-    mongodb:
-      condition: service_healthy
-    postgres:
-      condition: service_healthy
-    rabbitmq:
-      condition: service_healthy
-  restart: *restart-policy
+volumes:
+  postgres:
+  lavinmq:
+  redis:
 
 services:
+  ## Postgres is the database that is used by the services.
+  ## All downloaded metadata is stored in this database.
   postgres:
+    env_file: stack.env
+    healthcheck:
+      test: [ "CMD", "sh", "-c", "pg_isready -h localhost -U $$POSTGRES_USER" ]
+      timeout: 10s
+      interval: 10s
+      retries: 3
+      start_period: 10s
     image: postgres:latest
-    env_file: .env
-    environment:
-      PGUSER: postgres # needed for healthcheck.
     # # If you need the database to be accessible from outside, please open the below port.
-    # # Furthermore, please, please, please, change the username and password in the .env file.
+    # # Furthermore, please, please, please, change the username and password in the stack.env file.
     # # If you want to enhance your security even more, create a new user for the database with a strong password.
     # ports:
     #   - "5432:5432"
+    networks:
+      - knightcrawler-network
+    restart: unless-stopped
     volumes:
       - postgres:/var/lib/postgresql/data
-    healthcheck: *postgresdb-health
-    restart: *restart-policy
-    networks:
-      - knightcrawler-network
 
-  mongodb:
-    image: mongo:latest
-    env_file: .env
-    environment:
-      MONGO_INITDB_ROOT_USERNAME: ${MONGODB_USER:?Variable MONGODB_USER not set}
-      MONGO_INITDB_ROOT_PASSWORD: ${MONGODB_PASSWORD:?Variable MONGODB_PASSWORD not set}
-    # # If you need the database to be accessible from outside, please open the below port.
-    # # Furthermore, please, please, please, change the username and password in the .env file.
+  ## Redis is used as a cache for the services.
+  ## It is used to store the infohashes that are currently being processed in sagas, as well as intrim data.
+  redis:
+    env_file: stack.env
+    healthcheck:
+      test: ["CMD-SHELL", "redis-cli ping"]
+      timeout: 10s
+      interval: 10s
+      retries: 3
+      start_period: 10s
+    image: redis/redis-stack:latest
+    # # If you need redis to be accessible from outside, please open the below port.
     # ports:
-    #   - "27017:27017"
-    volumes:
-      - mongo:/data/db
-    restart: *restart-policy
-    healthcheck: *mongodb-health
+    #   - "6379:6379"
     networks:
       - knightcrawler-network
+    restart: unless-stopped
+    volumes:
+      - redis:/data
 
-  rabbitmq:
-    image: rabbitmq:3-management
+  ## LavinMQ is used as a message broker for the services.
+  ## It is a high performance drop in replacement for RabbitMQ.
+  ## It is used to communicate between the services.
+  lavinmq:
+    env_file: stack.env
     # # If you need the database to be accessible from outside, please open the below port.
-    # # Furthermore, please, please, please, look at the documentation for rabbit on how to secure the service.
+    # # Furthermore, please, please, please, look at the documentation for lavinmq / rabbitmq on how to secure the service.
     # ports:
     #   - "5672:5672"
     #   - "15672:15672"
     #   - "15692:15692"
+    image: cloudamqp/lavinmq:latest
+    healthcheck:
+      test: ["CMD-SHELL", "lavinmqctl status"]
+      timeout: 10s
+      interval: 10s
+      retries: 3
+      start_period: 10s
+    restart: unless-stopped
+    networks:
+      - knightcrawler-network
     volumes:
-      - rabbitmq:/var/lib/rabbitmq
-    hostname: ${RABBITMQ_HOST}
-    restart: *restart-policy
-    healthcheck: *rabbitmq-health
-    networks:
-      - knightcrawler-network
-
-  producer:
-    image: gabisonfire/knightcrawler-producer:latest
-    labels:
-      logging: "promtail"
-    env_file: .env
-    <<: *knightcrawler-app
-    networks:
-      - knightcrawler-network
-
-  consumer:
-    image: gabisonfire/knightcrawler-consumer:latest
-    env_file: .env
-    labels:
-      logging: "promtail"
-    deploy:
-      replicas: ${CONSUMER_REPLICAS}
-    <<: *knightcrawler-app
-    networks:
-      - knightcrawler-network
-
-  metadata:
-    image: gabisonfire/knightcrawler-metadata:latest
-    env_file: .env
-    labels:
-      logging: "promtail"
-    restart: no
-    networks:
-      - knightcrawler-network
+      - lavinmq:/var/lib/lavinmq/
 
+  ## The addon. This is what is used in stremio
   addon:
-    <<: *knightcrawler-app
-    env_file: .env
+    depends_on:
+      metadata:
+        condition: service_completed_successfully
+      migrator:
+        condition: service_completed_successfully
+      postgres:
+        condition: service_healthy
+      lavinmq:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    env_file: stack.env
     hostname: knightcrawler-addon
-    image: gabisonfire/knightcrawler-addon:latest
+    image: gabisonfire/knightcrawler-addon:2.0.17
     labels:
-      logging: "promtail"
+      logging: promtail
     networks:
       - knightcrawler-network
-      # - caddy
     ports:
       - "7000:7000"
+    restart: unless-stopped
 
+  ## The consumer is responsible for consuming infohashes and orchestrating download of metadata.
+  consumer:
+    depends_on:
+      metadata:
+        condition: service_completed_successfully
+      migrator:
+        condition: service_completed_successfully
+      postgres:
+        condition: service_healthy
+      lavinmq:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    env_file: stack.env
+    image: gabisonfire/knightcrawler-consumer:2.0.17
+    labels:
+      logging: promtail
+    networks:
+      - knightcrawler-network
+    restart: unless-stopped
 
-networks:
-  knightcrawler-network:
-    driver: bridge
-    name: knightcrawler-network
+  ## The debrid collector is responsible for downloading metadata from debrid services. (Currently only RealDebrid is supported)
+  debridcollector:
+    depends_on:
+      metadata:
+        condition: service_completed_successfully
+      migrator:
+        condition: service_completed_successfully
+      postgres:
+        condition: service_healthy
+      lavinmq:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    env_file: stack.env
+    image: gabisonfire/knightcrawler-debrid-collector:2.0.17
+    labels:
+      logging: promtail
+    networks:
+      - knightcrawler-network
+    restart: unless-stopped
 
-  # caddy:
-  #   name: caddy
-  #   external: true
+  ## The metadata service is responsible for downloading imdb publically available datasets.
+  ## This is used to enrich the metadata during production of ingested infohashes.
+  metadata:
+    depends_on:
+      migrator:
+        condition: service_completed_successfully
+    env_file: stack.env
+    image: gabisonfire/knightcrawler-metadata:2.0.17
+    networks:
+      - knightcrawler-network
+    restart: "no"
 
-volumes:
-  postgres:
-  mongo:
-  rabbitmq:
+  ## The migrator is responsible for migrating the database schema.
+  migrator:
+    depends_on:
+      postgres:
+        condition: service_healthy
+    env_file: stack.env
+    image: gabisonfire/knightcrawler-migrator:2.0.17
+    networks:
+      - knightcrawler-network
+    restart: "no"
+
+  ## The producer is responsible for producing infohashes by acquiring for various sites, including DMM.
+  producer:
+    depends_on:
+      metadata:
+        condition: service_completed_successfully
+      migrator:
+        condition: service_completed_successfully
+      postgres:
+        condition: service_healthy
+      lavinmq:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    env_file: stack.env
+    image: gabisonfire/knightcrawler-producer:2.0.17
+    labels:
+      logging: promtail
+    networks:
+      - knightcrawler-network
+    restart: unless-stopped
+
+  ## QBit collector utilizes QBitTorrent to download metadata.
+  qbitcollector:
+    depends_on:
+      metadata:
+        condition: service_completed_successfully
+      migrator:
+        condition: service_completed_successfully
+      postgres:
+        condition: service_healthy
+      lavinmq:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      qbittorrent:
+        condition: service_healthy
+    deploy:
+      replicas: ${QBIT_REPLICAS:-0}
+    env_file: stack.env
+    image: gabisonfire/knightcrawler-qbit-collector:2.0.17
+    labels:
+      logging: promtail
+    networks:
+      - knightcrawler-network
+    restart: unless-stopped
+
+  ## QBitTorrent is a torrent client that can be used to download torrents. In this case its used to download metadata.
+  ## The QBit collector requires this.
+  qbittorrent:
+    deploy:
+      replicas: ${QBIT_REPLICAS:-0}
+    env_file: stack.env
+    environment:
+      PGID: "1000"
+      PUID: "1000"
+      TORRENTING_PORT: "6881"
+      WEBUI_PORT: "8080"
+    healthcheck:
+      test: ["CMD-SHELL", "curl --fail http://localhost:8080"]
+      timeout: 10s
+      interval: 10s
+      retries: 3
+      start_period: 10s
+    image: lscr.io/linuxserver/qbittorrent:latest
+    networks:
+      - knightcrawler-network
+    ports:
+      - "6881:6881/tcp"
+      - "6881:6881/udp"
+    # if you want to expose the webui, uncomment the following line
+      # - "8001:8080"
+    restart: unless-stopped
+    volumes:
+      - ./config/qbit/qbittorrent.conf:/config/qBittorrent/qBittorrent.conf

deployment/docker/optional-services/caddy/configs/Caddyfile

@@ -4,7 +4,7 @@
 	## ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 	## Once you have confirmed Caddy works you should comment out
 	## the below line:
-	acme_ca https://acme-staging-v02.api.letsencrypt.org/director
+	acme_ca https://acme-staging-v02.api.letsencrypt.org/directory
 }
 
 (security-headers) {

deployment/docker/optional-services/grafana-metrics/config/prometheus/config.yml

@@ -16,7 +16,7 @@ rule_files:
 scrape_configs:
   - job_name: "rabbitmq"
     static_configs:
-      - targets: ["rabbitmq:15692"]
+      - targets: ["lavinmq:15692"]
   - job_name: "postgres-exporter"
     static_configs:
       - targets: ["postgres-exporter:9187"]

deployment/docker/src/components/infrastructure.yaml

@@ -0,0 +1,87 @@
+x-basehealth: &base-health
+  interval: 10s
+  timeout: 10s
+  retries: 3
+  start_period: 10s
+
+x-lavinhealth: &lavinmq-health
+  test: [ "CMD-SHELL", "lavinmqctl status" ]
+  <<: *base-health
+
+x-redishealth: &redis-health
+  test: redis-cli ping
+  <<: *base-health
+
+x-postgreshealth: &postgresdb-health
+  test: [ "CMD", "sh", "-c", "pg_isready -h localhost -U $$POSTGRES_USER" ]
+  <<: *base-health
+
+x-qbit: &qbit-health
+  test: "curl --fail http://localhost:8080"
+  <<: *base-health
+
+services:
+
+  postgres:
+    image: postgres:latest
+    environment:
+      PGUSER: postgres # needed for healthcheck.
+    # # If you need the database to be accessible from outside, please open the below port.
+    # # Furthermore, please, please, please, change the username and password in the .env file.
+    # # If you want to enhance your security even more, create a new user for the database with a strong password.
+    # ports:
+    #   - "5432:5432"
+    volumes:
+      - postgres:/var/lib/postgresql/data
+    healthcheck: *postgresdb-health
+    restart: unless-stopped
+    env_file: ../../.env
+    networks:
+      - knightcrawler-network
+
+  redis:
+    image: redis/redis-stack:latest
+    # # If you need redis to be accessible from outside, please open the below port.
+    # ports:
+    #   - "6379:6379"
+    volumes:
+      - redis:/data
+    restart: unless-stopped
+    healthcheck: *redis-health
+    env_file: ../../.env
+    networks:
+      - knightcrawler-network
+
+  lavinmq:
+    env_file: stack.env
+    # # If you need the database to be accessible from outside, please open the below port.
+    # # Furthermore, please, please, please, look at the documentation for lavinmq / rabbitmq on how to secure the service.
+    # ports:
+    #   - "5672:5672"
+    #   - "15672:15672"
+    #   - "15692:15692"
+    image: cloudamqp/lavinmq:latest
+    healthcheck: *lavinmq-health
+    restart: unless-stopped
+    volumes:
+      - lavinmq:/var/lib/lavinmq/
+
+  ## QBitTorrent is a torrent client that can be used to download torrents. In this case its used to download metadata.
+  ## The QBit collector requires this.
+  qbittorrent:
+    image: lscr.io/linuxserver/qbittorrent:latest
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - WEBUI_PORT=8080
+      - TORRENTING_PORT=6881
+    ports:
+      - 6881:6881
+      - 6881:6881/udp
+    env_file: ../../.env
+    networks:
+      - knightcrawler-network
+    restart: unless-stopped
+    healthcheck: *qbit-health
+    volumes:
+      - ../../config/qbit/qbittorrent.conf:/config/qBittorrent/qBittorrent.conf

deployment/docker/src/components/knightcrawler.yaml

@@ -0,0 +1,71 @@
+x-apps: &knightcrawler-app
+  labels:
+    logging: "promtail"
+  env_file: ../../.env
+  networks:
+    - knightcrawler-network
+
+x-depends: &knightcrawler-app-depends
+  depends_on:
+    redis:
+      condition: service_healthy
+    postgres:
+      condition: service_healthy
+    lavinmq:
+      condition: service_healthy
+    migrator:
+      condition: service_completed_successfully
+    metadata:
+      condition: service_completed_successfully
+
+services:
+  metadata:
+    image: gabisonfire/knightcrawler-metadata:2.0.17
+    env_file: ../../.env
+    networks:
+      - knightcrawler-network
+    restart: no
+    depends_on:
+      migrator:
+        condition: service_completed_successfully
+
+  migrator:
+    image: gabisonfire/knightcrawler-migrator:2.0.17
+    env_file: ../../.env
+    networks:
+      - knightcrawler-network
+    restart: no
+    depends_on: 
+      postgres:
+        condition: service_healthy
+
+  addon:
+    image: gabisonfire/knightcrawler-addon:2.0.17
+    <<: [*knightcrawler-app, *knightcrawler-app-depends]
+    restart: unless-stopped
+    hostname: knightcrawler-addon
+    ports:
+      - "7000:7000"
+
+  consumer:
+    image: gabisonfire/knightcrawler-consumer:2.0.17
+    <<: [*knightcrawler-app, *knightcrawler-app-depends]
+    restart: unless-stopped
+
+  debridcollector:
+    image: gabisonfire/knightcrawler-debrid-collector:2.0.17
+    <<: [*knightcrawler-app, *knightcrawler-app-depends]
+    restart: unless-stopped
+        
+  producer:
+    image: gabisonfire/knightcrawler-producer:2.0.17
+    <<: [*knightcrawler-app, *knightcrawler-app-depends]
+    restart: unless-stopped
+
+  qbitcollector:
+    image: gabisonfire/knightcrawler-qbit-collector:2.0.17
+    <<: [*knightcrawler-app, *knightcrawler-app-depends]
+    restart: unless-stopped
+    depends_on:
+      qbittorrent:
+        condition: service_healthy

deployment/docker/src/components/network.yaml

@@ -0,0 +1,4 @@
+networks:
+  knightcrawler-network:
+    driver: bridge
+    name: knightcrawler-network

deployment/docker/src/components/volumes.yaml

@@ -0,0 +1,4 @@
+volumes:
+  postgres:
+  redis:
+  lavinmq:

deployment/docker/src/compose.override.yaml

@@ -0,0 +1,7 @@
+services:
+  qbittorrent:
+    deploy:
+      replicas: 0
+  qbitcollector:
+    deploy:
+      replicas: 0

deployment/docker/src/compose.yaml

@@ -0,0 +1,7 @@
+version: "3.9"
+name: "knightcrawler"
+include:
+  - ./components/network.yaml
+  - ./components/volumes.yaml
+  - ./components/infrastructure.yaml
+  - ./components/knightcrawler.yaml

deployment/docker/stack.env

@@ -0,0 +1,43 @@
+# General environment variables
+TZ=London/Europe
+
+# PostgreSQL
+POSTGRES_HOST=postgres
+POSTGRES_PORT=5432
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=postgres
+POSTGRES_DB=knightcrawler
+
+# Redis
+REDIS_HOST=redis
+REDIS_PORT=6379
+REDIS_EXTRA=abortConnect=false,allowAdmin=true
+
+# AMQP
+RABBITMQ_HOST=lavinmq
+RABBITMQ_USER=guest
+RABBITMQ_PASSWORD=guest
+RABBITMQ_CONSUMER_QUEUE_NAME=ingested
+RABBITMQ_DURABLE=true
+RABBITMQ_MAX_QUEUE_SIZE=0
+RABBITMQ_MAX_PUBLISH_BATCH_SIZE=500
+RABBITMQ_PUBLISH_INTERVAL_IN_SECONDS=10
+
+# Metadata
+METADATA_INSERT_BATCH_SIZE=50000
+
+# Collectors
+COLLECTOR_QBIT_ENABLED=false
+COLLECTOR_DEBRID_ENABLED=true
+COLLECTOR_REAL_DEBRID_API_KEY=
+QBIT_HOST=http://qbittorrent:8080
+QBIT_TRACKERS_URL=https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_http.txt
+
+# Number of replicas for the qBittorrent collector and qBitTorrent client. Should be 0 or 1.
+QBIT_REPLICAS=0
+
+# Addon
+DEBUG_MODE=false
+
+# Producer
+GITHUB_PAT=

deployment/k8s/Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v2
+appVersion: 2.0.17
+description: A helm chart for Knightcrawler
+name: knightcrawler
+type: application
+version: 0.1.0

deployment/k8s/templates/NOTES.txt

@@ -0,0 +1,6 @@
+
+Congratulations,
+
+Knightcrawler is now deployed. This may take a while to be up and responding.
+
+

deployment/k8s/templates/config/knightcrawler.configmap.yaml

@@ -0,0 +1,27 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: '{{ .Release.Name }}-config'
+  labels:
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+data:
+  COLLECTOR_DEBRID_ENABLED: '{{ .Values.knightcrawler.debridcollector.enabled }}'
+  COLLECTOR_QBIT_ENABLED: '{{ .Values.knightcrawler.qbitcollector.enabled }}'
+  DEBUG_MODE: '{{ .Values.knightcrawler.debug }}'
+  METADATA_INSERT_BATCH_SIZE: '{{ .Values.environment.metadata.insertBatchSize }}'
+  POSTGRES_DB: '{{ .Values.environment.postgres.dbName }}'
+  POSTGRES_HOST: '{{ if .Values.environment.postgres.external }}{{ .Values.environment.postgres.host }}{{ else }}{{ .Release.Name }}-postgres{{ end }}'
+  POSTGRES_PORT: '{{ .Values.environment.postgres.port }}'
+  QBIT_HOST: '{{ .Values.environment.qbitcollector.qbitHost }}'
+  QBIT_TRACKERS_URL: '{{ .Values.environment.qbitcollector.trackersUrl }}'
+  RABBITMQ_CONSUMER_QUEUE_NAME: '{{ .Values.environment.producer.queueName }}'
+  RABBITMQ_DURABLE: '{{ .Values.environment.producer.durable }}'
+  RABBITMQ_HOST: '{{ if .Values.environment.lavinmq.external }}{{ .Values.environment.lavinmq.host }}{{ else }}{{ .Release.Name }}-lavinmq{{ end }}'
+  RABBITMQ_MAX_PUBLISH_BATCH_SIZE: '{{ .Values.environment.producer.maxPublishBatchSize }}'
+  RABBITMQ_MAX_QUEUE_SIZE: '{{ .Values.environment.producer.maxQueueSize }}'
+  RABBITMQ_PUBLISH_INTERVAL_IN_SECONDS: '{{ .Values.environment.producer.publishIntervalSeconds }}'
+  REDIS_EXTRA: '{{ .Values.environment.redis.extra }}'
+  REDIS_HOST: '{{ if .Values.environment.redis.external }}{{ .Values.environment.redis.host }}{{ else }}{{ .Release.Name }}-redis{{ end }}'
+  REDIS_PORT: '{{ .Values.environment.redis.port }}'
+  TZ: '{{ .Values.shared.timezone }}'

deployment/k8s/templates/config/knightcrawler.secrets.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: '{{ .Release.Name }}-secrets'
+  labels:
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+type: Opaque
+data:
+  GITHUB_PAT: '{{ .Values.environment.producer.githubPat | b64enc }}'
+  COLLECTOR_REAL_DEBRID_API_KEY: '{{ .Values.environment.debridcollector.realDebridApiKey | b64enc }}'
+  POSTGRES_USER: '{{ .Values.environment.postgres.user | b64enc }}'
+  POSTGRES_PASSWORD: '{{ .Values.environment.postgres.password | b64enc }}'
+  RABBITMQ_PASSWORD: '{{ .Values.environment.lavinmq.password | b64enc }}'
+  RABBITMQ_USER: '{{ .Values.environment.lavinmq.user | b64enc }}'

deployment/k8s/templates/infrastructure/lavinmq/lavinmq.service.yaml

@@ -0,0 +1,25 @@
+{{ if .Values.infrastructure.lavinmq.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: '{{ .Release.Name }}-lavinmq'
+  labels:
+    component: lavinmq
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+spec:
+  selector:
+    component: lavinmq
+    release: '{{ .Release.Name }}'
+  type: ClusterIP
+  ports:
+    - protocol: TCP
+      port: 5672
+      targetPort: 5672
+    - protocol: TCP
+      port: 15672
+      targetPort: 15672
+    - protocol: TCP
+      port: 15692
+      targetPort: 15692
+{{- end -}}

deployment/k8s/templates/infrastructure/lavinmq/lavinmq.statefulset.yaml

@@ -0,0 +1,60 @@
+{{ if .Values.infrastructure.lavinmq.enabled }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: '{{ .Release.Name }}-lavinmq'
+  labels:
+    component: lavinmq
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "0"
+spec:
+  serviceName: '{{ .Release.Name }}-lavinmq'
+  replicas: 1
+  selector:
+    matchLabels:
+      component: lavinmq
+      release: '{{ .Release.Name }}'
+  template:
+    metadata:
+      labels:
+        component: lavinmq
+        release: '{{ .Release.Name }}'
+    spec:
+      containers:
+        - name: lavinmq
+          image: '{{ .Values.infrastructure.lavinmq.image }}:{{ .Values.infrastructure.lavinmq.tag }}'
+          ports:
+            - name: lavinmq
+              containerPort: 5672
+            - name: lavinmq-15672
+              containerPort: 15672
+            - name: lavinmq-15692
+              containerPort: 15692
+          envFrom:
+            - configMapRef:
+                name: '{{ .Release.Name }}-config'
+            - secretRef:
+                name: '{{ .Release.Name }}-secrets'
+          volumeMounts:
+            - mountPath: /var/lib/lavinmq
+              name: lavinmq
+          livenessProbe:
+            exec:
+              command:
+                - lavinmqctl status
+            periodSeconds: 10
+            initialDelaySeconds: 10
+            successThreshold: 1
+            failureThreshold: 3
+  volumeClaimTemplates:
+    - metadata:
+        name: lavinmq
+      spec:
+        accessModes: [ "ReadWriteOnce" ]
+        resources:
+          requests:
+            storage: '{{ .Values.persistence.lavinmq.capacity }}'
+{{- end -}}

deployment/k8s/templates/infrastructure/postgres/postgres.service.yaml

@@ -0,0 +1,19 @@
+{{ if .Values.infrastructure.postgres.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: '{{ .Release.Name }}-postgres'
+  labels:
+    component: postgres
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+spec:
+  selector:
+    component: postgres
+    release: '{{ .Release.Name }}'
+  type: ClusterIP
+  ports:
+    - protocol: TCP
+      port: 5432
+      targetPort: 5432
+{{- end -}}

deployment/k8s/templates/infrastructure/postgres/postgres.statefulset.yaml

@@ -0,0 +1,58 @@
+{{ if .Values.infrastructure.postgres.enabled }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: '{{ .Release.Name }}-postgres'
+  labels:
+    component: postgres
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "0"
+spec:
+  serviceName: '{{ .Release.Name }}-postgres'
+  replicas: 1
+  selector:
+    matchLabels:
+      component: postgres
+      release: '{{ .Release.Name }}'
+  template:
+    metadata:
+      labels:
+        component: postgres
+        release: '{{ .Release.Name }}'
+    spec:
+      containers:
+        - name: postgres
+          image: '{{ .Values.infrastructure.postgres.image }}:{{ .Values.infrastructure.postgres.tag }}'
+          ports:
+            - name: postgres
+              containerPort: 5432
+          envFrom:
+            - configMapRef:
+                name: '{{ .Release.Name }}-config'
+            - secretRef:
+                name: '{{ .Release.Name }}-secrets'
+          volumeMounts:
+            - mountPath: /var/lib/postgresql/data
+              name: postgres
+          livenessProbe:
+            exec:
+              command:
+                - sh
+                - -c
+                - pg_isready -h localhost -U $POSTGRES_USER
+            periodSeconds: 10
+            initialDelaySeconds: 10
+            successThreshold: 1
+            failureThreshold: 3
+  volumeClaimTemplates:
+    - metadata:
+        name: postgres
+      spec:
+        accessModes: [ "ReadWriteOnce" ]
+        resources:
+          requests:
+            storage: '{{ .Values.persistence.postgres.capacity }}'
+{{- end -}}

deployment/k8s/templates/infrastructure/qbittorrent/qbittorrent.deployment.yaml

@@ -0,0 +1,57 @@
+{{ if .Values.knightcrawler.qbitcollector.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: '{{ .Release.Name }}-qbittorrent'
+  labels:
+    component: qbittorrent
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "0"
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      component: qbittorrent
+      release: '{{ .Release.Name }}'
+  template:
+    metadata:
+      labels:
+        component: qbittorrent
+        release: '{{ .Release.Name }}'
+    spec:
+      containers:
+        - name: qbittorrent
+          image: '{{ .Values.infrastructure.qbittorrent.image }}:{{ .Values.infrastructure.qbittorrent.tag }}'
+          ports:
+            - name: qbittorrent
+              containerPort: 6881
+            - name: qbittorrent-6881
+              containerPort: 6881
+            - name: qbittorrent-8080
+              containerPort: 8080
+          env:
+            - name: PUID
+              value: '{{ .Values.environment.qbittorrent.puid }}'
+            - name: PGID
+              value: '{{ .Values.environment.qbittorrent.pgid }}'
+            - name: TORRENTING_PORT
+              value: '{{ .Values.environment.qbittorrent.torrentingPort }}'
+            - name: WEBUI_PORT
+              value: '{{ .Values.environment.qbittorrent.webuiPort }}'
+          envFrom:
+            - configMapRef:
+                name: '{{ .Release.Name }}-config'
+            - secretRef:
+                name: '{{ .Release.Name }}-secrets'
+          livenessProbe:
+            exec:
+              command:
+                - curl --fail http://localhost:8080
+            periodSeconds: 10
+            initialDelaySeconds: 10
+            successThreshold: 1
+            failureThreshold: 3
+{{- end -}}

deployment/k8s/templates/infrastructure/qbittorrent/qbittorrent.service.yaml

@@ -0,0 +1,25 @@
+{{ if .Values.knightcrawler.qbitcollector.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: '{{ .Release.Name }}-qbittorrent'
+  labels:
+    component: qbittorrent
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+spec:
+  selector:
+    component: qbittorrent
+    release: '{{ .Release.Name }}'
+  type: ClusterIP
+  ports:
+    - protocol: TCP
+      port: 6881
+      targetPort: 6881
+    - protocol: TCP
+      port: 6881
+      targetPort: 6881
+    - protocol: TCP
+      port: 8080
+      targetPort: 8080
+{{- end -}}

deployment/k8s/templates/infrastructure/redis/redis.service.yaml

@@ -0,0 +1,19 @@
+{{ if .Values.infrastructure.redis.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: '{{ .Release.Name }}-redis'
+  labels:
+    component: redis
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+spec:
+  selector:
+    component: redis
+    release: '{{ .Release.Name }}'
+  type: ClusterIP
+  ports:
+    - protocol: TCP
+      port: 6379
+      targetPort: 6379
+{{- end -}}

deployment/k8s/templates/infrastructure/redis/redis.statefulset.yaml

@@ -0,0 +1,56 @@
+{{ if .Values.infrastructure.redis.enabled }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: '{{ .Release.Name }}-redis'
+  labels:
+    component: redis
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "0"
+spec:
+  serviceName: '{{ .Release.Name }}-redis'
+  replicas: 1
+  selector:
+    matchLabels:
+      component: redis
+      release: '{{ .Release.Name }}'
+  template:
+    metadata:
+      labels:
+        component: redis
+        release: '{{ .Release.Name }}'
+    spec:
+      containers:
+        - name: redis
+          image: '{{ .Values.infrastructure.redis.image }}:{{ .Values.infrastructure.redis.tag }}'
+          ports:
+            - name: redis
+              containerPort: 6379
+          envFrom:
+            - configMapRef:
+                name: '{{ .Release.Name }}-config'
+            - secretRef:
+                name: '{{ .Release.Name }}-secrets'
+          volumeMounts:
+            - mountPath: /data
+              name: redis
+          livenessProbe:
+            exec:
+              command:
+                - redis-cli ping
+            periodSeconds: 10
+            initialDelaySeconds: 10
+            successThreshold: 1
+            failureThreshold: 3
+  volumeClaimTemplates:
+    - metadata:
+        name: redis
+      spec:
+        accessModes: [ "ReadWriteOnce" ]
+        resources:
+          requests:
+            storage: '{{ .Values.persistence.redis.capacity }}'
+{{- end -}}

deployment/k8s/templates/jobs/metadata.job.yaml

@@ -0,0 +1,28 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: '{{ .Release.Name }}-metadata'
+  labels:
+    component: metadata
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "2"
+    "helm.sh/hook-delete-policy": hook-succeeded
+spec:
+  template:
+    metadata:
+      labels:
+        component: metadata
+        release: '{{ .Release.Name }}'
+    spec:
+      restartPolicy: OnFailure
+      containers:
+        - name: metadata
+          image: '{{ .Values.knightcrawler.metadata.image }}{{ if ne .Values.knightcrawler.globalImageTagOverride "" }}:{{ .Values.knightcrawler.globalImageTagOverride }}{{else}}:{{ .Values.knightcrawler.metadata.tag}}{{ end }}'
+          envFrom:
+            - configMapRef:
+                name: '{{ .Release.Name }}-config'
+            - secretRef:
+                name: '{{ .Release.Name }}-secrets'

deployment/k8s/templates/jobs/migrator.job.yaml

@@ -0,0 +1,28 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: '{{ .Release.Name }}-migrator'
+  labels:
+    component: migrator
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "1"
+    "helm.sh/hook-delete-policy": hook-succeeded
+spec:
+  template:
+    metadata:
+      labels:
+        component: migrator
+        release: '{{ .Release.Name }}'
+    spec:
+      restartPolicy: OnFailure
+      containers:
+        - name: migrator
+          image: '{{ .Values.knightcrawler.migrator.image }}{{ if ne .Values.knightcrawler.globalImageTagOverride "" }}:{{ .Values.knightcrawler.globalImageTagOverride }}{{else}}:{{ .Values.knightcrawler.migrator.tag}}{{ end }}'
+          envFrom:
+            - configMapRef:
+                name: '{{ .Release.Name }}-config'
+            - secretRef:
+                name: '{{ .Release.Name }}-secrets'

deployment/k8s/templates/kc/deployments/addon.deployment.yaml

@@ -0,0 +1,35 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: '{{ .Release.Name }}-addon'
+  labels:
+    component: addon
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "4"
+spec:
+  replicas: {{ .Values.knightcrawler.addon.replicas }}
+  selector:
+    matchLabels:
+      component: addon
+      release: '{{ .Release.Name }}'
+  template:
+    metadata:
+      labels:
+        component: addon
+        release: '{{ .Release.Name }}'
+    spec:
+      containers:
+        - name: addon
+          image: '{{ .Values.knightcrawler.addon.image }}{{ if ne .Values.knightcrawler.globalImageTagOverride "" }}:{{ .Values.knightcrawler.globalImageTagOverride }}{{else}}:{{ .Values.knightcrawler.addon.tag}}{{ end }}'
+          ports:
+            - name: addon
+              containerPort: 7000
+          envFrom:
+            - configMapRef:
+                name: '{{ .Release.Name }}-config'
+            - secretRef:
+                name: '{{ .Release.Name }}-secrets'
+

deployment/k8s/templates/kc/deployments/consumer.deployment.yaml

@@ -0,0 +1,32 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: '{{ .Release.Name }}-consumer'
+  labels:
+    component: consumer
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "4"
+spec:
+  replicas: {{ .Values.knightcrawler.consumer.replicas }}
+  selector:
+    matchLabels:
+      component: consumer
+      release: '{{ .Release.Name }}'
+  template:
+    metadata:
+      labels:
+        component: consumer
+        release: '{{ .Release.Name }}'
+    spec:
+      containers:
+        - name: consumer
+          image: '{{ .Values.knightcrawler.consumer.image }}{{ if ne .Values.knightcrawler.globalImageTagOverride "" }}:{{ .Values.knightcrawler.globalImageTagOverride }}{{else}}:{{ .Values.knightcrawler.consumer.tag}}{{ end }}'
+          envFrom:
+            - configMapRef:
+                name: '{{ .Release.Name }}-config'
+            - secretRef:
+                name: '{{ .Release.Name }}-secrets'
+

deployment/k8s/templates/kc/deployments/debridcollector.deployment.yaml

@@ -0,0 +1,31 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: '{{ .Release.Name }}-debridcollector'
+  labels:
+    component: debridcollector
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "4"
+spec:
+  replicas: {{ .Values.knightcrawler.debridcollector.replicas }}
+  selector:
+    matchLabels:
+      component: debridcollector
+      release: '{{ .Release.Name }}'
+  template:
+    metadata:
+      labels:
+        component: debridcollector
+        release: '{{ .Release.Name }}'
+    spec:
+      containers:
+        - name: debridcollector
+          image: '{{ .Values.knightcrawler.debridcollector.image }}{{ if ne .Values.knightcrawler.globalImageTagOverride "" }}:{{ .Values.knightcrawler.globalImageTagOverride }}{{else}}:{{ .Values.knightcrawler.debridcollector.tag}}{{ end }}'
+          envFrom:
+            - configMapRef:
+                name: '{{ .Release.Name }}-config'
+            - secretRef:
+                name: '{{ .Release.Name }}-secrets'

deployment/k8s/templates/kc/deployments/producer.deployment.yaml

@@ -0,0 +1,31 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: '{{ .Release.Name }}-producer'
+  labels:
+    component: producer
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "4"
+spec:
+  replicas: {{ .Values.knightcrawler.producer.replicas }}
+  selector:
+    matchLabels:
+      component: producer
+      release: '{{ .Release.Name }}'
+  template:
+    metadata:
+      labels:
+        component: producer
+        release: '{{ .Release.Name }}'
+    spec:
+      containers:
+        - name: producer
+          image: '{{ .Values.knightcrawler.producer.image }}{{ if ne .Values.knightcrawler.globalImageTagOverride "" }}:{{ .Values.knightcrawler.globalImageTagOverride }}{{else}}:{{ .Values.knightcrawler.producer.tag}}{{ end }}'
+          envFrom:
+            - configMapRef:
+                name: '{{ .Release.Name }}-config'
+            - secretRef:
+                name: '{{ .Release.Name }}-secrets'

deployment/k8s/templates/kc/deployments/qbitcollector.deployment.yaml

@@ -0,0 +1,33 @@
+{{ if .Values.knightcrawler.qbitcollector.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: '{{ .Release.Name }}-qbitcollector'
+  labels:
+    component: qbitcollector
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "4"
+spec:
+  replicas: {{ .Values.knightcrawler.qbitcollector.replicas }}
+  selector:
+    matchLabels:
+      component: qbitcollector
+      release: '{{ .Release.Name }}'
+  template:
+    metadata:
+      labels:
+        component: qbitcollector
+        release: '{{ .Release.Name }}'
+    spec:
+      containers:
+        - name: qbitcollector
+          image: '{{ .Values.knightcrawler.qbitcollector.image }}{{ if ne .Values.knightcrawler.globalImageTagOverride "" }}:{{ .Values.knightcrawler.globalImageTagOverride }}{{else}}:{{ .Values.knightcrawler.qbitcollector.tag}}{{ end }}'
+          envFrom:
+            - configMapRef:
+                name: '{{ .Release.Name }}-config'
+            - secretRef:
+                name: '{{ .Release.Name }}-secrets'
+{{- end -}}

deployment/k8s/templates/kc/services/addon.service.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: '{{ .Release.Name }}-addon'
+  labels:
+    component: addon
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+spec:
+  selector:
+    component: addon
+    release: '{{ .Release.Name }}'
+  type: ClusterIP
+  ports:
+    - protocol: TCP
+      port: 7000
+      targetPort: 7000

deployment/k8s/values.yaml

@@ -0,0 +1,100 @@
+knightcrawler:
+  debug: false
+  globalImageTagOverride: ""
+  addon:
+    image: "gabisonfire/knightcrawler-addon"
+    tag: "2.0.17"
+    replicas: 1
+  consumer:
+    image: "gabisonfire/knightcrawler-consumer"
+    tag: "2.0.17"
+    replicas: 1
+  metadata:
+    image: "gabisonfire/knightcrawler-metadata"
+    tag: "2.0.17"
+    replicas: 1
+  migrator:
+    image: "gabisonfire/knightcrawler-migrator"
+    tag: "2.0.17"
+    replicas: 1
+  debridcollector:
+    image: "gabisonfire/knightcrawler-debrid-collector"
+    tag: "2.0.17"
+    enabled: true
+    replicas: 1
+  qbitcollector:
+    image: "gabisonfire/knightcrawler-qbit-collector"
+    tag: "2.0.17"
+    enabled: false
+    replicas: 1
+  producer:
+    image: "gabisonfire/knightcrawler-producer"
+    tag: "2.0.17"
+    replicas: 1
+    
+infrastructure:
+  lavinmq:
+    image: "cloudamqp/lavinmq"
+    tag: "latest"
+    enabled: true
+  postgres:
+    image: "postgres"
+    tag: "latest"
+    enabled: true
+  redis:
+    image: "redis/redis-stack-server"
+    tag: "latest"
+    enabled: true
+  qbittorrent:
+    image: "lscr.io/linuxserver/qbittorrent"
+    tag: "latest"
+      
+environment:
+  redis:
+    external: false
+    host: ""
+    port: "6379"
+    extra: "abortConnect=false,allowAdmin=true"
+  postgres:
+    external: false
+    host: ""
+    port: "5432"
+    dbName: "knightcrawler"
+    user: "postgres"
+    password: "postgres"
+  lavinmq:
+    external: false
+    host: ""
+    user: "guest"
+    password: "guest"
+  qbitcollector:
+    qbitHost: "http://qbittorrent:8080"
+    trackersUrl: "https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_http.txt"
+  debridcollector:
+    realDebridApiKey: ""
+  producer:
+    githubPat: ""
+    queueName: "ingested"
+    durable: true
+    maxPublishBatchSize: 500
+    maxQueueSize: 0
+    publishIntervalSeconds: 10
+  metadata:
+    insertBatchSize: 50000
+  qbittorrent:
+    pgid: "1000"
+    puid: "1000"
+    torrentingPort: "6881"
+    webuiPort: "8080"
+      
+persistence:
+  storageClassName: ""
+  redis:
+    capacity: 1Gi
+  postgres:
+    capacity: 1Gi
+  lavinmq:
+    capacity: 1Gi
+    
+shared:
+  timezone: "London/Europe"

src/addon-jackett/package.json

@@ -14,7 +14,6 @@
     "axios": "^1.6.1",
     "bottleneck": "^2.19.5",
     "cache-manager": "^3.4.4",
-    "cache-manager-mongodb": "^0.3.0",
     "cors": "^2.8.5",
     "debrid-link-api": "^1.0.1",
     "express": "^4.18.2",
@@ -33,7 +32,11 @@
     "user-agents": "^1.0.1444",
     "video-name-parser": "^1.4.6",
     "xml-js": "^1.6.11",
-    "xml2js": "^0.6.2"
+    "xml2js": "^0.6.2",
+    "@redis/client": "^1.5.14",
+    "@redis/json": "^1.0.6",
+    "@redis/search": "^1.1.6",
+    "cache-manager-redis-store": "^2.0.0"
   },
   "devDependencies": {
     "@types/node": "^20.11.6",

src/addon-jackett/src/lib/cache.js

@@ -1,7 +1,7 @@
 import cacheManager from 'cache-manager';
-import mangodbStore from 'cache-manager-mongodb';
 import { isStaticUrl }  from '../moch/static.js';
 import {cacheConfig} from "./settings.js";
+import redisStore from 'cache-manager-redis-store';
 
 const STREAM_KEY_PREFIX = `${cacheConfig.GLOBAL_KEY_PREFIX}|stream`;
 const IMDB_KEY_PREFIX = `${cacheConfig.GLOBAL_KEY_PREFIX}|imdb`;
@@ -12,28 +12,20 @@ const memoryCache = initiateMemoryCache();
 const remoteCache = initiateRemoteCache();
 
 function initiateRemoteCache() {
-  if (cacheConfig.NO_CACHE) {
-    return null;
-  } else if (cacheConfig.MONGODB_URI) {
-    return cacheManager.caching({
-      store: mangodbStore,
-      uri: cacheConfig.MONGODB_URI,
-      options: {
-        collection: 'jackettio_addon_collection',
-        socketTimeoutMS: 120000,
-        useNewUrlParser: true,
-        useUnifiedTopology: false,
-        ttl: cacheConfig.STREAM_EMPTY_TTL
-      },
-      ttl: cacheConfig.STREAM_EMPTY_TTL,
-      ignoreCacheErrors: true
-    });
-  } else {
-    return cacheManager.caching({
-      store: 'memory',
-      ttl: cacheConfig.STREAM_EMPTY_TTL
-    });
-  }
+    if (cacheConfig.NO_CACHE) {
+        return null;
+    } else if (cacheConfig.REDIS_CONNECTION_STRING) {
+        return cacheManager.caching({
+            store: redisStore,
+            ttl: cacheConfig.STREAM_EMPTY_TTL,
+            url: cacheConfig.REDIS_CONNECTION_STRING
+        });
+    } else {
+        return cacheManager.caching({
+            store: 'memory',
+            ttl: cacheConfig.STREAM_EMPTY_TTL
+        });
+    }
 }
 
 function initiateMemoryCache() {

src/addon-jackett/src/lib/settings.js

@@ -25,7 +25,9 @@ export const cinemetaConfig = {
 }
 
 export const cacheConfig = {
-    MONGODB_URI: process.env.MONGODB_URI,
+    REDIS_HOST: process.env.REDIS_HOST || 'redis',
+    REDIS_PORT: process.env.REDIS_PORT || '6379',
+    REDIS_EXTRA: process.env.REDIS_EXTRA || '',
     NO_CACHE: parseBool(process.env.NO_CACHE, false),
     IMDB_TTL: parseInt(process.env.IMDB_TTL || 60 * 60 * 4), // 4 Hours
     STREAM_TTL: parseInt(process.env.STREAM_TTL || 60 * 60 * 4), // 1 Hour
@@ -40,3 +42,5 @@ export const cacheConfig = {
     STALE_ERROR_AGE: parseInt(process.env.STALE_ERROR_AGE) || 7 * 24 * 60 * 60, // 7 days
     GLOBAL_KEY_PREFIX: process.env.GLOBAL_KEY_PREFIX || 'jackettio-addon',
 }
+
+cacheConfig.REDIS_CONNECTION_STRING = 'redis://' + cacheConfig.REDIS_HOST + ':' + cacheConfig.REDIS_PORT + '?' + cacheConfig.REDIS_EXTRA;