helm chart [wip]

add qbit housekeeping service to remove stale torrents (#193 )
* Add housekeeping service to clean stale torrents * version bump
2024-03-30 23:54:34 +00:00 · 2024-03-30 11:52:23 +00:00 · 2024-03-30 04:47:36 +00:00 · 2024-03-30 01:54:06 +00:00 · 2024-03-30 01:43:21 +00:00 · 2024-03-30 01:26:02 +00:00
34 changed files with 857 additions and 34 deletions
--- a/deployment/docker/docker-compose.yaml
+++ b/deployment/docker/docker-compose.yaml
@@ -94,7 +94,7 @@ services:
        condition: service_healthy
    env_file: stack.env
    hostname: knightcrawler-addon
-    image: gabisonfire/knightcrawler-addon:2.0.12
+    image: gabisonfire/knightcrawler-addon:2.0.17
    labels:
      logging: promtail
    networks:
@@ -117,7 +117,7 @@ services:
      redis:
        condition: service_healthy
    env_file: stack.env
-    image: gabisonfire/knightcrawler-consumer:2.0.12
+    image: gabisonfire/knightcrawler-consumer:2.0.17
    labels:
      logging: promtail
    networks:
@@ -138,7 +138,7 @@ services:
      redis:
        condition: service_healthy
    env_file: stack.env
-    image: gabisonfire/knightcrawler-debrid-collector:2.0.12
+    image: gabisonfire/knightcrawler-debrid-collector:2.0.17
    labels:
      logging: promtail
    networks:
@@ -152,7 +152,7 @@ services:
      migrator:
        condition: service_completed_successfully
    env_file: stack.env
-    image: gabisonfire/knightcrawler-metadata:2.0.12
+    image: gabisonfire/knightcrawler-metadata:2.0.17
    networks:
      - knightcrawler-network
    restart: "no"
@@ -163,7 +163,7 @@ services:
      postgres:
        condition: service_healthy
    env_file: stack.env
-    image: gabisonfire/knightcrawler-migrator:2.0.12
+    image: gabisonfire/knightcrawler-migrator:2.0.17
    networks:
      - knightcrawler-network
    restart: "no"
@@ -182,7 +182,7 @@ services:
      redis:
        condition: service_healthy
    env_file: stack.env
-    image: gabisonfire/knightcrawler-producer:2.0.12
+    image: gabisonfire/knightcrawler-producer:2.0.17
    labels:
      logging: promtail
    networks:
@@ -207,7 +207,7 @@ services:
    deploy:
      replicas: ${QBIT_REPLICAS:-0}
    env_file: stack.env
-    image: gabisonfire/knightcrawler-qbit-collector:2.0.12
+    image: gabisonfire/knightcrawler-qbit-collector:2.0.17
    labels:
      logging: promtail
    networks:
--- a/deployment/docker/src/components/knightcrawler.yaml
+++ b/deployment/docker/src/components/knightcrawler.yaml
@@ -20,7 +20,7 @@ x-depends: &knightcrawler-app-depends

 services:
  metadata:
-    image: gabisonfire/knightcrawler-metadata:2.0.12
+    image: gabisonfire/knightcrawler-metadata:2.0.17
    env_file: ../../.env
    networks:
      - knightcrawler-network
@@ -30,7 +30,7 @@ services:
        condition: service_completed_successfully

  migrator:
-    image: gabisonfire/knightcrawler-migrator:2.0.12
+    image: gabisonfire/knightcrawler-migrator:2.0.17
    env_file: ../../.env
    networks:
      - knightcrawler-network
@@ -40,7 +40,7 @@ services:
        condition: service_healthy

  addon:
-    image: gabisonfire/knightcrawler-addon:2.0.12
+    image: gabisonfire/knightcrawler-addon:2.0.17
    <<: [*knightcrawler-app, *knightcrawler-app-depends]
    restart: unless-stopped
    hostname: knightcrawler-addon
@@ -48,22 +48,22 @@ services:
      - "7000:7000"

  consumer:
-    image: gabisonfire/knightcrawler-consumer:2.0.12
+    image: gabisonfire/knightcrawler-consumer:2.0.17
    <<: [*knightcrawler-app, *knightcrawler-app-depends]
    restart: unless-stopped

  debridcollector:
-    image: gabisonfire/knightcrawler-debrid-collector:2.0.12
+    image: gabisonfire/knightcrawler-debrid-collector:2.0.17
    <<: [*knightcrawler-app, *knightcrawler-app-depends]
    restart: unless-stopped
        
  producer:
-    image: gabisonfire/knightcrawler-producer:2.0.12
+    image: gabisonfire/knightcrawler-producer:2.0.17
    <<: [*knightcrawler-app, *knightcrawler-app-depends]
    restart: unless-stopped

  qbitcollector:
-    image: gabisonfire/knightcrawler-qbit-collector:2.0.12
+    image: gabisonfire/knightcrawler-qbit-collector:2.0.17
    <<: [*knightcrawler-app, *knightcrawler-app-depends]
    restart: unless-stopped
    depends_on:
--- a/deployment/k8s/Chart.yaml
+++ b/deployment/k8s/Chart.yaml
@@ -0,0 +1,6 @@
+apiVersion: v2
+appVersion: 2.0.17
+description: A helm chart for Knightcrawler
+name: knightcrawler
+type: application
+version: 0.1.0
--- a/deployment/k8s/templates/NOTES.txt
+++ b/deployment/k8s/templates/NOTES.txt
@@ -0,0 +1,6 @@
+
+Congratulations,
+
+Knightcrawler is now deployed. This may take a while to be up and responding.
+
+
--- a/deployment/k8s/templates/config/knightcrawler.configmap.yaml
+++ b/deployment/k8s/templates/config/knightcrawler.configmap.yaml
@@ -0,0 +1,27 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: '{{ .Release.Name }}-config'
+  labels:
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+data:
+  COLLECTOR_DEBRID_ENABLED: '{{ .Values.knightcrawler.debridcollector.enabled }}'
+  COLLECTOR_QBIT_ENABLED: '{{ .Values.knightcrawler.qbitcollector.enabled }}'
+  DEBUG_MODE: '{{ .Values.knightcrawler.debug }}'
+  METADATA_INSERT_BATCH_SIZE: '{{ .Values.environment.metadata.insertBatchSize }}'
+  POSTGRES_DB: '{{ .Values.environment.postgres.dbName }}'
+  POSTGRES_HOST: '{{ if .Values.environment.postgres.external }}{{ .Values.environment.postgres.host }}{{ else }}{{ .Release.Name }}-postgres{{ end }}'
+  POSTGRES_PORT: '{{ .Values.environment.postgres.port }}'
+  QBIT_HOST: '{{ .Values.environment.qbitcollector.qbitHost }}'
+  QBIT_TRACKERS_URL: '{{ .Values.environment.qbitcollector.trackersUrl }}'
+  RABBITMQ_CONSUMER_QUEUE_NAME: '{{ .Values.environment.producer.queueName }}'
+  RABBITMQ_DURABLE: '{{ .Values.environment.producer.durable }}'
+  RABBITMQ_HOST: '{{ if .Values.environment.lavinmq.external }}{{ .Values.environment.lavinmq.host }}{{ else }}{{ .Release.Name }}-lavinmq{{ end }}'
+  RABBITMQ_MAX_PUBLISH_BATCH_SIZE: '{{ .Values.environment.producer.maxPublishBatchSize }}'
+  RABBITMQ_MAX_QUEUE_SIZE: '{{ .Values.environment.producer.maxQueueSize }}'
+  RABBITMQ_PUBLISH_INTERVAL_IN_SECONDS: '{{ .Values.environment.producer.publishIntervalSeconds }}'
+  REDIS_EXTRA: '{{ .Values.environment.redis.extra }}'
+  REDIS_HOST: '{{ if .Values.environment.redis.external }}{{ .Values.environment.redis.host }}{{ else }}{{ .Release.Name }}-redis{{ end }}'
+  REDIS_PORT: '{{ .Values.environment.redis.port }}'
+  TZ: '{{ .Values.shared.timezone }}'
--- a/deployment/k8s/templates/config/knightcrawler.secrets.yaml
+++ b/deployment/k8s/templates/config/knightcrawler.secrets.yaml
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: '{{ .Release.Name }}-secrets'
+  labels:
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+type: Opaque
+data:
+  GITHUB_PAT: '{{ .Values.environment.producer.githubPat | b64enc }}'
+  COLLECTOR_REAL_DEBRID_API_KEY: '{{ .Values.environment.debridcollector.realDebridApiKey | b64enc }}'
+  POSTGRES_USER: '{{ .Values.environment.postgres.user | b64enc }}'
+  POSTGRES_PASSWORD: '{{ .Values.environment.postgres.password | b64enc }}'
+  RABBITMQ_PASSWORD: '{{ .Values.environment.lavinmq.password | b64enc }}'
+  RABBITMQ_USER: '{{ .Values.environment.lavinmq.user | b64enc }}'
--- a/deployment/k8s/templates/infrastructure/lavinmq/lavinmq.service.yaml
+++ b/deployment/k8s/templates/infrastructure/lavinmq/lavinmq.service.yaml
@@ -0,0 +1,25 @@
+{{ if .Values.infrastructure.lavinmq.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: '{{ .Release.Name }}-lavinmq'
+  labels:
+    component: lavinmq
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+spec:
+  selector:
+    component: lavinmq
+    release: '{{ .Release.Name }}'
+  type: ClusterIP
+  ports:
+    - protocol: TCP
+      port: 5672
+      targetPort: 5672
+    - protocol: TCP
+      port: 15672
+      targetPort: 15672
+    - protocol: TCP
+      port: 15692
+      targetPort: 15692
+{{- end -}}
--- a/deployment/k8s/templates/infrastructure/lavinmq/lavinmq.statefulset.yaml
+++ b/deployment/k8s/templates/infrastructure/lavinmq/lavinmq.statefulset.yaml
@@ -0,0 +1,60 @@
+{{ if .Values.infrastructure.lavinmq.enabled }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: '{{ .Release.Name }}-lavinmq'
+  labels:
+    component: lavinmq
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "0"
+spec:
+  serviceName: '{{ .Release.Name }}-lavinmq'
+  replicas: 1
+  selector:
+    matchLabels:
+      component: lavinmq
+      release: '{{ .Release.Name }}'
+  template:
+    metadata:
+      labels:
+        component: lavinmq
+        release: '{{ .Release.Name }}'
+    spec:
+      containers:
+        - name: lavinmq
+          image: '{{ .Values.infrastructure.lavinmq.image }}:{{ .Values.infrastructure.lavinmq.tag }}'
+          ports:
+            - name: lavinmq
+              containerPort: 5672
+            - name: lavinmq-15672
+              containerPort: 15672
+            - name: lavinmq-15692
+              containerPort: 15692
+          envFrom:
+            - configMapRef:
+                name: '{{ .Release.Name }}-config'
+            - secretRef:
+                name: '{{ .Release.Name }}-secrets'
+          volumeMounts:
+            - mountPath: /var/lib/lavinmq
+              name: lavinmq
+          livenessProbe:
+            exec:
+              command:
+                - lavinmqctl status
+            periodSeconds: 10
+            initialDelaySeconds: 10
+            successThreshold: 1
+            failureThreshold: 3
+  volumeClaimTemplates:
+    - metadata:
+        name: lavinmq
+      spec:
+        accessModes: [ "ReadWriteOnce" ]
+        resources:
+          requests:
+            storage: '{{ .Values.persistence.lavinmq.capacity }}'
+{{- end -}}
--- a/deployment/k8s/templates/infrastructure/postgres/postgres.service.yaml
+++ b/deployment/k8s/templates/infrastructure/postgres/postgres.service.yaml
@@ -0,0 +1,19 @@
+{{ if .Values.infrastructure.postgres.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: '{{ .Release.Name }}-postgres'
+  labels:
+    component: postgres
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+spec:
+  selector:
+    component: postgres
+    release: '{{ .Release.Name }}'
+  type: ClusterIP
+  ports:
+    - protocol: TCP
+      port: 5432
+      targetPort: 5432
+{{- end -}}
--- a/deployment/k8s/templates/infrastructure/postgres/postgres.statefulset.yaml
+++ b/deployment/k8s/templates/infrastructure/postgres/postgres.statefulset.yaml
@@ -0,0 +1,58 @@
+{{ if .Values.infrastructure.postgres.enabled }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: '{{ .Release.Name }}-postgres'
+  labels:
+    component: postgres
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "0"
+spec:
+  serviceName: '{{ .Release.Name }}-postgres'
+  replicas: 1
+  selector:
+    matchLabels:
+      component: postgres
+      release: '{{ .Release.Name }}'
+  template:
+    metadata:
+      labels:
+        component: postgres
+        release: '{{ .Release.Name }}'
+    spec:
+      containers:
+        - name: postgres
+          image: '{{ .Values.infrastructure.postgres.image }}:{{ .Values.infrastructure.postgres.tag }}'
+          ports:
+            - name: postgres
+              containerPort: 5432
+          envFrom:
+            - configMapRef:
+                name: '{{ .Release.Name }}-config'
+            - secretRef:
+                name: '{{ .Release.Name }}-secrets'
+          volumeMounts:
+            - mountPath: /var/lib/postgresql/data
+              name: postgres
+          livenessProbe:
+            exec:
+              command:
+                - sh
+                - -c
+                - pg_isready -h localhost -U $POSTGRES_USER
+            periodSeconds: 10
+            initialDelaySeconds: 10
+            successThreshold: 1
+            failureThreshold: 3
+  volumeClaimTemplates:
+    - metadata:
+        name: postgres
+      spec:
+        accessModes: [ "ReadWriteOnce" ]
+        resources:
+          requests:
+            storage: '{{ .Values.persistence.postgres.capacity }}'
+{{- end -}}
--- a/deployment/k8s/templates/infrastructure/qbittorrent/qbittorrent.deployment.yaml
+++ b/deployment/k8s/templates/infrastructure/qbittorrent/qbittorrent.deployment.yaml
@@ -0,0 +1,57 @@
+{{ if .Values.knightcrawler.qbitcollector.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: '{{ .Release.Name }}-qbittorrent'
+  labels:
+    component: qbittorrent
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "0"
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      component: qbittorrent
+      release: '{{ .Release.Name }}'
+  template:
+    metadata:
+      labels:
+        component: qbittorrent
+        release: '{{ .Release.Name }}'
+    spec:
+      containers:
+        - name: qbittorrent
+          image: '{{ .Values.infrastructure.qbittorrent.image }}:{{ .Values.infrastructure.qbittorrent.tag }}'
+          ports:
+            - name: qbittorrent
+              containerPort: 6881
+            - name: qbittorrent-6881
+              containerPort: 6881
+            - name: qbittorrent-8080
+              containerPort: 8080
+          env:
+            - name: PUID
+              value: '{{ .Values.environment.qbittorrent.puid }}'
+            - name: PGID
+              value: '{{ .Values.environment.qbittorrent.pgid }}'
+            - name: TORRENTING_PORT
+              value: '{{ .Values.environment.qbittorrent.torrentingPort }}'
+            - name: WEBUI_PORT
+              value: '{{ .Values.environment.qbittorrent.webuiPort }}'
+          envFrom:
+            - configMapRef:
+                name: '{{ .Release.Name }}-config'
+            - secretRef:
+                name: '{{ .Release.Name }}-secrets'
+          livenessProbe:
+            exec:
+              command:
+                - curl --fail http://localhost:8080
+            periodSeconds: 10
+            initialDelaySeconds: 10
+            successThreshold: 1
+            failureThreshold: 3
+{{- end -}}
--- a/deployment/k8s/templates/infrastructure/qbittorrent/qbittorrent.service.yaml
+++ b/deployment/k8s/templates/infrastructure/qbittorrent/qbittorrent.service.yaml
@@ -0,0 +1,25 @@
+{{ if .Values.knightcrawler.qbitcollector.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: '{{ .Release.Name }}-qbittorrent'
+  labels:
+    component: qbittorrent
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+spec:
+  selector:
+    component: qbittorrent
+    release: '{{ .Release.Name }}'
+  type: ClusterIP
+  ports:
+    - protocol: TCP
+      port: 6881
+      targetPort: 6881
+    - protocol: TCP
+      port: 6881
+      targetPort: 6881
+    - protocol: TCP
+      port: 8080
+      targetPort: 8080
+{{- end -}}
--- a/deployment/k8s/templates/infrastructure/redis/redis.service.yaml
+++ b/deployment/k8s/templates/infrastructure/redis/redis.service.yaml
@@ -0,0 +1,19 @@
+{{ if .Values.infrastructure.redis.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: '{{ .Release.Name }}-redis'
+  labels:
+    component: redis
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+spec:
+  selector:
+    component: redis
+    release: '{{ .Release.Name }}'
+  type: ClusterIP
+  ports:
+    - protocol: TCP
+      port: 6379
+      targetPort: 6379
+{{- end -}}
--- a/deployment/k8s/templates/infrastructure/redis/redis.statefulset.yaml
+++ b/deployment/k8s/templates/infrastructure/redis/redis.statefulset.yaml
@@ -0,0 +1,56 @@
+{{ if .Values.infrastructure.redis.enabled }}
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: '{{ .Release.Name }}-redis'
+  labels:
+    component: redis
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "0"
+spec:
+  serviceName: '{{ .Release.Name }}-redis'
+  replicas: 1
+  selector:
+    matchLabels:
+      component: redis
+      release: '{{ .Release.Name }}'
+  template:
+    metadata:
+      labels:
+        component: redis
+        release: '{{ .Release.Name }}'
+    spec:
+      containers:
+        - name: redis
+          image: '{{ .Values.infrastructure.redis.image }}:{{ .Values.infrastructure.redis.tag }}'
+          ports:
+            - name: redis
+              containerPort: 6379
+          envFrom:
+            - configMapRef:
+                name: '{{ .Release.Name }}-config'
+            - secretRef:
+                name: '{{ .Release.Name }}-secrets'
+          volumeMounts:
+            - mountPath: /data
+              name: redis
+          livenessProbe:
+            exec:
+              command:
+                - redis-cli ping
+            periodSeconds: 10
+            initialDelaySeconds: 10
+            successThreshold: 1
+            failureThreshold: 3
+  volumeClaimTemplates:
+    - metadata:
+        name: redis
+      spec:
+        accessModes: [ "ReadWriteOnce" ]
+        resources:
+          requests:
+            storage: '{{ .Values.persistence.redis.capacity }}'
+{{- end -}}
--- a/deployment/k8s/templates/jobs/metadata.job.yaml
+++ b/deployment/k8s/templates/jobs/metadata.job.yaml
@@ -0,0 +1,28 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: '{{ .Release.Name }}-metadata'
+  labels:
+    component: metadata
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "2"
+    "helm.sh/hook-delete-policy": hook-succeeded
+spec:
+  template:
+    metadata:
+      labels:
+        component: metadata
+        release: '{{ .Release.Name }}'
+    spec:
+      restartPolicy: OnFailure
+      containers:
+        - name: metadata
+          image: '{{ .Values.knightcrawler.metadata.image }}{{ if ne .Values.knightcrawler.globalImageTagOverride "" }}:{{ .Values.knightcrawler.globalImageTagOverride }}{{else}}:{{ .Values.knightcrawler.metadata.tag}}{{ end }}'
+          envFrom:
+            - configMapRef:
+                name: '{{ .Release.Name }}-config'
+            - secretRef:
+                name: '{{ .Release.Name }}-secrets'
--- a/deployment/k8s/templates/jobs/migrator.job.yaml
+++ b/deployment/k8s/templates/jobs/migrator.job.yaml
@@ -0,0 +1,28 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: '{{ .Release.Name }}-migrator'
+  labels:
+    component: migrator
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "1"
+    "helm.sh/hook-delete-policy": hook-succeeded
+spec:
+  template:
+    metadata:
+      labels:
+        component: migrator
+        release: '{{ .Release.Name }}'
+    spec:
+      restartPolicy: OnFailure
+      containers:
+        - name: migrator
+          image: '{{ .Values.knightcrawler.migrator.image }}{{ if ne .Values.knightcrawler.globalImageTagOverride "" }}:{{ .Values.knightcrawler.globalImageTagOverride }}{{else}}:{{ .Values.knightcrawler.migrator.tag}}{{ end }}'
+          envFrom:
+            - configMapRef:
+                name: '{{ .Release.Name }}-config'
+            - secretRef:
+                name: '{{ .Release.Name }}-secrets'
--- a/deployment/k8s/templates/kc/deployments/addon.deployment.yaml
+++ b/deployment/k8s/templates/kc/deployments/addon.deployment.yaml
@@ -0,0 +1,35 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: '{{ .Release.Name }}-addon'
+  labels:
+    component: addon
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "4"
+spec:
+  replicas: {{ .Values.knightcrawler.addon.replicas }}
+  selector:
+    matchLabels:
+      component: addon
+      release: '{{ .Release.Name }}'
+  template:
+    metadata:
+      labels:
+        component: addon
+        release: '{{ .Release.Name }}'
+    spec:
+      containers:
+        - name: addon
+          image: '{{ .Values.knightcrawler.addon.image }}{{ if ne .Values.knightcrawler.globalImageTagOverride "" }}:{{ .Values.knightcrawler.globalImageTagOverride }}{{else}}:{{ .Values.knightcrawler.addon.tag}}{{ end }}'
+          ports:
+            - name: addon
+              containerPort: 7000
+          envFrom:
+            - configMapRef:
+                name: '{{ .Release.Name }}-config'
+            - secretRef:
+                name: '{{ .Release.Name }}-secrets'
+
--- a/deployment/k8s/templates/kc/deployments/consumer.deployment.yaml
+++ b/deployment/k8s/templates/kc/deployments/consumer.deployment.yaml
@@ -0,0 +1,32 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: '{{ .Release.Name }}-consumer'
+  labels:
+    component: consumer
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "4"
+spec:
+  replicas: {{ .Values.knightcrawler.consumer.replicas }}
+  selector:
+    matchLabels:
+      component: consumer
+      release: '{{ .Release.Name }}'
+  template:
+    metadata:
+      labels:
+        component: consumer
+        release: '{{ .Release.Name }}'
+    spec:
+      containers:
+        - name: consumer
+          image: '{{ .Values.knightcrawler.consumer.image }}{{ if ne .Values.knightcrawler.globalImageTagOverride "" }}:{{ .Values.knightcrawler.globalImageTagOverride }}{{else}}:{{ .Values.knightcrawler.consumer.tag}}{{ end }}'
+          envFrom:
+            - configMapRef:
+                name: '{{ .Release.Name }}-config'
+            - secretRef:
+                name: '{{ .Release.Name }}-secrets'
+
--- a/deployment/k8s/templates/kc/deployments/debridcollector.deployment.yaml
+++ b/deployment/k8s/templates/kc/deployments/debridcollector.deployment.yaml
@@ -0,0 +1,31 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: '{{ .Release.Name }}-debridcollector'
+  labels:
+    component: debridcollector
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "4"
+spec:
+  replicas: {{ .Values.knightcrawler.debridcollector.replicas }}
+  selector:
+    matchLabels:
+      component: debridcollector
+      release: '{{ .Release.Name }}'
+  template:
+    metadata:
+      labels:
+        component: debridcollector
+        release: '{{ .Release.Name }}'
+    spec:
+      containers:
+        - name: debridcollector
+          image: '{{ .Values.knightcrawler.debridcollector.image }}{{ if ne .Values.knightcrawler.globalImageTagOverride "" }}:{{ .Values.knightcrawler.globalImageTagOverride }}{{else}}:{{ .Values.knightcrawler.debridcollector.tag}}{{ end }}'
+          envFrom:
+            - configMapRef:
+                name: '{{ .Release.Name }}-config'
+            - secretRef:
+                name: '{{ .Release.Name }}-secrets'
--- a/deployment/k8s/templates/kc/deployments/producer.deployment.yaml
+++ b/deployment/k8s/templates/kc/deployments/producer.deployment.yaml
@@ -0,0 +1,31 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: '{{ .Release.Name }}-producer'
+  labels:
+    component: producer
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "4"
+spec:
+  replicas: {{ .Values.knightcrawler.producer.replicas }}
+  selector:
+    matchLabels:
+      component: producer
+      release: '{{ .Release.Name }}'
+  template:
+    metadata:
+      labels:
+        component: producer
+        release: '{{ .Release.Name }}'
+    spec:
+      containers:
+        - name: producer
+          image: '{{ .Values.knightcrawler.producer.image }}{{ if ne .Values.knightcrawler.globalImageTagOverride "" }}:{{ .Values.knightcrawler.globalImageTagOverride }}{{else}}:{{ .Values.knightcrawler.producer.tag}}{{ end }}'
+          envFrom:
+            - configMapRef:
+                name: '{{ .Release.Name }}-config'
+            - secretRef:
+                name: '{{ .Release.Name }}-secrets'
--- a/deployment/k8s/templates/kc/deployments/qbitcollector.deployment.yaml
+++ b/deployment/k8s/templates/kc/deployments/qbitcollector.deployment.yaml
@@ -0,0 +1,33 @@
+{{ if .Values.knightcrawler.qbitcollector.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: '{{ .Release.Name }}-qbitcollector'
+  labels:
+    component: qbitcollector
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+  annotations:
+    "helm.sh/hook": post-install,post-upgrade
+    "helm.sh/hook-weight": "4"
+spec:
+  replicas: {{ .Values.knightcrawler.qbitcollector.replicas }}
+  selector:
+    matchLabels:
+      component: qbitcollector
+      release: '{{ .Release.Name }}'
+  template:
+    metadata:
+      labels:
+        component: qbitcollector
+        release: '{{ .Release.Name }}'
+    spec:
+      containers:
+        - name: qbitcollector
+          image: '{{ .Values.knightcrawler.qbitcollector.image }}{{ if ne .Values.knightcrawler.globalImageTagOverride "" }}:{{ .Values.knightcrawler.globalImageTagOverride }}{{else}}:{{ .Values.knightcrawler.qbitcollector.tag}}{{ end }}'
+          envFrom:
+            - configMapRef:
+                name: '{{ .Release.Name }}-config'
+            - secretRef:
+                name: '{{ .Release.Name }}-secrets'
+{{- end -}}
--- a/deployment/k8s/templates/kc/services/addon.service.yaml
+++ b/deployment/k8s/templates/kc/services/addon.service.yaml
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: '{{ .Release.Name }}-addon'
+  labels:
+    component: addon
+    project: '{{ .Chart.Name }}'
+    release: '{{ .Release.Name }}'
+spec:
+  selector:
+    component: addon
+    release: '{{ .Release.Name }}'
+  type: ClusterIP
+  ports:
+    - protocol: TCP
+      port: 7000
+      targetPort: 7000
--- a/deployment/k8s/values.yaml
+++ b/deployment/k8s/values.yaml
@@ -0,0 +1,100 @@
+knightcrawler:
+  debug: false
+  globalImageTagOverride: ""
+  addon:
+    image: "gabisonfire/knightcrawler-addon"
+    tag: "2.0.17"
+    replicas: 1
+  consumer:
+    image: "gabisonfire/knightcrawler-consumer"
+    tag: "2.0.17"
+    replicas: 1
+  metadata:
+    image: "gabisonfire/knightcrawler-metadata"
+    tag: "2.0.17"
+    replicas: 1
+  migrator:
+    image: "gabisonfire/knightcrawler-migrator"
+    tag: "2.0.17"
+    replicas: 1
+  debridcollector:
+    image: "gabisonfire/knightcrawler-debrid-collector"
+    tag: "2.0.17"
+    enabled: true
+    replicas: 1
+  qbitcollector:
+    image: "gabisonfire/knightcrawler-qbit-collector"
+    tag: "2.0.17"
+    enabled: false
+    replicas: 1
+  producer:
+    image: "gabisonfire/knightcrawler-producer"
+    tag: "2.0.17"
+    replicas: 1
+    
+infrastructure:
+  lavinmq:
+    image: "cloudamqp/lavinmq"
+    tag: "latest"
+    enabled: true
+  postgres:
+    image: "postgres"
+    tag: "latest"
+    enabled: true
+  redis:
+    image: "redis/redis-stack-server"
+    tag: "latest"
+    enabled: true
+  qbittorrent:
+    image: "lscr.io/linuxserver/qbittorrent"
+    tag: "latest"
+      
+environment:
+  redis:
+    external: false
+    host: ""
+    port: "6379"
+    extra: "abortConnect=false,allowAdmin=true"
+  postgres:
+    external: false
+    host: ""
+    port: "5432"
+    dbName: "knightcrawler"
+    user: "postgres"
+    password: "postgres"
+  lavinmq:
+    external: false
+    host: ""
+    user: "guest"
+    password: "guest"
+  qbitcollector:
+    qbitHost: "http://qbittorrent:8080"
+    trackersUrl: "https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_http.txt"
+  debridcollector:
+    realDebridApiKey: ""
+  producer:
+    githubPat: ""
+    queueName: "ingested"
+    durable: true
+    maxPublishBatchSize: 500
+    maxQueueSize: 0
+    publishIntervalSeconds: 10
+  metadata:
+    insertBatchSize: 50000
+  qbittorrent:
+    pgid: "1000"
+    puid: "1000"
+    torrentingPort: "6881"
+    webuiPort: "8080"
+      
+persistence:
+  storageClassName: ""
+  redis:
+    capacity: 1Gi
+  postgres:
+    capacity: 1Gi
+  lavinmq:
+    capacity: 1Gi
+    
+shared:
+  timezone: "London/Europe"
--- a/src/migrator/migrations/012_imdb_change_ratio.sql
+++ b/src/migrator/migrations/012_imdb_change_ratio.sql
@@ -0,0 +1,19 @@
+-- Remove the old search function
+DROP FUNCTION IF EXISTS search_imdb_meta(TEXT, TEXT, INT, INT);
+
+-- Add the new search function that allows for searching by year with a plus/minus one year range
+CREATE OR REPLACE FUNCTION search_imdb_meta(search_term TEXT, category_param TEXT DEFAULT NULL, year_param INT DEFAULT NULL, limit_param INT DEFAULT 10, similarity_threshold REAL DEFAULT 0.95)
+    RETURNS TABLE(imdb_id character varying(16), title character varying(1000),category character varying(50),year INT, score REAL) AS $$
+BEGIN
+    SET pg_trgm.similarity_threshold = similarity_threshold;
+    RETURN QUERY
+        SELECT imdb_metadata.imdb_id, imdb_metadata.title, imdb_metadata.category, imdb_metadata.year, similarity(imdb_metadata.title, search_term) as score
+        FROM imdb_metadata
+        WHERE (imdb_metadata.title % search_term)
+          AND (imdb_metadata.adult = FALSE)
+          AND (category_param IS NULL OR imdb_metadata.category = category_param)
+          AND (year_param IS NULL OR imdb_metadata.year BETWEEN year_param - 1 AND year_param + 1)
+        ORDER BY score DESC
+        LIMIT limit_param;
+END; $$
+    LANGUAGE plpgsql;
--- a/src/migrator/migrations/013_imdb_change_ratio_fix.sql
+++ b/src/migrator/migrations/013_imdb_change_ratio_fix.sql
@@ -0,0 +1,19 @@
+-- Remove the old search function
+DROP FUNCTION IF EXISTS search_imdb_meta(TEXT, TEXT, INT, INT);
+
+-- Add the new search function that allows for searching by year with a plus/minus one year range
+CREATE OR REPLACE FUNCTION search_imdb_meta(search_term TEXT, category_param TEXT DEFAULT NULL, year_param INT DEFAULT NULL, limit_param INT DEFAULT 10, similarity_threshold REAL DEFAULT 0.95)
+    RETURNS TABLE(imdb_id character varying(16), title character varying(1000),category character varying(50),year INT, score REAL) AS $$
+BEGIN
+    EXECUTE format('SET pg_trgm.similarity_threshold = %L', similarity_threshold);
+    RETURN QUERY
+        SELECT imdb_metadata.imdb_id, imdb_metadata.title, imdb_metadata.category, imdb_metadata.year, similarity(imdb_metadata.title, search_term) as score
+        FROM imdb_metadata
+        WHERE (imdb_metadata.title % search_term)
+          AND (imdb_metadata.adult = FALSE)
+          AND (category_param IS NULL OR imdb_metadata.category = category_param)
+          AND (year_param IS NULL OR imdb_metadata.year BETWEEN year_param - 1 AND year_param + 1)
+        ORDER BY score DESC
+        LIMIT limit_param;
+END; $$
+    LANGUAGE plpgsql;
--- a/src/producer/src/Features/Crawlers/Dmm/DebridMediaManagerCrawler.cs
+++ b/src/producer/src/Features/Crawlers/Dmm/DebridMediaManagerCrawler.cs
@@ -108,14 +108,18 @@ public partial class DebridMediaManagerCrawler(
            return null;
        }
        
-        var parsedTorrent = rankTorrentName.Parse(torrentTitle.CleanTorrentTitleForImdb());
+        var parsedTorrent = rankTorrentName.Parse(torrentTitle);
        
        if (!parsedTorrent.Success)
        {
            return null;
        }
        
-        var (cached, cachedResult) = await CheckIfInCacheAndReturn(parsedTorrent.Response.ParsedTitle);
+        var torrentType = parsedTorrent.Response.IsMovie ? "movie" : "tvSeries";
+        
+        var cacheKey = GetCacheKey(torrentType, parsedTorrent.Response.ParsedTitle, parsedTorrent.Response.Year);
+        
+        var (cached, cachedResult) = await CheckIfInCacheAndReturn(cacheKey);
        
        if (cached)
        {
@@ -124,14 +128,14 @@ public partial class DebridMediaManagerCrawler(
        }

        int? year = parsedTorrent.Response.Year != 0 ? parsedTorrent.Response.Year : null;
-        var imdbEntry = await Storage.FindImdbMetadata(parsedTorrent.Response.ParsedTitle, parsedTorrent.Response.IsMovie ? "movies" : "tv", year);
+        var imdbEntry = await Storage.FindImdbMetadata(parsedTorrent.Response.ParsedTitle, torrentType, year);

        if (imdbEntry is null)
        {
            return null;
        }
        
-        await AddToCache(parsedTorrent.Response.ParsedTitle.ToLowerInvariant(), imdbEntry);
+        await AddToCache(cacheKey, imdbEntry);
        
        logger.LogInformation("[{ImdbId}] Found best match for {Title}: {BestMatch} with score {Score}", imdbEntry.ImdbId, parsedTorrent.Response.ParsedTitle, imdbEntry.Title, imdbEntry.Score);

@@ -148,27 +152,24 @@ public partial class DebridMediaManagerCrawler(
            InfoHash = hashElement.ToString(),
            Seeders = 0,
            Leechers = 0,
-            Category = parsedTorrent.Response.IsMovie switch
-            {
-                true => "movies",
-                false => "tv",
-            },
+            Category = AssignCategory(result),
            RtnResponse = parsedTorrent.Response.ToJson(),
        };
+    

-    private Task AddToCache(string lowerCaseTitle, ImdbEntry best)
+    private Task AddToCache(string cacheKey, ImdbEntry best)
    {
        var cacheOptions = new DistributedCacheEntryOptions
        {
            AbsoluteExpirationRelativeToNow = TimeSpan.FromDays(1),
        };
        
-        return cache.SetStringAsync(lowerCaseTitle, JsonSerializer.Serialize(best), cacheOptions);
+        return cache.SetStringAsync(cacheKey, JsonSerializer.Serialize(best), cacheOptions);
    }

-    private async Task<(bool Success, ImdbEntry? Entry)> CheckIfInCacheAndReturn(string title)
+    private async Task<(bool Success, ImdbEntry? Entry)> CheckIfInCacheAndReturn(string cacheKey)
    {
-        var cachedImdbId = await cache.GetStringAsync(title.ToLowerInvariant());
+        var cachedImdbId = await cache.GetStringAsync(cacheKey);
        
        if (!string.IsNullOrEmpty(cachedImdbId))
        {
@@ -208,4 +209,14 @@ public partial class DebridMediaManagerCrawler(

        return (pageIngested, name);
    }
+    
+    private static string AssignCategory(ImdbEntry entry) =>
+        entry.Category.ToLower() switch
+        {
+            var category when string.Equals(category, "movie", StringComparison.OrdinalIgnoreCase) => "movies",
+            var category when string.Equals(category, "tvSeries", StringComparison.OrdinalIgnoreCase) => "tv",
+            _ => "unknown",
+        };
+    
+    private static string GetCacheKey(string category, string title, int year) => $"{category.ToLowerInvariant()}:{year}:{title.ToLowerInvariant()}";
 }
--- a/src/producer/src/requirements.txt
+++ b/src/producer/src/requirements.txt
@@ -1 +1 @@
-rank-torrent-name==0.1.9
+rank-torrent-name==0.1.8
--- a/src/qbit-collector/Extensions/ServiceCollectionExtensions.cs
+++ b/src/qbit-collector/Extensions/ServiceCollectionExtensions.cs
@@ -18,6 +18,7 @@ public static class ServiceCollectionExtensions
        services.AddHttpClient();
        services.AddSingleton<ITrackersService, TrackersService>();
        services.AddHostedService<TrackersBackgroundService>();
+        services.AddHostedService<HousekeepingBackgroundService>();

        return services;
    }
--- a/src/qbit-collector/Features/Qbit/HousekeepingBackgroundService.cs
+++ b/src/qbit-collector/Features/Qbit/HousekeepingBackgroundService.cs
@@ -0,0 +1,52 @@
+namespace QBitCollector.Features.Qbit;
+
+public class HousekeepingBackgroundService(IQBittorrentClient client, ILogger<HousekeepingBackgroundService> logger) : BackgroundService
+{
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        logger.LogInformation("Service is Running.");
+
+        await DoWork();
+
+        using PeriodicTimer timer = new(TimeSpan.FromMinutes(2));
+
+        try
+        {
+            while (await timer.WaitForNextTickAsync(stoppingToken))
+            {
+                await DoWork();
+            }
+        }
+        catch (OperationCanceledException)
+        {
+            logger.LogInformation("Service stopping.");
+        }
+    }
+
+    private async Task DoWork()
+    {
+        try
+        {
+            logger.LogInformation("Cleaning Stale Entries in Qbit...");
+
+            var torrents = await client.GetTorrentListAsync();
+
+            foreach (var torrentInfo in torrents)
+            {
+                if (!(torrentInfo.AddedOn < DateTimeOffset.UtcNow.AddMinutes(-1)))
+                {
+                    continue;
+                }
+                
+                logger.LogInformation("Torrent [{InfoHash}] Identified as stale because was added at {AddedOn}", torrentInfo.Hash, torrentInfo.AddedOn);
+
+                await client.DeleteAsync(new[] {torrentInfo.Hash}, deleteDownloadedData: true);
+                logger.LogInformation("Cleaned up stale torrent: [{InfoHash}]", torrentInfo.Hash);
+            }
+        }
+        catch (Exception e)
+        {
+            logger.LogError(e, "Error cleaning up stale torrents this interval.");
+        }
+    }
+}
--- a/src/shared/Dapper/DapperDataStorage.cs
+++ b/src/shared/Dapper/DapperDataStorage.cs
@@ -118,7 +118,7 @@ public class DapperDataStorage(PostgresConfiguration configuration, RabbitMqConf
    public async Task<ImdbEntry?> FindImdbMetadata(string? parsedTorrentTitle, string torrentType, int? year, CancellationToken cancellationToken = default) =>
        await ExecuteCommandAsync(async connection =>
        {
-            var query = $"select \"imdb_id\" as \"ImdbId\", \"title\" as \"Title\", \"year\" as \"Year\", \"score\" as Score from search_imdb_meta('{parsedTorrentTitle.Replace("'", "").Replace("\"", "")}', '{(torrentType.Equals("movie", StringComparison.OrdinalIgnoreCase) ? "movie" : "tvSeries")}'";
+            var query = $"select \"imdb_id\" as \"ImdbId\", \"title\" as \"Title\", \"year\" as \"Year\", \"score\" as Score, \"category\" as Category from search_imdb_meta('{parsedTorrentTitle.Replace("'", "").Replace("\"", "")}', '{torrentType}'";
            query += year is not null ? $", {year}" : ", NULL";
            query += ", 1)";
            
--- a/src/shared/Extensions/StringExtensions.cs
+++ b/src/shared/Extensions/StringExtensions.cs
@@ -1,5 +1,3 @@
-using System.Text.RegularExpressions;
-
 namespace SharedContracts.Extensions;

 public static partial class StringExtensions
--- a/src/shared/GlobalUsings.cs
+++ b/src/shared/GlobalUsings.cs
@@ -2,6 +2,7 @@

 global using System.Text.Json;
 global using System.Text.Json.Serialization;
+global using System.Text.RegularExpressions;
 global using Dapper;
 global using MassTransit;
 global using Microsoft.AspNetCore.Builder;
--- a/src/shared/Python/RTN/RankTorrentName.cs
+++ b/src/shared/Python/RTN/RankTorrentName.cs
@@ -20,8 +20,7 @@ public class RankTorrentName : IRankTorrentName
                var result = _rtn?.parse(title);
                return ParseResult(result);
            }, new ParseTorrentTitleResponse(false, null), nameof(Parse), throwOnErrors: false, logErrors: false);
-
-
+    
    private static ParseTorrentTitleResponse ParseResult(dynamic result)
    {
        if (result == null)
--- a/src/shared/Python/RTN/RtnResponse.cs
+++ b/src/shared/Python/RTN/RtnResponse.cs
@@ -77,7 +77,22 @@ public class RtnResponse
    [JsonPropertyName("extended")]
    public bool Extended { get; set; }
    
-    public bool IsMovie => (Season == null && Episode == null) || (Season?.Count == 0 && Episode?.Count == 0);
+    // [JsonPropertyName("is_show")]
+    // public bool IsTvShow { get; set; }
+    //
+    // [JsonPropertyName("is_movie")]
+    // public bool IsMovie { get; set; }
    
    public string ToJson() => this.AsJson();
+    
+    public bool IsMovie => !TvRegexes.Any(regex => regex.IsMatch(RawTitle)) && Season?.Count == 0 && Episode?.Count == 0;
+    
+    private static List<Regex> TvRegexes { get; set; } =
+    [
+        new(@"[se]\d\d", RegexOptions.IgnoreCase),
+        new(@"\b(tv|complete)\b", RegexOptions.IgnoreCase),
+        new(@"\b(saisons?|stages?|seasons?).?\d", RegexOptions.IgnoreCase),
+        new(@"[a-z]\s?\-\s?\d{2,4}\b", RegexOptions.IgnoreCase),
+        new(@"\d{2,4}\s?\-\s?\d{2,4}\b", RegexOptions.IgnoreCase),
+    ];
 }
Author	SHA1	Message	Date
iPromKnight	f56f205bbe	helm chart [wip]	2024-03-30 23:54:34 +00:00
iPromKnight	c75ecd2707	add qbit housekeeping service to remove stale torrents (#193 ) * Add housekeeping service to clean stale torrents * version bump	2024-03-30 11:52:23 +00:00
iPromKnight	c493ef3376	Hotfix category, and roll back RTN to 0.1.8 (#192 ) * Hotfix categories Also roll back RTN to 0.1.8 as regression introduced in 0.2 * bump version	2024-03-30 04:47:36 +00:00
iPromKnight	655a39e35c	patch the query with execute (#191 )	2024-03-30 01:54:06 +00:00
iPromKnight	cfeee62f6b	patch ratio (#190 ) * add configurable threshold, default 0.95 * version bump	2024-03-30 01:43:21 +00:00
iPromKnight	c6d4c06d70	hotfix categories from imdb result instead (#189 ) * category mapping from imdb * version bump	2024-03-30 01:26:02 +00:00