From cc2adbfca5b86cc797ea38e11194582850e55307 Mon Sep 17 00:00:00 2001 From: iPromKnight <156901906+iPromKnight@users.noreply.github.com> Date: Wed, 27 Mar 2024 12:21:40 +0000 Subject: [PATCH] Recreate single docker-compose file (#174) Clean it up - also comment all services --- deployment/docker/.env.example | 5 +- deployment/docker/compose.yaml | 7 - .../config/qbit/qbittorrent.conf | 0 deployment/docker/docker-compose.yaml | 266 ++++++++++++++++++ .../{ => src}/components/infrastructure.yaml | 10 +- .../{ => src}/components/knightcrawler.yaml | 6 +- .../docker/{ => src}/components/network.yaml | 0 .../docker/{ => src}/components/volumes.yaml | 0 .../docker/{ => src}/compose.override.yaml | 0 deployment/docker/src/compose.yaml | 7 + 10 files changed, 285 insertions(+), 16 deletions(-) delete mode 100644 deployment/docker/compose.yaml rename deployment/docker/{components => }/config/qbit/qbittorrent.conf (100%) mode change 100644 => 100755 create mode 100644 deployment/docker/docker-compose.yaml rename deployment/docker/{ => src}/components/infrastructure.yaml (92%) rename deployment/docker/{ => src}/components/knightcrawler.yaml (95%) rename deployment/docker/{ => src}/components/network.yaml (100%) rename deployment/docker/{ => src}/components/volumes.yaml (100%) rename deployment/docker/{ => src}/compose.override.yaml (100%) create mode 100644 deployment/docker/src/compose.yaml diff --git a/deployment/docker/.env.example b/deployment/docker/.env.example index bb08140..6897e52 100644 --- a/deployment/docker/.env.example +++ b/deployment/docker/.env.example @@ -31,7 +31,10 @@ COLLECTOR_QBIT_ENABLED=false COLLECTOR_DEBRID_ENABLED=true COLLECTOR_REAL_DEBRID_API_KEY= QBIT_HOST=http://qbittorrent:8080 -QBIT_TRACKERS_URL=https://ngosang.github.io/trackerslist/trackers_all.txt +QBIT_TRACKERS_URL=https://raw.githubusercontent.com/ngosang/trackerslist/master/trackers_all_http.txt + +# Number of replicas for the qBittorrent collector and qBitTorrent client. Should be 0 or 1. +QBIT_REPLICAS=0 # Addon DEBUG_MODE=false diff --git a/deployment/docker/compose.yaml b/deployment/docker/compose.yaml deleted file mode 100644 index ab80725..0000000 --- a/deployment/docker/compose.yaml +++ /dev/null @@ -1,7 +0,0 @@ -version: "3.9" -name: "knightcrawler" -include: - - components/network.yaml - - components/volumes.yaml - - components/infrastructure.yaml - - components/knightcrawler.yaml \ No newline at end of file diff --git a/deployment/docker/components/config/qbit/qbittorrent.conf b/deployment/docker/config/qbit/qbittorrent.conf old mode 100644 new mode 100755 similarity index 100% rename from deployment/docker/components/config/qbit/qbittorrent.conf rename to deployment/docker/config/qbit/qbittorrent.conf diff --git a/deployment/docker/docker-compose.yaml b/deployment/docker/docker-compose.yaml new file mode 100644 index 0000000..40bba32 --- /dev/null +++ b/deployment/docker/docker-compose.yaml @@ -0,0 +1,266 @@ +version: "3.9" + +name: knightcrawler + +networks: + knightcrawler-network: + name: knightcrawler-network + driver: bridge + +volumes: + postgres: + rabbitmq: + redis: + +services: + ## Postgres is the database that is used by the services. + ## All downloaded metadata is stored in this database. + postgres: + env_file: .env + environment: + PGUSER: ${POSTGRES_USER} + healthcheck: + test: + - CMD-SHELL + - pg_isready + timeout: 10s + interval: 10s + retries: 3 + start_period: 10s + image: postgres:latest + # # If you need the database to be accessible from outside, please open the below port. + # # Furthermore, please, please, please, change the username and password in the .env file. + # # If you want to enhance your security even more, create a new user for the database with a strong password. + # ports: + # - "5432:5432" + networks: + - knightcrawler-network + restart: unless-stopped + volumes: + - postgres:/var/lib/postgresql/data + + ## Redis is used as a cache for the services. + ## It is used to store the infohashes that are currently being processed in sagas, as well as intrim data. + redis: + env_file: .env + healthcheck: + test: + - CMD-SHELL + - redis-cli ping + timeout: 10s + interval: 10s + retries: 3 + start_period: 10s + image: redis/redis-stack:latest + # # If you need redis to be accessible from outside, please open the below port. + # ports: + # - "6379:6379" + networks: + - knightcrawler-network + restart: unless-stopped + volumes: + - redis:/data + + ## RabbitMQ is used as a message broker for the services. + ## It is used to communicate between the services. + rabbitmq: + env_file: .env + healthcheck: + test: + - CMD-SHELL + - rabbitmq-diagnostics -q ping + timeout: 10s + interval: 10s + retries: 3 + start_period: 10s + # # If you need the database to be accessible from outside, please open the below port. + # # Furthermore, please, please, please, look at the documentation for rabbit on how to secure the service. + # ports: + # - "5672:5672" + # - "15672:15672" + # - "15692:15692" + image: rabbitmq:3-management + networks: + - knightcrawler-network + restart: unless-stopped + volumes: + - rabbitmq:/var/lib/rabbitmq + + ## The addon. This is what is used in stremio + addon: + depends_on: + metadata: + condition: service_completed_successfully + required: true + migrator: + condition: service_completed_successfully + required: true + postgres: + condition: service_healthy + required: true + rabbitmq: + condition: service_healthy + required: true + redis: + condition: service_healthy + required: true + env_file: .env + hostname: knightcrawler-addon + image: gabisonfire/knightcrawler-addon:2.0.7 + labels: + logging: promtail + networks: + - knightcrawler-network + ports: + - "7000:7000" + restart: unless-stopped + + ## The consumer is responsible for consuming infohashes and orchestrating download of metadata. + consumer: + depends_on: + metadata: + condition: service_completed_successfully + required: true + migrator: + condition: service_completed_successfully + required: true + postgres: + condition: service_healthy + required: true + rabbitmq: + condition: service_healthy + required: true + redis: + condition: service_healthy + required: true + env_file: .env + image: gabisonfire/knightcrawler-consumer:2.0.7 + labels: + logging: promtail + networks: + - knightcrawler-network + restart: unless-stopped + + ## The debrid collector is responsible for downloading metadata from debrid services. (Currently only RealDebrid is supported) + debridcollector: + depends_on: + metadata: + condition: service_completed_successfully + required: true + migrator: + condition: service_completed_successfully + required: true + postgres: + condition: service_healthy + required: true + rabbitmq: + condition: service_healthy + required: true + redis: + condition: service_healthy + required: true + env_file: .env + image: gabisonfire/knightcrawler-debrid-collector:2.0.7 + labels: + logging: promtail + networks: + - knightcrawler-network + restart: unless-stopped + + ## The metadata service is responsible for downloading imdb publically available datasets. + ## This is used to enrich the metadata during production of ingested infohashes. + metadata: + depends_on: + migrator: + condition: service_completed_successfully + required: true + env_file: .env + image: gabisonfire/knightcrawler-metadata:2.0.7 + networks: + - knightcrawler-network + restart: "no" + + ## The migrator is responsible for migrating the database schema. + migrator: + depends_on: + postgres: + condition: service_healthy + required: true + env_file: .env + image: gabisonfire/knightcrawler-migrator:2.0.7 + networks: + - knightcrawler-network + restart: "no" + + ## The producer is responsible for producing infohashes by acquiring for various sites, including DMM. + producer: + depends_on: + metadata: + condition: service_completed_successfully + required: true + migrator: + condition: service_completed_successfully + required: true + postgres: + condition: service_healthy + required: true + rabbitmq: + condition: service_healthy + required: true + redis: + condition: service_healthy + required: true + env_file: .env + image: gabisonfire/knightcrawler-producer:2.0.7 + labels: + logging: promtail + networks: + - knightcrawler-network + restart: unless-stopped + + ## QBit collector utilizes QBitTorrent to download metadata. + qbitcollector: + depends_on: + qbittorrent: + condition: service_healthy + required: true + deploy: + replicas: ${QBIT_REPLICAS:-0} + env_file: .env + image: gabisonfire/knightcrawler-qbit-collector:2.0.7 + labels: + logging: promtail + networks: + - knightcrawler-network + restart: unless-stopped + + ## QBitTorrent is a torrent client that can be used to download torrents. In this case its used to download metadata. + ## The QBit collector requires this. + qbittorrent: + deploy: + replicas: ${QBIT_REPLICAS:-0} + env_file: .env + environment: + PGID: "1000" + PUID: "1000" + TORRENTING_PORT: "6881" + WEBUI_PORT: "8080" + healthcheck: + test: + - CMD-SHELL + - curl --fail http://localhost:8080 + timeout: 10s + interval: 10s + retries: 3 + start_period: 10s + image: lscr.io/linuxserver/qbittorrent:latest + networks: + - knightcrawler-network + ports: + - "6881:6881/tcp" + - "6881:6881/udp" + # if you want to expose the webui, uncomment the following line + # - "8001:8080" + restart: unless-stopped + volumes: + - ./config/qbit/qbittorrent.conf:/config/qBittorrent/qBittorrent.conf diff --git a/deployment/docker/components/infrastructure.yaml b/deployment/docker/src/components/infrastructure.yaml similarity index 92% rename from deployment/docker/components/infrastructure.yaml rename to deployment/docker/src/components/infrastructure.yaml index 23eb22c..4de9400 100644 --- a/deployment/docker/components/infrastructure.yaml +++ b/deployment/docker/src/components/infrastructure.yaml @@ -35,7 +35,7 @@ services: - postgres:/var/lib/postgresql/data healthcheck: *postgresdb-health restart: unless-stopped - env_file: ../.env + env_file: ../../.env networks: - knightcrawler-network @@ -48,7 +48,7 @@ services: - redis:/data restart: unless-stopped healthcheck: *redis-health - env_file: ../.env + env_file: ../../.env networks: - knightcrawler-network @@ -64,7 +64,7 @@ services: - rabbitmq:/var/lib/rabbitmq restart: unless-stopped healthcheck: *rabbitmq-health - env_file: ../.env + env_file: ../../.env networks: - knightcrawler-network @@ -80,10 +80,10 @@ services: ports: - 6881:6881 - 6881:6881/udp - env_file: ../.env + env_file: ../../.env networks: - knightcrawler-network restart: unless-stopped healthcheck: *qbit-health volumes: - - ./config/qbit/qbittorrent.conf:/config/qBittorrent/qBittorrent.conf \ No newline at end of file + - ../../config/qbit/qbittorrent.conf:/config/qBittorrent/qBittorrent.conf \ No newline at end of file diff --git a/deployment/docker/components/knightcrawler.yaml b/deployment/docker/src/components/knightcrawler.yaml similarity index 95% rename from deployment/docker/components/knightcrawler.yaml rename to deployment/docker/src/components/knightcrawler.yaml index 5154f66..3e677bb 100644 --- a/deployment/docker/components/knightcrawler.yaml +++ b/deployment/docker/src/components/knightcrawler.yaml @@ -1,7 +1,7 @@ x-apps: &knightcrawler-app labels: logging: "promtail" - env_file: ../.env + env_file: ../../.env networks: - knightcrawler-network @@ -21,7 +21,7 @@ x-depends: &knightcrawler-app-depends services: metadata: image: gabisonfire/knightcrawler-metadata:2.0.7 - env_file: ../.env + env_file: ../../.env networks: - knightcrawler-network restart: no @@ -31,7 +31,7 @@ services: migrator: image: gabisonfire/knightcrawler-migrator:2.0.7 - env_file: ../.env + env_file: ../../.env networks: - knightcrawler-network restart: no diff --git a/deployment/docker/components/network.yaml b/deployment/docker/src/components/network.yaml similarity index 100% rename from deployment/docker/components/network.yaml rename to deployment/docker/src/components/network.yaml diff --git a/deployment/docker/components/volumes.yaml b/deployment/docker/src/components/volumes.yaml similarity index 100% rename from deployment/docker/components/volumes.yaml rename to deployment/docker/src/components/volumes.yaml diff --git a/deployment/docker/compose.override.yaml b/deployment/docker/src/compose.override.yaml similarity index 100% rename from deployment/docker/compose.override.yaml rename to deployment/docker/src/compose.override.yaml diff --git a/deployment/docker/src/compose.yaml b/deployment/docker/src/compose.yaml new file mode 100644 index 0000000..320ceb6 --- /dev/null +++ b/deployment/docker/src/compose.yaml @@ -0,0 +1,7 @@ +version: "3.9" +name: "knightcrawler" +include: + - ./components/network.yaml + - ./components/volumes.yaml + - ./components/infrastructure.yaml + - ./components/knightcrawler.yaml \ No newline at end of file