From daafeb79b75d93a4e8c2ab1aae381f3381946f18 Mon Sep 17 00:00:00 2001 From: Brock H Caldwell Date: Sun, 18 May 2025 19:48:10 -0500 Subject: [PATCH] fix: container DB race condition, feat: separate images for app & worker, chore: example compose & .env files --- .env.dev.dist | 9 --- .env.dist | 29 ------- Dockerfile.prod | 3 - bash/build/build_image_linux.sh | 3 +- bash/build/build_image_macos.sh | 3 +- bash/build/push_image.sh | 3 +- config/packages/cache.yaml | 2 +- config/services.yaml | 8 ++ docker/Dockerfile.app | 16 ++++ docker/Dockerfile.worker | 12 +++ .env.example.dist => docs/examples/.env | 14 ++-- docs/examples/compose.yml | 93 +++++++++++++++++++++ example.compose.yml | 102 ------------------------ src/Command/StartupStatusCommand.php | 48 +++++++++++ 14 files changed, 191 insertions(+), 154 deletions(-) delete mode 100644 .env.dev.dist delete mode 100644 .env.dist create mode 100644 docker/Dockerfile.app create mode 100644 docker/Dockerfile.worker rename .env.example.dist => docs/examples/.env (74%) create mode 100644 docs/examples/compose.yml delete mode 100644 example.compose.yml create mode 100644 src/Command/StartupStatusCommand.php diff --git a/.env.dev.dist b/.env.dev.dist deleted file mode 100644 index 19dc9e2..0000000 --- a/.env.dev.dist +++ /dev/null @@ -1,9 +0,0 @@ -APP_URL="https://dev.caldwell.digital" -DATABASE_URL="mysql://root:password@database:3306/app?serverVersion=10.6.19.2-MariaDB&charset=utf8mb4" -APP_SECRET=70169beadfbc8101c393cbfbba27a313 - -DOWNLOAD_DIR=./movies -MERCURE_URL=http://mercure/.well-known/mercure -MERCURE_PUBLIC_URL=https://dev.caldwell.digital/hub/.well-known/mercure -MERCURE_JWT_SECRET="!ChangeThisMercureHubJWTSecretKey!" -MONITOR_FREQUENCY="* * * * *" diff --git a/.env.dist b/.env.dist deleted file mode 100644 index ed8f1cc..0000000 --- a/.env.dist +++ /dev/null @@ -1,29 +0,0 @@ -APP_ENV=%%app_env%% -APP_SECRET="%%app_secret%%" -DATABASE_URL="%%db_url%%" -DOWNLOAD_DIR=%%download_dir%% -REAL_DEBRID_KEY="%%rd_key%%" -TMDB_API=%%tmdb_api%% -MERCURE_URL=%%mercure_url%% -MERCURE_PUBLIC_URL=%%mercure_public_url%% -MERCURE_JWT_SECRET="%%mercure_jwt_secret%%" -REDIS_HOST="%%redis_host%%" - - - -LDAP_HOST= -LDAP_PORT= -LDAP_ENCRYPTION= -LDAP_BASE_DN= -LDAP_BIND_USER= -LDAP_BIND_PASS= -LDAP_DN_STRING= -LDAP_UID_KEY="uid" - -# LDAP group that identifies an Admin -# Users with this LDAP group will automatically -# get the admin role in this system. -LDAP_ADMIN_ROLE_DN="" -LDAP_EMAIL_ATTRIBUTE=mail -LDAP_USERNAME_ATTRIBUTE=uid -LDAP_NAME_ATTRIBUTE=displayname diff --git a/Dockerfile.prod b/Dockerfile.prod index c0c0bf2..1f906e9 100644 --- a/Dockerfile.prod +++ b/Dockerfile.prod @@ -7,8 +7,5 @@ RUN apt-get update && \ docker-php-ext-install ldap COPY --chown=www-data:www-data . /var/www -COPY --chmod=0775 ./bash/entrypoint.sh /usr/local/bin/ COPY ./bash/vhost.conf /etc/apache2/sites-enabled/vhost.conf RUN rm /etc/apache2/sites-enabled/000-default.conf - -ENTRYPOINT [ "/usr/local/bin/entrypoint.sh" ] diff --git a/bash/build/build_image_linux.sh b/bash/build/build_image_linux.sh index 2de634c..503bc8e 100755 --- a/bash/build/build_image_linux.sh +++ b/bash/build/build_image_linux.sh @@ -1,2 +1,3 @@ echo "> Building ${IMG} for linux/amd64" -docker buildx build --platform linux/amd64 -f Dockerfile.prod -t ${IMG} . +docker buildx build --platform linux/amd64 -f Dockerfile.app -t "${IMG}-app" . +docker buildx build --platform linux/amd64 -f Dockerfile.worker -t "${IMG}-worker" . diff --git a/bash/build/build_image_macos.sh b/bash/build/build_image_macos.sh index 76a3c90..dcda8f8 100755 --- a/bash/build/build_image_macos.sh +++ b/bash/build/build_image_macos.sh @@ -1,2 +1,3 @@ echo "> Building ${IMG} for linux/arm/v8" -docker buildx build --platform linux/arm/v8 -f Dockerfile.prod -t ${IMG} . \ No newline at end of file +docker buildx build --platform linux/arm/v8 -f Dockerfile.app -t "${IMG}-app" . +docker buildx build --platform linux/arm/v8 -f Dockerfile.app -t "${IMG}-worker" . \ No newline at end of file diff --git a/bash/build/push_image.sh b/bash/build/push_image.sh index cfe1402..717e061 100755 --- a/bash/build/push_image.sh +++ b/bash/build/push_image.sh @@ -1,2 +1,3 @@ echo "> Pushing ${IMG}" -docker push ${IMG} \ No newline at end of file +docker push "${IMG}-app" +docker push "${IMG}-worker" \ No newline at end of file diff --git a/config/packages/cache.yaml b/config/packages/cache.yaml index d166b27..c554045 100644 --- a/config/packages/cache.yaml +++ b/config/packages/cache.yaml @@ -9,7 +9,7 @@ framework: # Redis app: cache.adapter.redis - default_redis_provider: '%env(REDIS_HOST)%' + default_redis_provider: '%app.cache.redis.host%' # APCu (not recommended with heavy random-write workloads as memory fragmentation can cause perf issues) #app: cache.adapter.apcu diff --git a/config/services.yaml b/config/services.yaml index 2b97e31..06debfe 100644 --- a/config/services.yaml +++ b/config/services.yaml @@ -4,14 +4,22 @@ # Put parameters here that don't need to change on each machine where the app is deployed # https://symfony.com/doc/current/best_practices.html#use-parameters-for-application-configuration parameters: + # Media media.default_movies_dir: movies media.default_tvshows_dir: tvshows media.movies_path: '/var/download/%env(default:media.default_movies_dir:MOVIES_PATH)%' media.tvshows_path: '/var/download/%env(default:media.default_tvshows_dir:TVSHOWS_PATH)%' + # Mercure app.mercure.url: 'http://mercure/.well-known/mercure' app.mercure.public_url: '%env(APP_URL)%/hub/.well-known/mercure' + # Cache + app.cache.adapter: '%env(default:app.cache.adapter.default:CACHE_ADAPTER)%' + app.cache.redis.host: '%env(default:app.cache.redis.host.default:REDIS_HOST)%' + app.cache.adapter.default: 'filesystem' + app.cache.redis.host.default: 'redis://redis' + services: # default configuration for services in *this* file _defaults: diff --git a/docker/Dockerfile.app b/docker/Dockerfile.app new file mode 100644 index 0000000..f26e838 --- /dev/null +++ b/docker/Dockerfile.app @@ -0,0 +1,16 @@ +FROM registry.caldwell.digital/library/php:8.4-apache + +RUN apt-get update && \ + apt-get install libldap2-dev -y && \ + rm -rf /var/lib/apt/lists/* && \ + docker-php-ext-configure ldap --with-libdir=lib/x86_64-linux-gnu/ && \ + docker-php-ext-install ldap + +COPY --chown=www-data:www-data . /var/www +COPY --chmod=0775 ./bash/entrypoint.sh /usr/local/bin/ +COPY ./bash/vhost.conf /etc/apache2/sites-enabled/vhost.conf +RUN rm /etc/apache2/sites-enabled/000-default.conf + +HEALTHCHECK --interval=5s --timeout=5s --retries=5 CMD [ "php", "/var/www/bin/console", "startup:status" ] + +ENTRYPOINT [ "/usr/local/bin/entrypoint.sh" ] diff --git a/docker/Dockerfile.worker b/docker/Dockerfile.worker new file mode 100644 index 0000000..b3dfa86 --- /dev/null +++ b/docker/Dockerfile.worker @@ -0,0 +1,12 @@ +FROM registry.caldwell.digital/library/php:8.4-apache + +RUN apt-get update && \ + apt-get install libldap2-dev -y && \ + rm -rf /var/lib/apt/lists/* && \ + docker-php-ext-configure ldap --with-libdir=lib/x86_64-linux-gnu/ && \ + docker-php-ext-install ldap + +COPY --chown=www-data:www-data . /var/www + +COPY ./bash/vhost.conf /etc/apache2/sites-enabled/vhost.conf +RUN rm /etc/apache2/sites-enabled/000-default.conf diff --git a/.env.example.dist b/docs/examples/.env similarity index 74% rename from .env.example.dist rename to docs/examples/.env index ba97059..1d0eeff 100644 --- a/.env.example.dist +++ b/docs/examples/.env @@ -1,9 +1,9 @@ # App must be served over HTTPS (requirement of Mercure) # Either serve behind an SSL terminating reverse proxy # or pass your certificates into the 'app' container. -# Please omit any trailing slashes. The APP_URL is passed +# Please omit any trailing slashes. The APP_URL is # used to generate the Mercure URL behind the scenes. -APP_URL="https://dev.caldwell.digital" +APP_URL="https://torsearch-test.caldwell.digital" APP_SECRET="70169beadfbc8101c393cbfbba27a313" # Use the DATABASE_URL below to use the MariaDB container @@ -14,23 +14,23 @@ DATABASE_URL="mysql://root:password@database:3306/app?serverVersion=10.6.19.2-Ma # Fill in your MySQL/MariaDB connection details #DATABASE_URL="mysql://:@:3306/?serverVersion=10.6.19.2-MariaDB&charset=utf8mb4" -# Enter you Real Debrid API key +# Enter your Real Debrid API key # This key is never saved anywhere # else and is passed to Torrentio # to retrieve download options -REAL_DEBRID_KEY="QYYBR7OSQ4VEFKWASDEZ2B4VO67KHUJY6IWOT7HHA7ATXO7QCYDQ" +REAL_DEBRID_KEY="" # Enter you TMDB API key # This is used to provide rich search results # when searching for media and rendering the # Popular Movies and TV Shows section. -TMDB_API="eyJhbGciOiJIUzI1NiJ9.eyJhdWQiOiI0ZTJjYjJhOGUzOGJhNjdiNjVhOGU1NGM0ZWI1MzhmOCIsIm5iZiI6MTczNzkyNjA0NC41NjQsInN1YiI6IjY3OTZhNTljYzdiMDFiNzJjNzIzZWM5YiIsInNjb3BlcyI6WyJhcGlfcmVhZCJdLCJ2ZXJzaW9uIjoxfQ.e8DbNe9qrSBC1y-ANRv-VWBAtls-ZS2r7aNCiI68mpw" +TMDB_API="" MERCURE_JWT_SECRET="!ChangeThisMercureHubJWTSecretKey!" -# Use your own Redis instance to use the +# Use your own Redis instance or use the # below value to use the container included -# in the example.compose.yml file. +# in the example compose.yml file. REDIS_HOST="redis://redis" # LDAP Config: To use LDAP, enter the below fields diff --git a/docs/examples/compose.yml b/docs/examples/compose.yml new file mode 100644 index 0000000..605b0ee --- /dev/null +++ b/docs/examples/compose.yml @@ -0,0 +1,93 @@ +services: + # This container runs the actual web app in a php:8.4-apache + # base container. If not running behind a reverse proxy, + # inject your SSL certificates into this container + app: + image: registry.caldwell.digital/home/torsearch:test-app + ports: + - "8006:80" + env_file: + - .env + depends_on: + database: + condition: service_healthy + + # Downloads happen asynchronously in this container. Replicate + # this container to run multiple downloads simultaneously. + # Map your "movies" folder to /var/download/movies + # Map your TV shows folder to /var/download/tvshows + # If your folders are on another machine, use an NFS volume. + # This container runs a Symfony worker process. + # See: https://symfony.com/doc/current/messenger.html + worker: + image: registry.caldwell.digital/home/torsearch:test-worker + volumes: + - ./downloads/movies:/var/download/movies + - ./downloads/tvshows:/var/download/tvshows + command: php ./bin/console messenger:consume async -v --time-limit=3600 --limit=10 + env_file: + - .env + depends_on: + app: + condition: service_healthy + + # This container handles the monitoring for new media. When new + # monitors are added, jobs are periodically dispatched to this + # container, and the desired media is searched for and downloaded. + # This container runs a Symfony worker process. + # See: https://symfony.com/doc/current/messenger.html + scheduler: + image: registry.caldwell.digital/home/torsearch:test-worker + volumes: + - ./downloads:/var/download + command: php ./bin/console messenger:consume scheduler_monitor -vv --time-limit=3600 + env_file: + - .env + depends_on: + app: + condition: service_healthy + + # This container facilitates viewing the progress of downloads + # in realtime. It also handles sending alerts and notifications. + # The MERCURE_PUBLISHER_JWT key & MERCURE_SUBSCRIBER_JWT_KEY should + # match the MERCURE_JWT_SECRET environment variable. + mercure: + image: dunglas/mercure + restart: unless-stopped + ports: + - "3001:80" + environment: + SERVER_NAME: ':80' + MERCURE_PUBLISHER_JWT_KEY: '!ChangeThisMercureHubJWTSecretKey!' + MERCURE_SUBSCRIBER_JWT_KEY: '!ChangeThisMercureHubJWTSecretKey!' + MERCURE_EXTRA_DIRECTIVES: | + cors_origins * + anonymous + command: /usr/bin/caddy run --config /etc/caddy/dev.Caddyfile + volumes: + - mercure_data:/data + - mercure_config:/config + + database: + image: mariadb:10.11.2 + volumes: + - mysql:/var/lib/mysql + environment: + MYSQL_DATABASE: app + MYSQL_USERNAME: app + MYSQL_PASSWORD: password + MYSQL_ROOT_PASSWORD: password + healthcheck: + test: [ "CMD", "mysqladmin" ,"ping", "-h", "localhost" ] + timeout: 10s + retries: 10 + + adminer: + image: adminer + ports: + - "8081:8080" + +volumes: + mysql: + mercure_config: + mercure_data: diff --git a/example.compose.yml b/example.compose.yml deleted file mode 100644 index 2146096..0000000 --- a/example.compose.yml +++ /dev/null @@ -1,102 +0,0 @@ -services: - caddy: - image: caddy:2.9.1 - restart: unless-stopped - cap_add: - - NET_ADMIN - ports: - - "80:80" - - "443:443" - - "443:443/udp" - volumes: - - $PWD/bash/caddy:/etc/caddy - - $PWD/bash/certs:/etc/ssl - - app: - image: torsearch_test - ports: - - "8001:80" - environment: - APP_ENV: dev - APP_URL: ${APP_URL} - APP_SECRET: ${APP_SECRET} - DATABASE_URL: ${DATABASE_URL} - READ_DEBRID_KEY: ${REAL_DEBRID_KEY} - TMDB_API: ${TMDB_API} - MERCURE_JWT_SECRET: ${MERCURE_JWT_SECRET} - REDIS_HOST: ${REDIS_HOST} - - LDAP_HOST: directory.caldwell.local - LDAP_PORT: 389 - LDAP_ENCRYPTION: none - LDAP_BASE_DN: "dc=caldwell,dc=local" - LDAP_BIND_USER: "uid=admin,cn=users,cn=accounts,dc=caldwell,dc=local" - LDAP_BIND_PASS: "Caldwell.24272911" - LDAP_DN_STRING: "uid={user_identifier},cn=users,cn=accounts,dc=caldwell,dc=local" - LDAP_UID_KEY: "uid" - LDAP_ADMIN_ROLE_DN: "" - LDAP_EMAIL_ATTRIBUTE: mail - LDAP_USERNAME_ATTRIBUTE: uid - LDAP_NAME_ATTRIBUTE: displayname - - worker: - image: torsearch_test - volumes: - - ./downloads/:/var/download - command: php ./bin/console messenger:consume async -v --time-limit=3600 --limit=10 - environment: - APP_URL: ${APP_URL} - APP_SECRET: ${APP_SECRET} - DATABASE_URL: ${DATABASE_URL} - READ_DEBRID_KEY: ${REAL_DEBRID_KEY} - TMDB_API: ${TMDB_API} - MERCURE_JWT_SECRET: ${MERCURE_JWT_SECRET} - REDIS_HOST: ${REDIS_HOST} - - scheduler: - image: torsearch_test - volumes: - - ./downloads:/var/download - command: php ./bin/console messenger:consume scheduler_monitor -vv --time-limit=3600 - environment: - APP_URL: ${APP_URL} - APP_SECRET: ${APP_SECRET} - DATABASE_URL: ${DATABASE_URL} - READ_DEBRID_KEY: ${REAL_DEBRID_KEY} - TMDB_API: ${TMDB_API} - MERCURE_JWT_SECRET: ${MERCURE_JWT_SECRET} - REDIS_HOST: ${REDIS_HOST} - - mercure: - image: dunglas/mercure - restart: unless-stopped - ports: - - "3000:80" - environment: - SERVER_NAME: ':80' - MERCURE_PUBLISHER_JWT_KEY: ${MERCURE_JWT_SECRET} - MERCURE_SUBSCRIBER_JWT_KEY: ${MERCURE_JWT_SECRET} - MERCURE_EXTRA_DIRECTIVES: | - cors_origins * - anonymous - command: /usr/bin/caddy run --config /etc/caddy/dev.Caddyfile - volumes: - - mercure_data:/data - - mercure_config:/config - - database: - image: mariadb:10.11.2 - ports: - - "3306:3306" - volumes: - - mysql:/var/lib/mysql - environment: - MYSQL_DATABASE: app - MYSQL_USERNAME: app - MYSQL_PASSWORD: password - MYSQL_ROOT_PASSWORD: password - -volumes: - mysql: - mercure_config: - mercure_data: diff --git a/src/Command/StartupStatusCommand.php b/src/Command/StartupStatusCommand.php new file mode 100644 index 0000000..95b468e --- /dev/null +++ b/src/Command/StartupStatusCommand.php @@ -0,0 +1,48 @@ +entityManager = $entityManager; + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $io = new SymfonyStyle($input, $output); + + $ready = true; + $messengerTableExists = $this->doesMessengerTableExist(); + + if (false === $messengerTableExists) { + $ready = false; + } + + return (true === $ready) ? Command::SUCCESS : Command::FAILURE; + } + + private function doesMessengerTableExist() + { + return $this->entityManager->getConnection() + ->createSchemaManager() + ->tablesExist('messenger_messages'); + } +}