diff --git a/.gitignore b/.gitignore index 0c759aa..cd4d1ba 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ .now .DS_Store .idea -.env +**/.env ## Ignore Visual Studio temporary files, build results, and ## files generated by popular Visual Studio add-ons. diff --git a/README.md b/README.md index e2aa27d..5e649e2 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,7 @@ We can search DebridMediaManager hash lists which are hosted on GitHub. This all Before running the project, you need to set up the environment variables. Copy the `.env.example` file to `.env`: ```sh +cd deployment/docker cp .env.example .env ``` @@ -77,6 +78,7 @@ Then set any of th values you'd like to customize. Open a terminal in the directory and run the command: ```sh +cd deployment/docker docker compose up -d ``` @@ -99,6 +101,7 @@ Here's how to set up and use Grafana and Prometheus for monitoring RabbitMQ: 1. **Start Grafana and Prometheus**: Run the following command to start both Grafana and Prometheus: ```sh + cd deployment/docker docker compose -f docker-compose-metrics.yml up -d ``` diff --git a/.env.example b/deployment/docker/.env.example similarity index 100% rename from .env.example rename to deployment/docker/.env.example diff --git a/deployment/docker/config/grafana/dashboards/dashboards.yml b/deployment/docker/config/grafana/dashboards/dashboards.yml new file mode 100644 index 0000000..131310a --- /dev/null +++ b/deployment/docker/config/grafana/dashboards/dashboards.yml @@ -0,0 +1,8 @@ +apiVersion: 1 + +providers: + - name: Knightcrawler + folder: Dashboards + type: file + options: + path: /var/lib/grafana/dashboards \ No newline at end of file diff --git a/deployment/docker/config/grafana/dashboards/logs.json b/deployment/docker/config/grafana/dashboards/logs.json new file mode 100644 index 0000000..3b96f95 --- /dev/null +++ b/deployment/docker/config/grafana/dashboards/logs.json @@ -0,0 +1,581 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Logs for Producer and Consumer", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 4, + "panels": [], + "title": "Consumer Data", + "type": "row" + }, + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "fieldMinMax": false, + "mappings": [], + "min": 0, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "none", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "text": {} + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "builder", + "expr": "sum(count_over_time({container=~\"knightcrawler-consumer-[0-9]+\"} | json | __error__=`` | level > 30 [15m]))", + "queryType": "range", + "refId": "ConsumerErrorsLast15m" + } + ], + "title": "Errors - Last 15m", + "transparent": true, + "type": "gauge" + }, + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "code", + "expr": "sum(count_over_time({container=~\"knightcrawler-consumer-[0-9]+\"} | json | __error__=`` | level = `30` [15m]))", + "queryType": "range", + "refId": "InfoLogsLast15mSum" + } + ], + "title": "Info Logs - Last 15m", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false, + "minWidth": 50 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unitScale": true + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "level" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + }, + { + "id": "custom.align", + "value": "center" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.width", + "value": 200 + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 5, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "code", + "expr": "{container=~\"knightcrawler-consumer-[0-9]+\"} | json | __error__=``", + "queryType": "range", + "refId": "A" + } + ], + "title": "Log entries", + "transformations": [ + { + "id": "extractFields", + "options": { + "format": "json", + "jsonPaths": [ + { + "alias": "level", + "path": "level" + }, + { + "alias": "Message", + "path": "msg" + } + ], + "keepTime": true, + "replace": true, + "source": "Line" + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 6, + "panels": [], + "title": "Producer Data", + "type": "row" + }, + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "fieldMinMax": false, + "mappings": [], + "min": 0, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "none", + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 7, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "text": {} + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "code", + "expr": "sum(\n count_over_time({container=~\"knightcrawler-producer-[0-9]+\"} |= \"[Warning]\" [15m])\n) \n+\nsum(\n count_over_time({container=~\"knightcrawler-producer-[0-9]+\"} |= \"[Error]\" [15m])\n)", + "queryType": "range", + "refId": "ConsumerErrorsLast15m" + } + ], + "title": "Errors and Warnings - Last 15m", + "transparent": true, + "type": "gauge" + }, + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unitScale": true + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 19 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "code", + "expr": "sum(\n count_over_time({container=~\"knightcrawler-producer-[0-9]+\"} |= \"[Information]\" [15m])\n) ", + "queryType": "range", + "refId": "InfoLogsLast15mSum" + } + ], + "title": "Info Logs - Last 15m", + "transparent": true, + "type": "stat" + }, + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false, + "minWidth": 50 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unitScale": true + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "labels" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "tsNs" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "id" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.width", + "value": 200 + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 9, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "code", + "expr": "{container=~\"knightcrawler-producer-[0-9]+\"}", + "queryType": "range", + "refId": "A" + } + ], + "title": "Log entries", + "transformations": [], + "type": "table" + } + ], + "refresh": "10s", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Knightcrawler Logs", + "uid": "knightcrawler-logs", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/deployment/docker/config/grafana/datasources/loki-datasource.yml b/deployment/docker/config/grafana/datasources/loki-datasource.yml new file mode 100644 index 0000000..a8ad746 --- /dev/null +++ b/deployment/docker/config/grafana/datasources/loki-datasource.yml @@ -0,0 +1,12 @@ +apiVersion: 1 +datasources: + - name: Loki + uid: loki + type: loki + access: proxy + orgId: 1 + url: http://loki:3100 + basicAuth: false + isDefault: false + version: 1 + editable: true \ No newline at end of file diff --git a/grafana/datasource.yml b/deployment/docker/config/grafana/datasources/prometheus-datasource.yml similarity index 80% rename from grafana/datasource.yml rename to deployment/docker/config/grafana/datasources/prometheus-datasource.yml index 7e63fb7..8de3f79 100644 --- a/grafana/datasource.yml +++ b/deployment/docker/config/grafana/datasources/prometheus-datasource.yml @@ -1,7 +1,8 @@ apiVersion: 1 datasources: - - name: RabbitMQ Prometheus + - name: Prometheus type: prometheus + uid: prometheus access: proxy orgId: 1 url: "http://prometheus:9090" diff --git a/deployment/docker/config/loki/config.yml b/deployment/docker/config/loki/config.yml new file mode 100644 index 0000000..c8efd11 --- /dev/null +++ b/deployment/docker/config/loki/config.yml @@ -0,0 +1,35 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + +limits_config: + split_queries_by_interval: 24h + max_query_parallelism: 100 + +query_scheduler: + max_outstanding_requests_per_tenant: 4096 + +frontend: + max_outstanding_per_tenant: 4096 + +common: + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2020-10-24 + store: boltdb-shipper + object_store: filesystem + schema: v11 + index: + prefix: index_ + period: 24h \ No newline at end of file diff --git a/prometheus/prometheus.yml b/deployment/docker/config/prometheus/config.yml similarity index 100% rename from prometheus/prometheus.yml rename to deployment/docker/config/prometheus/config.yml diff --git a/deployment/docker/config/promtail/config.yml b/deployment/docker/config/promtail/config.yml new file mode 100644 index 0000000..05428d1 --- /dev/null +++ b/deployment/docker/config/promtail/config.yml @@ -0,0 +1,27 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + tenant_id: knightcrawler + +scrape_configs: + - job_name: scraper + docker_sd_configs: + - host: unix:///var/run/docker.sock + refresh_interval: 15s + filters: + - name: label + values: ["logging=promtail","com.docker.compose.project=knightcrawler"] + relabel_configs: + - source_labels: ["__meta_docker_container_name"] + regex: "/(.*)" + target_label: "container" + - source_labels: ["__meta_docker_container_log_stream"] + target_label: "logstream" + - source_labels: ["__meta_docker_container_label_logging_jobname"] + target_label: "job" diff --git a/deployment/docker/docker-compose-metrics.yml b/deployment/docker/docker-compose-metrics.yml new file mode 100644 index 0000000..a5d4d49 --- /dev/null +++ b/deployment/docker/docker-compose-metrics.yml @@ -0,0 +1,74 @@ +version: '3.8' +name: knightcrawler-metrics + +services: + prometheus: + image: prom/prometheus:v2.20.1 + volumes: + - ./config/prometheus/config.yml:/etc/prometheus/prometheus.yml + command: + - '--config.file=/etc/prometheus/prometheus.yml' + ports: + - "9090:9090" + networks: + - knightcrawler-network + + grafana: + image: grafana/grafana:latest + volumes: + - ./config/grafana/datasources:/etc/grafana/provisioning/datasources + - ./config/grafana/dashboards/dashboards.yml:/etc/grafana/provisioning/dashboards/dashboards.yml + - ./config/grafana/dashboards/logs.json:/var/lib/grafana/dashboards/logs.json + - grafana-data:/var/lib/grafana + ports: + - "3000:3000" + environment: + - GF_PATHS_PROVISIONING=/etc/grafana/provisioning + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=admin_password + depends_on: + - prometheus + networks: + - knightcrawler-network + + postgres-exporter: + image: prometheuscommunity/postgres-exporter + env_file: + - .env + environment: + DATA_SOURCE_NAME: "postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}?sslmode=disable" + networks: + - knightcrawler-network + + promtail: + image: grafana/promtail:2.9.4 + volumes: + - /var/lib/docker/containers:/var/lib/docker/containers:ro + - /var/run/docker.sock:/var/run/docker.sock + - ./config/promtail/config.yml:/etc/promtail/config.yml + command: -config.file=/etc/promtail/config.yml + depends_on: + - prometheus + - loki + networks: + - knightcrawler-network + + loki: + image: grafana/loki:2.9.4 + command: -config.file=/etc/loki/local-config.yml + depends_on: + - prometheus + - grafana + volumes: + - loki-data:/loki + - ./config/loki/config.yml:/etc/loki/local-config.yml + networks: + - knightcrawler-network + +volumes: + loki-data: + grafana-data: + +networks: + knightcrawler-network: + external: true diff --git a/docker-compose.yaml b/deployment/docker/docker-compose.yaml similarity index 96% rename from docker-compose.yaml rename to deployment/docker/docker-compose.yaml index 019b0f1..3d45bf2 100644 --- a/docker-compose.yaml +++ b/deployment/docker/docker-compose.yaml @@ -84,6 +84,8 @@ services: build: context: src/producer dockerfile: Dockerfile + labels: + logging: "promtail" env_file: - .env <<: *knightcrawler-app @@ -96,6 +98,8 @@ services: dockerfile: Dockerfile env_file: - .env + labels: + logging: "promtail" deploy: replicas: 3 <<: *knightcrawler-app @@ -108,6 +112,8 @@ services: dockerfile: Dockerfile ports: - "7000:7000" + labels: + logging: "promtail" env_file: - .env <<: *knightcrawler-app diff --git a/docker-compose-metrics.yml b/docker-compose-metrics.yml deleted file mode 100644 index 709c979..0000000 --- a/docker-compose-metrics.yml +++ /dev/null @@ -1,39 +0,0 @@ -version: '3.8' -name: knightcrawler-metrics - -services: - prometheus: - image: prom/prometheus:v2.20.1 - volumes: - - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml - command: - - '--config.file=/etc/prometheus/prometheus.yml' - ports: - - "9090:9090" - networks: - - knightcrawler-network - - grafana: - image: grafana/grafana:latest - volumes: - - ./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasources.yml - ports: - - "3000:3000" - environment: - - GF_SECURITY_ADMIN_USER=admin - - GF_SECURITY_ADMIN_PASSWORD=admin_password - depends_on: - - prometheus - networks: - - knightcrawler-network - - postgres-exporter: - image: prometheuscommunity/postgres-exporter - environment: - DATA_SOURCE_NAME: "postgresql://postgres:postgres@postgres:5432/knightcrawler?sslmode=disable" - networks: - - knightcrawler-network - -networks: - knightcrawler-network: - external: true