tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Logging expansion

Browse Source 
Adds Promtail and Loki to the metrics stack, configuring them as a datasource in grafana on deployment
Adds a dashboard to monitor the system (logs for producer and consumer)
Restructures the base directory so deployment manifests are moved into a deployments folder
Adds selective labels to producer, consumer and addon so that their logs are ingested by promtail
This commit is contained in:
iPromKnight
2024-02-04 19:50:28 +00:00
parent eef3265850
commit b773ffcb20
13 changed files with 749 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@@ -2,7 +2,7 @@
.now
.DS_Store
.idea
.env
**/.env
## Ignore Visual Studio temporary files, build results, and
## files generated by popular Visual Studio add-ons.

3
README.md
View File

@@ -67,6 +67,7 @@ We can search DebridMediaManager hash lists which are hosted on GitHub. This all
Before running the project, you need to set up the environment variables. Copy the `.env.example` file to `.env`:
```sh
cd deployment/docker
cp .env.example .env
```
@@ -77,6 +78,7 @@ Then set any of th values you'd like to customize.
Open a terminal in the directory and run the command:
```sh
cd deployment/docker
docker compose up -d
```
@@ -99,6 +101,7 @@ Here's how to set up and use Grafana and Prometheus for monitoring RabbitMQ:
1. **Start Grafana and Prometheus**: Run the following command to start both Grafana and Prometheus:
```sh
cd deployment/docker
docker compose -f docker-compose-metrics.yml up -d
```

0
.env.example → deployment/docker/.env.example
View File

8
deployment/docker/config/grafana/dashboards/dashboards.yml Normal file
View File

@@ -0,0 +1,8 @@
apiVersion: 1
providers:
- name: Knightcrawler
folder: Dashboards
type: file
options:
path: /var/lib/grafana/dashboards

581
deployment/docker/config/grafana/dashboards/logs.json Normal file
View File

@@ -0,0 +1,581 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"description": "Logs for Producer and Consumer",
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 2,
"links": [],
"liveNow": false,
"panels": [
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 4,
"panels": [],
"title": "Consumer Data",
"type": "row"
},
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"decimals": 0,
"fieldMinMax": false,
"mappings": [],
"min": 0,
"noValue": "0",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 1
}
]
},
"unit": "none",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 12,
"x": 0,
"y": 1
},
"id": 1,
"options": {
"minVizHeight": 75,
"minVizWidth": 75,
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showThresholdLabels": false,
"showThresholdMarkers": true,
"sizing": "auto",
"text": {}
},
"pluginVersion": "10.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"editorMode": "builder",
"expr": "sum(count_over_time({container=~\"knightcrawler-consumer-[0-9]+\"} | json | __error__=`` | level > 30 [15m]))",
"queryType": "range",
"refId": "ConsumerErrorsLast15m"
}
],
"title": "Errors - Last 15m",
"transparent": true,
"type": "gauge"
},
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 12,
"x": 12,
"y": 1
},
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "10.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"editorMode": "code",
"expr": "sum(count_over_time({container=~\"knightcrawler-consumer-[0-9]+\"} | json | __error__=`` | level = `30` [15m]))",
"queryType": "range",
"refId": "InfoLogsLast15mSum"
}
],
"title": "Info Logs - Last 15m",
"transparent": true,
"type": "stat"
},
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "left",
"cellOptions": {
"type": "auto"
},
"filterable": true,
"inspect": false,
"minWidth": 50
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unitScale": true
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "level"
},
"properties": [
{
"id": "custom.width",
"value": 100
},
{
"id": "custom.align",
"value": "center"
}
]
},
{
"matcher": {
"id": "byName",
"options": "Time"
},
"properties": [
{
"id": "custom.width",
"value": 200
}
]
}
]
},
"gridPos": {
"h": 12,
"w": 24,
"x": 0,
"y": 6
},
"id": 5,
"options": {
"cellHeight": "sm",
"footer": {
"countRows": false,
"enablePagination": false,
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true
},
"pluginVersion": "10.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"editorMode": "code",
"expr": "{container=~\"knightcrawler-consumer-[0-9]+\"} | json | __error__=``",
"queryType": "range",
"refId": "A"
}
],
"title": "Log entries",
"transformations": [
{
"id": "extractFields",
"options": {
"format": "json",
"jsonPaths": [
{
"alias": "level",
"path": "level"
},
{
"alias": "Message",
"path": "msg"
}
],
"keepTime": true,
"replace": true,
"source": "Line"
}
}
],
"type": "table"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 18
},
"id": 6,
"panels": [],
"title": "Producer Data",
"type": "row"
},
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"decimals": 0,
"fieldMinMax": false,
"mappings": [],
"min": 0,
"noValue": "0",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 1
}
]
},
"unit": "none",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 12,
"x": 0,
"y": 19
},
"id": 7,
"options": {
"minVizHeight": 75,
"minVizWidth": 75,
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showThresholdLabels": false,
"showThresholdMarkers": true,
"sizing": "auto",
"text": {}
},
"pluginVersion": "10.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"editorMode": "code",
"expr": "sum(\n count_over_time({container=~\"knightcrawler-producer-[0-9]+\"} |= \"[Warning]\" [15m])\n) \n+\nsum(\n count_over_time({container=~\"knightcrawler-producer-[0-9]+\"} |= \"[Error]\" [15m])\n)",
"queryType": "range",
"refId": "ConsumerErrorsLast15m"
}
],
"title": "Errors and Warnings - Last 15m",
"transparent": true,
"type": "gauge"
},
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 12,
"x": 12,
"y": 19
},
"id": 8,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "10.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"editorMode": "code",
"expr": "sum(\n count_over_time({container=~\"knightcrawler-producer-[0-9]+\"} |= \"[Information]\" [15m])\n) ",
"queryType": "range",
"refId": "InfoLogsLast15mSum"
}
],
"title": "Info Logs - Last 15m",
"transparent": true,
"type": "stat"
},
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "left",
"cellOptions": {
"type": "auto"
},
"filterable": true,
"inspect": false,
"minWidth": 50
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unitScale": true
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "labels"
},
"properties": [
{
"id": "custom.hidden",
"value": true
}
]
},
{
"matcher": {
"id": "byName",
"options": "tsNs"
},
"properties": [
{
"id": "custom.hidden",
"value": true
}
]
},
{
"matcher": {
"id": "byName",
"options": "id"
},
"properties": [
{
"id": "custom.hidden",
"value": true
}
]
},
{
"matcher": {
"id": "byName",
"options": "Time"
},
"properties": [
{
"id": "custom.width",
"value": 200
}
]
}
]
},
"gridPos": {
"h": 12,
"w": 24,
"x": 0,
"y": 24
},
"id": 9,
"options": {
"cellHeight": "sm",
"footer": {
"countRows": false,
"enablePagination": false,
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true
},
"pluginVersion": "10.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"editorMode": "code",
"expr": "{container=~\"knightcrawler-producer-[0-9]+\"}",
"queryType": "range",
"refId": "A"
}
],
"title": "Log entries",
"transformations": [],
"type": "table"
}
],
"refresh": "10s",
"schemaVersion": 39,
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-24h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Knightcrawler Logs",
"uid": "knightcrawler-logs",
"version": 1,
"weekStart": ""
}

12
deployment/docker/config/grafana/datasources/loki-datasource.yml Normal file
View File

@@ -0,0 +1,12 @@
apiVersion: 1
datasources:
- name: Loki
uid: loki
type: loki
access: proxy
orgId: 1
url: http://loki:3100
basicAuth: false
isDefault: false
version: 1
editable: true

3
grafana/datasource.yml → deployment/docker/config/grafana/datasources/prometheus-datasource.yml
View File

@@ -1,7 +1,8 @@
apiVersion: 1
datasources:
- name: RabbitMQ Prometheus
- name: Prometheus
type: prometheus
uid: prometheus
access: proxy
orgId: 1
url: "http://prometheus:9090"

35
deployment/docker/config/loki/config.yml Normal file
View File

@@ -0,0 +1,35 @@
auth_enabled: false
server:
http_listen_port: 3100
limits_config:
split_queries_by_interval: 24h
max_query_parallelism: 100
query_scheduler:
max_outstanding_requests_per_tenant: 4096
frontend:
max_outstanding_per_tenant: 4096
common:
path_prefix: /loki
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
replication_factor: 1
ring:
kvstore:
store: inmemory
schema_config:
configs:
- from: 2020-10-24
store: boltdb-shipper
object_store: filesystem
schema: v11
index:
prefix: index_
period: 24h

0
prometheus/prometheus.yml → deployment/docker/config/prometheus/config.yml
View File

27
deployment/docker/config/promtail/config.yml Normal file
View File

@@ -0,0 +1,27 @@
server:
http_listen_port: 9080
grpc_listen_port: 0
positions:
filename: /tmp/positions.yaml
clients:
- url: http://loki:3100/loki/api/v1/push
tenant_id: knightcrawler
scrape_configs:
- job_name: scraper
docker_sd_configs:
- host: unix:///var/run/docker.sock
refresh_interval: 15s
filters:
- name: label
values: ["logging=promtail","com.docker.compose.project=knightcrawler"]
relabel_configs:
- source_labels: ["__meta_docker_container_name"]
regex: "/(.*)"
target_label: "container"
- source_labels: ["__meta_docker_container_log_stream"]
target_label: "logstream"
- source_labels: ["__meta_docker_container_label_logging_jobname"]
target_label: "job"

74
deployment/docker/docker-compose-metrics.yml Normal file
View File

@@ -0,0 +1,74 @@
version: '3.8'
name: knightcrawler-metrics
services:
prometheus:
image: prom/prometheus:v2.20.1
volumes:
- ./config/prometheus/config.yml:/etc/prometheus/prometheus.yml
command:
- '--config.file=/etc/prometheus/prometheus.yml'
ports:
- "9090:9090"
networks:
- knightcrawler-network
grafana:
image: grafana/grafana:latest
volumes:
- ./config/grafana/datasources:/etc/grafana/provisioning/datasources
- ./config/grafana/dashboards/dashboards.yml:/etc/grafana/provisioning/dashboards/dashboards.yml
- ./config/grafana/dashboards/logs.json:/var/lib/grafana/dashboards/logs.json
- grafana-data:/var/lib/grafana
ports:
- "3000:3000"
environment:
- GF_PATHS_PROVISIONING=/etc/grafana/provisioning
- GF_SECURITY_ADMIN_USER=admin
- GF_SECURITY_ADMIN_PASSWORD=admin_password
depends_on:
- prometheus
networks:
- knightcrawler-network
postgres-exporter:
image: prometheuscommunity/postgres-exporter
env_file:
- .env
environment:
DATA_SOURCE_NAME: "postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}?sslmode=disable"
networks:
- knightcrawler-network
promtail:
image: grafana/promtail:2.9.4
volumes:
- /var/lib/docker/containers:/var/lib/docker/containers:ro
- /var/run/docker.sock:/var/run/docker.sock
- ./config/promtail/config.yml:/etc/promtail/config.yml
command: -config.file=/etc/promtail/config.yml
depends_on:
- prometheus
- loki
networks:
- knightcrawler-network
loki:
image: grafana/loki:2.9.4
command: -config.file=/etc/loki/local-config.yml
depends_on:
- prometheus
- grafana
volumes:
- loki-data:/loki
- ./config/loki/config.yml:/etc/loki/local-config.yml
networks:
- knightcrawler-network
volumes:
loki-data:
grafana-data:
networks:
knightcrawler-network:
external: true

6
docker-compose.yaml → deployment/docker/docker-compose.yaml
View File

@@ -84,6 +84,8 @@ services:
build:
context: src/producer
dockerfile: Dockerfile
labels:
logging: "promtail"
env_file:
- .env
<<: *knightcrawler-app
@@ -96,6 +98,8 @@ services:
dockerfile: Dockerfile
env_file:
- .env
labels:
logging: "promtail"
deploy:
replicas: 3
<<: *knightcrawler-app
@@ -108,6 +112,8 @@ services:
dockerfile: Dockerfile
ports:
- "7000:7000"
labels:
logging: "promtail"
env_file:
- .env
<<: *knightcrawler-app

39
docker-compose-metrics.yml
View File

@@ -1,39 +0,0 @@
version: '3.8'
name: knightcrawler-metrics
services:
prometheus:
image: prom/prometheus:v2.20.1
volumes:
- ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
command:
- '--config.file=/etc/prometheus/prometheus.yml'
ports:
- "9090:9090"
networks:
- knightcrawler-network
grafana:
image: grafana/grafana:latest
volumes:
- ./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasources.yml
ports:
- "3000:3000"
environment:
- GF_SECURITY_ADMIN_USER=admin
- GF_SECURITY_ADMIN_PASSWORD=admin_password
depends_on:
- prometheus
networks:
- knightcrawler-network
postgres-exporter:
image: prometheuscommunity/postgres-exporter
environment:
DATA_SOURCE_NAME: "postgresql://postgres:postgres@postgres:5432/knightcrawler?sslmode=disable"
networks:
- knightcrawler-network
networks:
knightcrawler-network:
external: true