tools/knightcrawler
1
0
Fork 0
mirror of https://github.com/knightcrawler-stremio/knightcrawler.git synced 2024-12-20 03:29:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Move optional services into sub directory

Browse Source
This commit is contained in:
purple_emily
2024-03-09 14:01:48 +00:00
committed by iPromKnight
parent 67e6f97e96
commit f79c49e472
14 changed files with 0 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
deployment/docker/optional-services/caddy/configs/Caddyfile Normal file
View File

@@ -0,0 +1,62 @@
{
## ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
## Let's Encrpyt staging environment
## ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
## Once you have confirmed Caddy works you should comment out
## the below line:
acme_ca https://acme-staging-v02.api.letsencrypt.org/director
}
(security-headers) {
## These are the recommended default settings in Caddy documentation
## https://caddyserver.com/docs/caddyfile/directives/header
header {
## disable FLoC tracking
Permissions-Policy "interest-cohort=()"
## enable HSTS
Strict-Transport-Security "max-age=300;" # 5 minutes
## NOTE: I have dramatically lowered the above for testing.
## Once you have confirmed that everything works, start increasing the number
## the goal is to have HSTS set to one year with subdomains and preloading :
##
# `Strict-Transport-Security: max-age=31536000; includeSubDomains; preload`
##
## Warning: You should ensure that you fully understand the implications
## of HSTS preloading before you include the directive in your policy and
## before you submit. It means that your entire domain and all subdomains,
## including those managed or maintained by third parties, will only work
## with HTTPS. Preloading should be viewed as a one way ticket.
## Whilst it is possible to be removed, it can take a long time and you
## may not be removed from all browsers.
## disable clients from sniffing the media type
X-Content-Type-Options "nosniff"
## clickjacking protection
X-Frame-Options "DENY"
}
}
(cloudflare-tunnel-protection) {
import ./snippets/cloudflare-replace-X-Forwarded-For
trusted_proxies 172.17.0.0/16 # This needs to be your docker subnet
# I believe this is what is configured by default.
# If you can't make it work ask for my help on discord.
}
knightcrawler.your-domain.com {
## Uncomment to enable logging
# log {
# output file /var/log/caddy/knightcrawler.your-domain.com.log {
# roll_size 10mb
# roll_keep 5
# roll_keep_for 720h
# }
# }
encode gzip
## DO NOT ENABLE UNTIL YOU HAVE DISABLED THE TESTING ENVIRONMENT
# import security-headers
reverse_proxy knightcrawler-addon:7000
}

2
deployment/docker/optional-services/caddy/configs/snippets/cloudflare-replace-X-Forwarded-For Normal file
View File

@@ -0,0 +1,2 @@
header_up X-Real-IP {http.request.header.CF-Connecting-IP}
header_up X-Forwarded-For {http.request.header.CF-Connecting-IP}

39
deployment/docker/optional-services/caddy/docker-compose.yaml Normal file
View File

@@ -0,0 +1,39 @@
name: caddy
networks:
caddy:
name: caddy
external: true
# cloudflare-tunnel:
# name: cloudflare-tunnel
# external: true
volumes:
config:
data:
services:
# Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS
# https://github.com/caddyserver/caddy
caddy:
cap_add:
- NET_ADMIN
container_name: caddy
hostname: caddy
image: caddy:2.7.6 # May be out of date, but using the `:latest` tag is how you accidentally break your server
networks:
- caddy
# - cloudflare-tunnel
ports:
- 80:80
- 443:443
- 443:443/udp
restart: always
security_opt:
- no-new-privileges:true
volumes:
- ./configs/:/etc/caddy/ # /etc/caddy/Caddyfile and /etc/caddy/snippets/
- ./logs:/var/log/caddy/
- config:/config
- data:/data

1
deployment/docker/optional-services/cloudflare-tunnel/.env.example Normal file
View File

@@ -0,0 +1 @@
TOKEN=cloudflare-tunnel-token-here

18
deployment/docker/optional-services/cloudflare-tunnel/docker-compose.yml Normal file
View File

@@ -0,0 +1,18 @@
name: cloudflared
networks:
cloudflare-tunnel:
name: cloudflare-tunnel
external: true
services:
cloudflared:
container_name: cloudflared
command: tunnel --no-autoupdate run --token ${TOKEN}
hostname: cloudflared
image: cloudflare/cloudflared:latest
networks:
- cloudflare-tunnel
restart: always
security_opt:
- no-new-privileges:true

8
deployment/docker/optional-services/grafana-metrics/config/grafana/dashboards/dashboards.yml Normal file
View File

@@ -0,0 +1,8 @@
apiVersion: 1
providers:
- name: Knightcrawler
folder: Dashboards
type: file
options:
path: /var/lib/grafana/dashboards

581
deployment/docker/optional-services/grafana-metrics/config/grafana/dashboards/logs.json Normal file
View File

@@ -0,0 +1,581 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"description": "Logs for Producer and Consumer",
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 2,
"links": [],
"liveNow": false,
"panels": [
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 4,
"panels": [],
"title": "Consumer Data",
"type": "row"
},
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"decimals": 0,
"fieldMinMax": false,
"mappings": [],
"min": 0,
"noValue": "0",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 1
}
]
},
"unit": "none",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 12,
"x": 0,
"y": 1
},
"id": 1,
"options": {
"minVizHeight": 75,
"minVizWidth": 75,
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showThresholdLabels": false,
"showThresholdMarkers": true,
"sizing": "auto",
"text": {}
},
"pluginVersion": "10.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"editorMode": "builder",
"expr": "sum(count_over_time({container=~\"knightcrawler-consumer-[0-9]+\"} | json | __error__=`` | level > 30 [15m]))",
"queryType": "range",
"refId": "ConsumerErrorsLast15m"
}
],
"title": "Errors - Last 15m",
"transparent": true,
"type": "gauge"
},
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 12,
"x": 12,
"y": 1
},
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "10.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"editorMode": "code",
"expr": "sum(count_over_time({container=~\"knightcrawler-consumer-[0-9]+\"} | json | __error__=`` | level = `30` [15m]))",
"queryType": "range",
"refId": "InfoLogsLast15mSum"
}
],
"title": "Info Logs - Last 15m",
"transparent": true,
"type": "stat"
},
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "left",
"cellOptions": {
"type": "auto"
},
"filterable": true,
"inspect": false,
"minWidth": 50
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unitScale": true
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "level"
},
"properties": [
{
"id": "custom.width",
"value": 100
},
{
"id": "custom.align",
"value": "center"
}
]
},
{
"matcher": {
"id": "byName",
"options": "Time"
},
"properties": [
{
"id": "custom.width",
"value": 200
}
]
}
]
},
"gridPos": {
"h": 12,
"w": 24,
"x": 0,
"y": 6
},
"id": 5,
"options": {
"cellHeight": "sm",
"footer": {
"countRows": false,
"enablePagination": false,
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true
},
"pluginVersion": "10.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"editorMode": "code",
"expr": "{container=~\"knightcrawler-consumer-[0-9]+\"} | json | __error__=``",
"queryType": "range",
"refId": "A"
}
],
"title": "Log entries",
"transformations": [
{
"id": "extractFields",
"options": {
"format": "json",
"jsonPaths": [
{
"alias": "level",
"path": "level"
},
{
"alias": "Message",
"path": "msg"
}
],
"keepTime": true,
"replace": true,
"source": "Line"
}
}
],
"type": "table"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 18
},
"id": 6,
"panels": [],
"title": "Producer Data",
"type": "row"
},
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"decimals": 0,
"fieldMinMax": false,
"mappings": [],
"min": 0,
"noValue": "0",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 1
}
]
},
"unit": "none",
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 12,
"x": 0,
"y": 19
},
"id": 7,
"options": {
"minVizHeight": 75,
"minVizWidth": 75,
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showThresholdLabels": false,
"showThresholdMarkers": true,
"sizing": "auto",
"text": {}
},
"pluginVersion": "10.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"editorMode": "code",
"expr": "sum(\n count_over_time({container=~\"knightcrawler-producer-[0-9]+\"} |= \"[Warning]\" [15m])\n) \n+\nsum(\n count_over_time({container=~\"knightcrawler-producer-[0-9]+\"} |= \"[Error]\" [15m])\n)",
"queryType": "range",
"refId": "ConsumerErrorsLast15m"
}
],
"title": "Errors and Warnings - Last 15m",
"transparent": true,
"type": "gauge"
},
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unitScale": true
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 12,
"x": 12,
"y": 19
},
"id": 8,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "10.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"editorMode": "code",
"expr": "sum(\n count_over_time({container=~\"knightcrawler-producer-[0-9]+\"} |= \"[Information]\" [15m])\n) ",
"queryType": "range",
"refId": "InfoLogsLast15mSum"
}
],
"title": "Info Logs - Last 15m",
"transparent": true,
"type": "stat"
},
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "left",
"cellOptions": {
"type": "auto"
},
"filterable": true,
"inspect": false,
"minWidth": 50
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
},
"unitScale": true
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "labels"
},
"properties": [
{
"id": "custom.hidden",
"value": true
}
]
},
{
"matcher": {
"id": "byName",
"options": "tsNs"
},
"properties": [
{
"id": "custom.hidden",
"value": true
}
]
},
{
"matcher": {
"id": "byName",
"options": "id"
},
"properties": [
{
"id": "custom.hidden",
"value": true
}
]
},
{
"matcher": {
"id": "byName",
"options": "Time"
},
"properties": [
{
"id": "custom.width",
"value": 200
}
]
}
]
},
"gridPos": {
"h": 12,
"w": 24,
"x": 0,
"y": 24
},
"id": 9,
"options": {
"cellHeight": "sm",
"footer": {
"countRows": false,
"enablePagination": false,
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true
},
"pluginVersion": "10.3.1",
"targets": [
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"editorMode": "code",
"expr": "{container=~\"knightcrawler-producer-[0-9]+\"}",
"queryType": "range",
"refId": "A"
}
],
"title": "Log entries",
"transformations": [],
"type": "table"
}
],
"refresh": "10s",
"schemaVersion": 39,
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-24h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Knightcrawler Logs",
"uid": "knightcrawler-logs",
"version": 1,
"weekStart": ""
}

12
deployment/docker/optional-services/grafana-metrics/config/grafana/datasources/loki-datasource.yml Normal file
View File

@@ -0,0 +1,12 @@
apiVersion: 1
datasources:
- name: Loki
uid: loki
type: loki
access: proxy
orgId: 1
url: http://loki:3100
basicAuth: false
isDefault: false
version: 1
editable: true

11
deployment/docker/optional-services/grafana-metrics/config/grafana/datasources/prometheus-datasource.yml Normal file
View File

@@ -0,0 +1,11 @@
apiVersion: 1
datasources:
- name: Prometheus
type: prometheus
uid: prometheus
access: proxy
orgId: 1
url: "http://prometheus:9090"
basicAuth: false
isDefault: true
editable: true

35
deployment/docker/optional-services/grafana-metrics/config/loki/config.yml Normal file
View File

@@ -0,0 +1,35 @@
auth_enabled: false
server:
http_listen_port: 3100
limits_config:
split_queries_by_interval: 24h
max_query_parallelism: 100
query_scheduler:
max_outstanding_requests_per_tenant: 4096
frontend:
max_outstanding_per_tenant: 4096
common:
path_prefix: /loki
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
replication_factor: 1
ring:
kvstore:
store: inmemory
schema_config:
configs:
- from: 2020-10-24
store: boltdb-shipper
object_store: filesystem
schema: v11
index:
prefix: index_
period: 24h

22
deployment/docker/optional-services/grafana-metrics/config/prometheus/config.yml Normal file
View File

@@ -0,0 +1,22 @@
global:
scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
# scrape_timeout is set to the global default (10s).
alerting:
alertmanagers:
- static_configs:
- targets:
# - alertmanager:9093
rule_files:
# - "first_rules.yml"
# - "second_rules.yml"
scrape_configs:
- job_name: "rabbitmq"
static_configs:
- targets: ["rabbitmq:15692"]
- job_name: "postgres-exporter"
static_configs:
- targets: ["postgres-exporter:9187"]

27
deployment/docker/optional-services/grafana-metrics/config/promtail/config.yml Normal file
View File

@@ -0,0 +1,27 @@
server:
http_listen_port: 9080
grpc_listen_port: 0
positions:
filename: /tmp/positions.yaml
clients:
- url: http://loki:3100/loki/api/v1/push
tenant_id: knightcrawler
scrape_configs:
- job_name: scraper
docker_sd_configs:
- host: unix:///var/run/docker.sock
refresh_interval: 15s
filters:
- name: label
values: ["logging=promtail","com.docker.compose.project=knightcrawler"]
relabel_configs:
- source_labels: ["__meta_docker_container_name"]
regex: "/(.*)"
target_label: "container"
- source_labels: ["__meta_docker_container_log_stream"]
target_label: "logstream"
- source_labels: ["__meta_docker_container_label_logging_jobname"]
target_label: "job"

75
deployment/docker/optional-services/grafana-metrics/docker-compose.yml Normal file
View File

@@ -0,0 +1,75 @@
name: knightcrawler-metrics
networks:
knightcrawler-network:
external: true
volumes:
grafana-data:
loki-data:
services:
prometheus:
command:
- '--config.file=/etc/prometheus/prometheus.yml'
image: prom/prometheus:v2.20.1
networks:
- knightcrawler-network
ports:
- 9090:9090
volumes:
- ./config/prometheus/config.yml:/etc/prometheus/prometheus.yml
grafana:
depends_on:
- prometheus
environment:
- GF_PATHS_PROVISIONING=/etc/grafana/provisioning
- GF_SECURITY_ADMIN_PASSWORD=admin_password
- GF_SECURITY_ADMIN_USER=admin
image: grafana/grafana:latest
networks:
- knightcrawler-network
ports:
- "3000:3000"
volumes:
- ./config/grafana/dashboards/dashboards.yml:/etc/grafana/provisioning/dashboards/dashboards.yml
- ./config/grafana/dashboards/logs.json:/var/lib/grafana/dashboards/logs.json
- ./config/grafana/datasources:/etc/grafana/provisioning/datasources
- grafana-data:/var/lib/grafana
postgres-exporter:
env_file:
- ../.env
environment:
DATA_SOURCE_NAME: "postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}?sslmode=disable"
image: prometheuscommunity/postgres-exporter
networks:
- knightcrawler-network
promtail:
image: grafana/promtail:2.9.4
volumes:
- ./config/promtail/config.yml:/etc/promtail/config.yml
- /var/lib/docker/containers:/var/lib/docker/containers:ro
- /var/run/docker.sock:/var/run/docker.sock
command:
- '-config.file=/etc/promtail/config.yml'
depends_on:
- prometheus
- loki
networks:
- knightcrawler-network
loki:
command: '-config.file=/etc/loki/local-config.yml'
depends_on:
- grafana
- prometheus
image: grafana/loki:2.9.4
networks:
- knightcrawler-network
volumes:
- ./config/loki/config.yml:/etc/loki/local-config.yml
- loki-data:/loki