From 8e991488ce3f69c6d80bfa41054ac96bed9ba86b Mon Sep 17 00:00:00 2001 From: HDwayne Date: Sat, 3 Feb 2024 17:09:10 +0100 Subject: [PATCH 1/5] Add monitoring capabilities with Grafana and Prometheus --- README.md | 35 +++++++++++++++++++++++++++++++++++ docker-compose-metrics.yml | 32 ++++++++++++++++++++++++++++++++ docker-compose.yaml | 24 +++++++++++++++++++++--- grafana/datasource.yml | 10 ++++++++++ prometheus/prometheus.yml | 19 +++++++++++++++++++ 5 files changed, 117 insertions(+), 3 deletions(-) create mode 100644 docker-compose-metrics.yml create mode 100644 grafana/datasource.yml create mode 100644 prometheus/prometheus.yml diff --git a/README.md b/README.md index a4be98c..9bb27e2 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ A self-hosted Stremio addon for streaming torrents via a debrid service. - [Using](#using) - [Initial setup (optional)](#initial-setup-optional) - [Run the project](#run-the-project) + - [Monitoring with Grafana and Prometheus (Optional)](#monitoring-with-grafana-and-prometheus-optional) - [Importing external dumps](#importing-external-dumps) - [Import data into database](#import-data-into-database) - [INSERT INTO ingested\_torrents](#insert-into-ingested_torrents) @@ -64,6 +65,40 @@ It will take a while to find and add the torrents to the database. During initia To add the addon to Stremio, open a web browser and navigate to: [http://127.0.0.1:7000](http://127.0.0.1:7000) +### Monitoring with Grafana and Prometheus (Optional) + +To enhance your monitoring capabilities, you can use Grafana and Prometheus in addition to RabbitMQ's built-in management interface. This allows you to visualize and analyze RabbitMQ metrics with more flexibility. + +#### Accessing RabbitMQ Management + +You can still monitor RabbitMQ by accessing its management interface at [http://127.0.0.1:15672/](http://127.0.0.1:15672/). Use the provided credentials to log in and explore RabbitMQ's monitoring features (the default username and password are `guest`). + +#### Using Grafana and Prometheus + +Here's how to set up and use Grafana and Prometheus for monitoring RabbitMQ: + +1. **Start Grafana and Prometheus**: Run the following command to start both Grafana and Prometheus: + + ```bash + docker-compose -f docker-compose-metrics.yml up -d + ``` + + - Grafana will be available at [http://127.0.0.1:3000](http://127.0.0.1:3000). + - Prometheus will be available at [http://127.0.0.1:9090](http://127.0.0.1:9090). + + - The default admin user for Grafana is `admin`, and the password is `admin_password`. + +2. **Import Grafana Dashboard**: Import the RabbitMQ monitoring dashboard into Grafana: + + - You can use the following dashboard from Grafana's official library: [RabbitMQ Overview Dashboard](https://grafana.com/grafana/dashboards/10991-rabbitmq-overview/). + + The Prometheus data source is already configured in Grafana, you just have to select it when importing the dashboard. + + +Now, you can use the imported dashboard to visualize RabbitMQ metrics collected by Prometheus. + +Note: If you encounter issues with missing or unavailable data in Grafana, please ensure on [Prometheus's target page](http://127.0.0.1:9090/targets) that the RabbitMQ target is up and running. + ## Importing external dumps A brief record of the steps required to import external data, in this case the rarbg dump which can be found on RD: diff --git a/docker-compose-metrics.yml b/docker-compose-metrics.yml new file mode 100644 index 0000000..52e5d51 --- /dev/null +++ b/docker-compose-metrics.yml @@ -0,0 +1,32 @@ +version: '3.8' +name: torrentio-metrics + +services: + prometheus: + image: prom/prometheus:v2.20.1 + volumes: + - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml + command: + - '--config.file=/etc/prometheus/prometheus.yml' + ports: + - "9090:9090" + networks: + - torrentio-network + + grafana: + image: grafana/grafana:latest + volumes: + - ./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasources.yml + ports: + - "3000:3000" + environment: + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=admin_password + depends_on: + - prometheus + networks: + - torrentio-network + +networks: + torrentio-network: + external: true diff --git a/docker-compose.yaml b/docker-compose.yaml index 62d7650..772800d 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -46,7 +46,9 @@ services: - postgres:/var/lib/postgresql/data healthcheck: *postgresdb-health restart: *restart-policy - + networks: + - torrentio-network + mongodb: image: mongo:latest environment: @@ -58,16 +60,21 @@ services: - mongo:/data/db restart: *restart-policy healthcheck: *mongodb-health - + networks: + - torrentio-network + rabbitmq: image: rabbitmq:3-management ports: - "5672:5672" - "15672:15672" + - "15692:15692" volumes: - rabbitmq:/var/lib/rabbitmq restart: *restart-policy healthcheck: *rabbitmq-health + networks: + - torrentio-network producer: build: @@ -76,6 +83,8 @@ services: env_file: - env/producer.env <<: *selfhostio-app + networks: + - torrentio-network consumer: build: @@ -86,7 +95,9 @@ services: deploy: replicas: 3 <<: *selfhostio-app - + networks: + - torrentio-network + addon: build: context: src/node/addon @@ -96,6 +107,13 @@ services: env_file: - env/addon.env <<: *selfhostio-app + networks: + - torrentio-network + +networks: + torrentio-network: + driver: bridge + name: torrentio-network volumes: postgres: diff --git a/grafana/datasource.yml b/grafana/datasource.yml new file mode 100644 index 0000000..7e63fb7 --- /dev/null +++ b/grafana/datasource.yml @@ -0,0 +1,10 @@ +apiVersion: 1 +datasources: + - name: RabbitMQ Prometheus + type: prometheus + access: proxy + orgId: 1 + url: "http://prometheus:9090" + basicAuth: false + isDefault: true + editable: true diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml new file mode 100644 index 0000000..9ccd1d6 --- /dev/null +++ b/prometheus/prometheus.yml @@ -0,0 +1,19 @@ +global: + scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. + evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. + # scrape_timeout is set to the global default (10s). + +alerting: + alertmanagers: + - static_configs: + - targets: + # - alertmanager:9093 + +rule_files: + # - "first_rules.yml" + # - "second_rules.yml" + +scrape_configs: + - job_name: "rabbitmq" + static_configs: + - targets: ["rabbitmq:15692"] From 679a65846343c005fb72e4051b50abd7de79a53e Mon Sep 17 00:00:00 2001 From: HDwayne Date: Sat, 3 Feb 2024 17:17:13 +0100 Subject: [PATCH 2/5] Fix docker-compose command in README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9bb27e2..d258287 100644 --- a/README.md +++ b/README.md @@ -79,8 +79,8 @@ Here's how to set up and use Grafana and Prometheus for monitoring RabbitMQ: 1. **Start Grafana and Prometheus**: Run the following command to start both Grafana and Prometheus: - ```bash - docker-compose -f docker-compose-metrics.yml up -d + ```sh + docker compose -f docker-compose-metrics.yml up -d ``` - Grafana will be available at [http://127.0.0.1:3000](http://127.0.0.1:3000). From a56b04f04b990effb89b517b128cce67a6000958 Mon Sep 17 00:00:00 2001 From: HDwayne Date: Sat, 3 Feb 2024 22:06:13 +0100 Subject: [PATCH 3/5] Add Temporary statistics endpoint --- src/node/addon/src/lib/repository.js | 29 ++++++++++++++++++++++++++++ src/node/addon/src/serverless.js | 15 ++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/src/node/addon/src/lib/repository.js b/src/node/addon/src/lib/repository.js index acce491..1c26864 100644 --- a/src/node/addon/src/lib/repository.js +++ b/src/node/addon/src/lib/repository.js @@ -130,3 +130,32 @@ export function getKitsuIdSeriesEntries(kitsuId, episode) { ] }); } + + +export async function computeTorrentStatistics() { + try { + // Global Stats + const totalTorrents = await Torrent.count(); + + // Stats by Provider + const torrentsByProvider = await Torrent.findAll({ + attributes: ['provider', [Sequelize.fn('COUNT', Sequelize.col('provider')), 'count']], + group: ['provider'] + }); + + // Stats by Content Type (Movie, Series, etc.) + const torrentsByType = await Torrent.findAll({ + attributes: ['type', [Sequelize.fn('COUNT', Sequelize.col('type')), 'count']], + group: ['type'] + }); + + return { + totalTorrents, + torrentsByProvider, + torrentsByType, + }; + } catch (error) { + console.error('Error computing statistics:', error); + throw error; + } +} diff --git a/src/node/addon/src/serverless.js b/src/node/addon/src/serverless.js index 1cf275b..177ac6e 100644 --- a/src/node/addon/src/serverless.js +++ b/src/node/addon/src/serverless.js @@ -9,6 +9,7 @@ import { parseConfiguration } from './lib/configuration.js'; import landingTemplate from './lib/landingTemplate.js'; import { manifest } from './lib/manifest.js'; import * as moch from './moch/moch.js'; +import { computeTorrentStatistics } from './lib/repository.js'; const router = new Router(); const limiter = rateLimit({ @@ -99,6 +100,20 @@ router.get('/:moch/:apiKey/:infoHash/:cachedEntryInfo/:fileIndex/:filename?', (r }); }); +/** + * Temporary solution to provide statistics (NOT FOR PRODUCTION USE) + * easy way to know how many torrents are stored in the PG database + * These data should not be exposed to the public + */ +router.get('/statistics', async (req, res) => { + try { + const stats = await computeTorrentStatistics(); + res.json(stats); + } catch (error) { + res.status(500).json({ error: 'Failed to compute statistics' }); + } +}); + export default function (req, res) { router(req, res, function () { res.statusCode = 404; From 5f691ef87768af8a7c67ef06d095df612de99a16 Mon Sep 17 00:00:00 2001 From: HDwayne Date: Sun, 4 Feb 2024 01:18:34 +0100 Subject: [PATCH 4/5] Add Postgres exporter service --- docker-compose-metrics.yml | 9 +++++++++ prometheus/prometheus.yml | 3 +++ 2 files changed, 12 insertions(+) diff --git a/docker-compose-metrics.yml b/docker-compose-metrics.yml index 52e5d51..d944246 100644 --- a/docker-compose-metrics.yml +++ b/docker-compose-metrics.yml @@ -27,6 +27,15 @@ services: networks: - torrentio-network + postgres-exporter: + image: prometheuscommunity/postgres-exporter + ports: + - "9187:9187" + environment: + DATA_SOURCE_NAME: "postgresql://postgres:postgres@postgres:5432/selfhostio?sslmode=disable" + networks: + - torrentio-network + networks: torrentio-network: external: true diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml index 9ccd1d6..8a9d923 100644 --- a/prometheus/prometheus.yml +++ b/prometheus/prometheus.yml @@ -17,3 +17,6 @@ scrape_configs: - job_name: "rabbitmq" static_configs: - targets: ["rabbitmq:15692"] + - job_name: "postgres-exporter" + static_configs: + - targets: ["postgres-exporter:9187"] From 45e17799b2b1b760412a8bfdcf6e4b4040e10890 Mon Sep 17 00:00:00 2001 From: HDwayne Date: Sun, 4 Feb 2024 01:29:55 +0100 Subject: [PATCH 5/5] Add monitoring for Postgres metrics in README --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d258287..a1237c6 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ To add the addon to Stremio, open a web browser and navigate to: [http://127.0.0 ### Monitoring with Grafana and Prometheus (Optional) -To enhance your monitoring capabilities, you can use Grafana and Prometheus in addition to RabbitMQ's built-in management interface. This allows you to visualize and analyze RabbitMQ metrics with more flexibility. +To enhance your monitoring capabilities, you can use Grafana and Prometheus in addition to RabbitMQ's built-in management interface. This allows you to visualize and analyze RabbitMQ metrics with more flexibility. With postgres-exporter service, you can also monitor Postgres metrics. #### Accessing RabbitMQ Management @@ -92,10 +92,12 @@ Here's how to set up and use Grafana and Prometheus for monitoring RabbitMQ: - You can use the following dashboard from Grafana's official library: [RabbitMQ Overview Dashboard](https://grafana.com/grafana/dashboards/10991-rabbitmq-overview/). + - You can alse use the following dashboard [PostgreSQL Database](https://grafana.com/grafana/dashboards/9628-postgresql-database/) to monitor Postgres metrics. + The Prometheus data source is already configured in Grafana, you just have to select it when importing the dashboard. -Now, you can use the imported dashboard to visualize RabbitMQ metrics collected by Prometheus. +Now, you can use these dashboards to monitor RabbitMQ and Postgres metrics. Note: If you encounter issues with missing or unavailable data in Grafana, please ensure on [Prometheus's target page](http://127.0.0.1:9090/targets) that the RabbitMQ target is up and running.