Merge pull request #48 from HDwayne/master

Monitoring with Grafana and Prometheus

README.md

@@ -10,6 +10,7 @@ A self-hosted Stremio addon for streaming torrents via a debrid service.
   - [Using](#using)
     - [Initial setup (optional)](#initial-setup-optional)
     - [Run the project](#run-the-project)
+    - [Monitoring with Grafana and Prometheus (Optional)](#monitoring-with-grafana-and-prometheus-optional)
   - [Importing external dumps](#importing-external-dumps)
     - [Import data into database](#import-data-into-database)
     - [INSERT INTO ingested\_torrents](#insert-into-ingested_torrents)
@@ -64,6 +65,42 @@ It will take a while to find and add the torrents to the database. During initia
 
 To add the addon to Stremio, open a web browser and navigate to: [http://127.0.0.1:7000](http://127.0.0.1:7000)
 
+### Monitoring with Grafana and Prometheus (Optional)
+
+To enhance your monitoring capabilities, you can use Grafana and Prometheus in addition to RabbitMQ's built-in management interface. This allows you to visualize and analyze RabbitMQ metrics with more flexibility. With postgres-exporter service, you can also monitor Postgres metrics.
+
+#### Accessing RabbitMQ Management
+
+You can still monitor RabbitMQ by accessing its management interface at [http://127.0.0.1:15672/](http://127.0.0.1:15672/). Use the provided credentials to log in and explore RabbitMQ's monitoring features (the default username and password are `guest`).
+
+#### Using Grafana and Prometheus
+
+Here's how to set up and use Grafana and Prometheus for monitoring RabbitMQ:
+
+1. **Start Grafana and Prometheus**: Run the following command to start both Grafana and Prometheus:
+
+   ```sh
+   docker compose -f docker-compose-metrics.yml up -d
+   ```
+
+   - Grafana will be available at [http://127.0.0.1:3000](http://127.0.0.1:3000).
+   - Prometheus will be available at [http://127.0.0.1:9090](http://127.0.0.1:9090).
+
+   - The default admin user for Grafana is `admin`, and the password is `admin_password`.
+
+2. **Import Grafana Dashboard**: Import the RabbitMQ monitoring dashboard into Grafana:
+
+   - You can use the following dashboard from Grafana's official library: [RabbitMQ Overview Dashboard](https://grafana.com/grafana/dashboards/10991-rabbitmq-overview/).
+
+   - You can alse use the following dashboard [PostgreSQL Database](https://grafana.com/grafana/dashboards/9628-postgresql-database/) to monitor Postgres metrics.
+
+   The Prometheus data source is already configured in Grafana, you just have to select it when importing the dashboard.
+
+
+Now, you can use these dashboards to monitor RabbitMQ and Postgres metrics.
+
+Note: If you encounter issues with missing or unavailable data in Grafana, please ensure on [Prometheus's target page](http://127.0.0.1:9090/targets) that the RabbitMQ target is up and running.
+
 ## Importing external dumps
 
 A brief record of the steps required to import external data, in this case the rarbg dump which can be found on RD:

docker-compose-metrics.yml

@@ -0,0 +1,41 @@
+version: '3.8'
+name: torrentio-metrics
+
+services:
+  prometheus:
+    image: prom/prometheus:v2.20.1
+    volumes:
+      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+    ports:
+      - "9090:9090"
+    networks:
+      - torrentio-network
+
+  grafana:
+    image: grafana/grafana:latest
+    volumes:
+      - ./grafana/datasource.yml:/etc/grafana/provisioning/datasources/datasources.yml
+    ports:
+      - "3000:3000"
+    environment:
+      - GF_SECURITY_ADMIN_USER=admin
+      - GF_SECURITY_ADMIN_PASSWORD=admin_password
+    depends_on:
+      - prometheus
+    networks:
+      - torrentio-network
+
+  postgres-exporter:
+    image: prometheuscommunity/postgres-exporter
+    ports:
+      - "9187:9187"
+    environment:
+      DATA_SOURCE_NAME: "postgresql://postgres:postgres@postgres:5432/selfhostio?sslmode=disable"
+    networks:
+      - torrentio-network
+
+networks:
+  torrentio-network:
+    external: true

docker-compose.yaml

@@ -46,7 +46,9 @@ services:
       - postgres:/var/lib/postgresql/data
     healthcheck: *postgresdb-health
     restart: *restart-policy
-    
+    networks:
+      - torrentio-network
+
   mongodb:
     image: mongo:latest
     environment:
@@ -58,16 +60,21 @@ services:
       - mongo:/data/db
     restart: *restart-policy
     healthcheck: *mongodb-health
-    
+    networks:
+      - torrentio-network
+
   rabbitmq:
     image: rabbitmq:3-management
     ports:
       - "5672:5672"
       - "15672:15672"
+      - "15692:15692"
     volumes:
       - rabbitmq:/var/lib/rabbitmq
     restart: *restart-policy
     healthcheck: *rabbitmq-health
+    networks:
+      - torrentio-network
 
   producer:
     build:
@@ -76,6 +83,8 @@ services:
     env_file:
       - env/producer.env
     <<: *selfhostio-app
+    networks:
+      - torrentio-network
 
   consumer:
     build:
@@ -86,7 +95,9 @@ services:
     deploy:
       replicas: 3
     <<: *selfhostio-app
-    
+    networks:
+      - torrentio-network
+
   addon:
     build:
       context: src/node/addon
@@ -96,6 +107,13 @@ services:
     env_file:
       - env/addon.env
     <<: *selfhostio-app
+    networks:
+      - torrentio-network
+
+networks:
+  torrentio-network:
+    driver: bridge
+    name: torrentio-network
 
 volumes:
   postgres:

grafana/datasource.yml

@@ -0,0 +1,10 @@
+apiVersion: 1
+datasources:
+  - name: RabbitMQ Prometheus
+    type: prometheus
+    access: proxy
+    orgId: 1
+    url: "http://prometheus:9090"
+    basicAuth: false
+    isDefault: true
+    editable: true

prometheus/prometheus.yml

@@ -0,0 +1,22 @@
+global:
+  scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
+  # scrape_timeout is set to the global default (10s).
+
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets:
+          # - alertmanager:9093
+
+rule_files:
+  # - "first_rules.yml"
+  # - "second_rules.yml"
+
+scrape_configs:
+  - job_name: "rabbitmq"
+    static_configs:
+      - targets: ["rabbitmq:15692"]
+  - job_name: "postgres-exporter"
+    static_configs:
+      - targets: ["postgres-exporter:9187"]

src/node/addon/src/lib/repository.js

@@ -130,3 +130,32 @@ export function getKitsuIdSeriesEntries(kitsuId, episode) {
     ]
   });
 }
+
+
+export async function computeTorrentStatistics() {
+  try {
+    // Global Stats
+    const totalTorrents = await Torrent.count();
+
+    // Stats by Provider
+    const torrentsByProvider = await Torrent.findAll({
+      attributes: ['provider', [Sequelize.fn('COUNT', Sequelize.col('provider')), 'count']],
+      group: ['provider']
+    });
+
+    // Stats by Content Type (Movie, Series, etc.)
+    const torrentsByType = await Torrent.findAll({
+      attributes: ['type', [Sequelize.fn('COUNT', Sequelize.col('type')), 'count']],
+      group: ['type']
+    });
+
+    return {
+      totalTorrents,
+      torrentsByProvider,
+      torrentsByType,
+    };
+  } catch (error) {
+    console.error('Error computing statistics:', error);
+    throw error;
+  }
+}

src/node/addon/src/serverless.js

@@ -9,6 +9,7 @@ import { parseConfiguration } from './lib/configuration.js';
 import landingTemplate from './lib/landingTemplate.js';
 import { manifest } from './lib/manifest.js';
 import * as moch from './moch/moch.js';
+import { computeTorrentStatistics } from './lib/repository.js';
 
 const router = new Router();
 const limiter = rateLimit({
@@ -99,6 +100,20 @@ router.get('/:moch/:apiKey/:infoHash/:cachedEntryInfo/:fileIndex/:filename?', (r
       });
 });
 
+/**
+ * Temporary solution to provide statistics (NOT FOR PRODUCTION USE)
+ * easy way to know how many torrents are stored in the PG database
+ * These data should not be exposed to the public
+ */
+router.get('/statistics', async (req, res) => {
+  try {
+    const stats = await computeTorrentStatistics();
+    res.json(stats);
+  } catch (error) {
+    res.status(500).json({ error: 'Failed to compute statistics' });
+  }
+});
+
 export default function (req, res) {
   router(req, res, function () {
     res.statusCode = 404;