feat(prometheus): Docker Daemon scraping for monitoring (#1865)

* Docker Daemon scraping using otel

* Dashboard for the docker statistics updated with internal network implementation

---------

Co-authored-by: Kushang Haria <kushangh@192.168.1.3>
Co-authored-by: Kushang Haria <kushangh@MacBook-Pro.local>

grafana.yml

@@ -11,3 +11,5 @@ services:
       - ./log-analyser/grafana-provisioning/dashboards/:/etc/grafana/provisioning/dashboards/
     ports:
       - "3000:3000"
+    networks:
+      meet.jitsi:

log-analyser.yml

@@ -19,3 +19,6 @@ services:
     volumes:
       - ./log-analyser/otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml
       - ./log-analyser/jitsi-logs/:/tmp/jitsi-logs/
+      - /var/run/docker.sock:/var/run/docker.sock
+    networks:
+      meet.jitsi:

log-analyser/grafana-provisioning/dashboards/docker-statistics.json

@@ -0,0 +1,706 @@
+{
+    "annotations": {
+      "list": [
+        {
+          "builtIn": 1,
+          "datasource": {
+            "type": "grafana",
+            "uid": "-- Grafana --"
+          },
+          "enable": true,
+          "hide": true,
+          "iconColor": "rgba(0, 211, 255, 1)",
+          "name": "Annotations & Alerts",
+          "type": "dashboard"
+        }
+      ]
+    },
+    "editable": true,
+    "fiscalYearStartMonth": 0,
+    "graphTooltip": 0,
+    "links": [],
+    "liveNow": false,
+    "panels": [
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "d301145e-8c4e-4027-bf6e-43e81f095020"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "thresholds"
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 8,
+          "w": 7,
+          "x": 0,
+          "y": 0
+        },
+        "id": 4,
+        "options": {
+          "minVizHeight": 75,
+          "minVizWidth": 75,
+          "orientation": "auto",
+          "reduceOptions": {
+            "calcs": [
+              "lastNotNull"
+            ],
+            "fields": "",
+            "values": false
+          },
+          "showThresholdLabels": false,
+          "showThresholdMarkers": true
+        },
+        "pluginVersion": "10.2.0",
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "d301145e-8c4e-4027-bf6e-43e81f095020"
+            },
+            "editorMode": "code",
+            "expr": "container_memory_percent_ratio * 100",
+            "instant": false,
+            "legendFormat": "Memory Usage (in %)",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "Memory Usage",
+        "type": "gauge"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "d301145e-8c4e-4027-bf6e-43e81f095020"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "thresholds"
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 8,
+          "w": 7,
+          "x": 7,
+          "y": 0
+        },
+        "id": 2,
+        "options": {
+          "minVizHeight": 75,
+          "minVizWidth": 75,
+          "orientation": "auto",
+          "reduceOptions": {
+            "calcs": [
+              "lastNotNull"
+            ],
+            "fields": "",
+            "values": false
+          },
+          "showThresholdLabels": false,
+          "showThresholdMarkers": true
+        },
+        "pluginVersion": "10.2.0",
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "d301145e-8c4e-4027-bf6e-43e81f095020"
+            },
+            "editorMode": "code",
+            "exemplar": false,
+            "expr": "container_cpu_utilization_ratio * 100",
+            "format": "time_series",
+            "instant": false,
+            "legendFormat": "CPU Utilization (in %)",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "CPU Utilization",
+        "type": "gauge"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "d301145e-8c4e-4027-bf6e-43e81f095020"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 8,
+          "w": 10,
+          "x": 14,
+          "y": 0
+        },
+        "id": 1,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "d301145e-8c4e-4027-bf6e-43e81f095020"
+            },
+            "editorMode": "code",
+            "expr": "sum(rate(container_blockio_io_service_bytes_recursive_total{operation=\"read\"}[5m])) ",
+            "hide": false,
+            "instant": false,
+            "legendFormat": "Read Operation",
+            "range": true,
+            "refId": "B"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "d301145e-8c4e-4027-bf6e-43e81f095020"
+            },
+            "editorMode": "code",
+            "expr": "sum(rate(container_blockio_io_service_bytes_recursive_total{operation=\"write\"}[5m])) ",
+            "hide": false,
+            "instant": false,
+            "legendFormat": "Write Operation",
+            "range": true,
+            "refId": "C"
+          }
+        ],
+        "title": "Block IO in bytes",
+        "type": "timeseries"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "d301145e-8c4e-4027-bf6e-43e81f095020"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 8,
+          "w": 12,
+          "x": 0,
+          "y": 8
+        },
+        "id": 3,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "table",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "pluginVersion": "10.2.0",
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "d301145e-8c4e-4027-bf6e-43e81f095020"
+            },
+            "editorMode": "code",
+            "expr": "rate(container_cpu_usage_kernelmode_nanoseconds_total[5m])",
+            "instant": false,
+            "legendFormat": "Kernel Mode",
+            "range": true,
+            "refId": "A"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "d301145e-8c4e-4027-bf6e-43e81f095020"
+            },
+            "editorMode": "code",
+            "expr": "rate(container_cpu_usage_usermode_nanoseconds_total[5m])",
+            "hide": false,
+            "instant": false,
+            "legendFormat": "User Mode",
+            "range": true,
+            "refId": "B"
+          }
+        ],
+        "title": "CPU usage in different modes",
+        "type": "timeseries"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "d301145e-8c4e-4027-bf6e-43e81f095020"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 8,
+          "w": 12,
+          "x": 12,
+          "y": 8
+        },
+        "id": 5,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "pluginVersion": "10.2.0",
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "d301145e-8c4e-4027-bf6e-43e81f095020"
+            },
+            "editorMode": "code",
+            "expr": "(container_memory_usage_total_bytes / container_memory_usage_limit_bytes) * 100",
+            "instant": false,
+            "legendFormat": "Total Bytes Read/Written",
+            "range": true,
+            "refId": "A"
+          }
+        ],
+        "title": "Container memory usage vs limit",
+        "type": "timeseries"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "d301145e-8c4e-4027-bf6e-43e81f095020"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 8,
+          "w": 12,
+          "x": 0,
+          "y": 16
+        },
+        "id": 6,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "d301145e-8c4e-4027-bf6e-43e81f095020"
+            },
+            "editorMode": "code",
+            "exemplar": false,
+            "expr": "rate(container_network_io_usage_rx_bytes_total{interface=\"eth0\"}[5m])",
+            "instant": false,
+            "legendFormat": "Network bytes received",
+            "range": true,
+            "refId": "A"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "d301145e-8c4e-4027-bf6e-43e81f095020"
+            },
+            "editorMode": "code",
+            "expr": "rate(container_network_io_usage_tx_bytes_total{interface=\"eth0\"}[5m])",
+            "hide": false,
+            "instant": false,
+            "legendFormat": "Network bytes Sent",
+            "range": true,
+            "refId": "B"
+          }
+        ],
+        "title": "Network bytes sent and received",
+        "type": "timeseries"
+      },
+      {
+        "datasource": {
+          "type": "prometheus",
+          "uid": "d301145e-8c4e-4027-bf6e-43e81f095020"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "color": {
+              "mode": "palette-classic"
+            },
+            "custom": {
+              "axisBorderShow": false,
+              "axisCenteredZero": false,
+              "axisColorMode": "text",
+              "axisLabel": "",
+              "axisPlacement": "auto",
+              "barAlignment": 0,
+              "drawStyle": "line",
+              "fillOpacity": 0,
+              "gradientMode": "none",
+              "hideFrom": {
+                "legend": false,
+                "tooltip": false,
+                "viz": false
+              },
+              "insertNulls": false,
+              "lineInterpolation": "linear",
+              "lineWidth": 1,
+              "pointSize": 5,
+              "scaleDistribution": {
+                "type": "linear"
+              },
+              "showPoints": "auto",
+              "spanNulls": false,
+              "stacking": {
+                "group": "A",
+                "mode": "none"
+              },
+              "thresholdsStyle": {
+                "mode": "off"
+              }
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green",
+                  "value": null
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 8,
+          "w": 12,
+          "x": 12,
+          "y": 16
+        },
+        "id": 7,
+        "options": {
+          "legend": {
+            "calcs": [],
+            "displayMode": "list",
+            "placement": "bottom",
+            "showLegend": true
+          },
+          "tooltip": {
+            "mode": "single",
+            "sort": "none"
+          }
+        },
+        "targets": [
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "d301145e-8c4e-4027-bf6e-43e81f095020"
+            },
+            "editorMode": "code",
+            "expr": "rate(container_network_io_usage_rx_dropped_total{interface=\"eth0\"}[5m])",
+            "instant": false,
+            "legendFormat": "Network Bytes Dropped in Receiving",
+            "range": true,
+            "refId": "A"
+          },
+          {
+            "datasource": {
+              "type": "prometheus",
+              "uid": "d301145e-8c4e-4027-bf6e-43e81f095020"
+            },
+            "editorMode": "code",
+            "expr": "rate(container_network_io_usage_tx_dropped_total{interface=\"eth0\"}[5m])",
+            "hide": false,
+            "instant": false,
+            "legendFormat": "Network Bytes dropped in Sending",
+            "range": true,
+            "refId": "B"
+          }
+        ],
+        "title": "Network Packets dropped",
+        "type": "timeseries"
+      }
+    ],
+    "refresh": "5s",
+    "schemaVersion": 38,
+    "tags": [],
+    "templating": {
+      "list": []
+    },
+    "time": {
+      "from": "now-24h",
+      "to": "now"
+    },
+    "timepicker": {},
+    "timezone": "",
+    "title": "Docker Statistics",
+    "uid": "c67742f2-7db9-489b-90fa-a13f4655806a",
+    "version": 3,
+    "weekStart": ""
+  }

log-analyser/otel-collector-config.yaml

@@ -24,6 +24,8 @@ receivers:
     operators:
       - type: regex_parser
         regex: "^(?P<message>\\[\\w+-\\w+\\.\\w+\\]\\s+(\\w+\\s+)*\\w+:?\\s*.*$)"
+  docker_stats:
+    endpoint: "unix:///var/run/docker.sock"
 processors:
   batch:
 
@@ -32,6 +34,8 @@ exporters:
     loglevel: debug
   loki:
     endpoint: "http://loki:3100/loki/api/v1/push"
+  prometheus:
+    endpoint: "0.0.0.0:9464"
 
 service:
   pipelines:
@@ -51,3 +55,8 @@ service:
       receivers: [otlp, filelog/jitsi-web]
       processors: [batch]
       exporters: [loki]
+    metrics:
+      receivers: [docker_stats]
+      processors: [batch]
+      exporters: [prometheus]
+

prometheus/README.md

@@ -0,0 +1,73 @@
+# Prometheus Scraping & Grafana Dashboard for Jitsi
+
+## Overview
+
+This project aims to integrate **Prometheus** and **Grafana** with Jitsi to monitor and visualize performance metrics.
+
+## Features
+
+- **Prometheus Integration**: Collects metrics from Jitsi containers.
+- **Grafana Dashboards**: Visualizes the metrics for easy analysis.
+
+## Installation
+
+### Prerequisites
+
+- Docker
+- Docker Compose
+
+### Steps
+
+1. **Setup Jitsi with Docker Compose**
+
+   Follow the [Jitsi Docker](https://github.com/jitsi/docker-jitsi-meet) setup instructions. <br>
+   Also, you could follow Self - Hosting guide of Jitsi Meet: [Jitsi handbook](https://jitsi.github.io/handbook/docs/devops-guide/devops-guide-docker/)
+
+2. **Configure Prometheus**
+
+   Edit `/prometheus/prometheus.yml` with any **Port / Container name** changes are there to scrape metrics from Jitsi containers:
+
+   ```yaml
+   scrape_configs:
+     - job_name: "jitsi"
+       static_configs:
+         - targets: ["prosody:5280", "jvb:8080", "jicofo:8888", "otel:9464"]
+   ```
+
+3. **Run Docker Compose**
+
+   The following command turns up the Jitsi Meet:
+
+   ```bash
+   docker-compose up -d
+   ```
+
+   If you want to add the Prometheus and Grafana for monitoring purpose. Use the following command:
+
+   ```bash
+   docker-compose -f docker-compose.yml -f prometheus.yml -f grafana.yml up -d
+   ```
+
+   To monitor Docker Engine we need to enable **Open Telemetry** service, which can be turned up from `log-analyser.yml`. Use the following command:
+
+   ```bash
+   docker-compose -f docker-compose.yml -f prometheus.yml -f grafana.yml -f log-analyser.yml up -d
+   ```
+
+## Usage
+
+1. **View the Prometheus Targets**
+
+   Open [http://localhost:9090](http://localhost:9090) in your browser.
+
+2. **Access Grafana Dashboard**
+
+   Open [http://localhost:3000](http://localhost:3000) in your browser.
+
+3. **Import Dashboard**
+
+   Import the provided JSON file in Grafana to visualize Jitsi metrics.
+
+## Contributer
+
+[@24kushang](https://github.com/24kushang).

prometheus/prometheus.yml

@@ -2,4 +2,4 @@ scrape_configs:
   - job_name: "prometheus"
     scrape_interval: 5s
     static_configs:
-      - targets: ["prosody:5280","jvb:8080","jicofo:8888"]
+      - targets: ["prosody:5280","jvb:8080","jicofo:8888","otel:9464"]