diff --git a/.gitignore b/.gitignore index 00b6d2cf1..603dcd78b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Ignore environment files .env +infra/compose/prometheus/openrag_token # generated files and folders .files/ diff --git a/docs/assets/env_example.env b/docs/assets/env_example.env index 28461de32..dfc9a97bc 100644 --- a/docs/assets/env_example.env +++ b/docs/assets/env_example.env @@ -105,8 +105,14 @@ CHAINLIT_AUTH_SECRET=openrag-dev-secret # API/auth, so it is the OIDC front door. Zero-config otherwise (same-origin, no # CORS); VITE_* build-time options are documented in the env vars reference. # ADMIN_UI_PORT=8081 - - +# GRAFANA_ADMIN_USER=admin +# GRAFANA_ADMIN_PASSWORD=replace-with-a-strong-password +# Direct Grafana access: +# GRAFANA_URL=http://localhost:3000/d/openrag-http/openrag-http-metrics +# Or through the Admin UI proxy: +# GRAFANA_URL=http://localhost:8081/grafana/d/openrag-http/openrag-http-metrics +# GF_SERVER_ROOT_URL=http://localhost:8081/grafana/ +# GF_SERVER_SERVE_FROM_SUB_PATH=true # ── Ray (kept as-is by the compose stack; see the docs for what each does) ─── RAY_DEDUP_LOGS=0 @@ -117,4 +123,4 @@ RAY_ENABLE_UV_RUN_RUNTIME_ENV=0 # RAY_memory_monitor_refresh_ms=0 # ── Logging (DEBUG on dev, INFO on prod) ── -LOG_LEVEL=DEBUG \ No newline at end of file +LOG_LEVEL=DEBUG diff --git a/docs/assets/env_linux_gpu.env b/docs/assets/env_linux_gpu.env index 28f2fd327..068fef865 100644 --- a/docs/assets/env_linux_gpu.env +++ b/docs/assets/env_linux_gpu.env @@ -52,7 +52,16 @@ RAY_ENABLE_UV_RUN_RUNTIME_ENV=0 # critical with the newest version of UV ## replace X.X.X.X with localhost (local) or your server IP, and APP_PORT with your FastAPI port # ADMIN_UI_PORT=8081 # Host port for the admin UI (nginx). Default is 8081. +# GRAFANA_ADMIN_USER=admin # Grafana login used on first startup. +# GRAFANA_ADMIN_PASSWORD=replace-with-a-strong-password +# Direct Grafana access: +# GRAFANA_URL=http://localhost:3000/d/openrag-http/openrag-http-metrics +# # Runtime dashboard link. Restart the API after changing it. +# Or through the Admin UI proxy: +# GRAFANA_URL=http://localhost:8081/grafana/d/openrag-http/openrag-http-metrics +# GF_SERVER_ROOT_URL=http://localhost:8081/grafana/ +# GF_SERVER_SERVE_FROM_SUB_PATH=true # Serve Grafana through the admin UI's /grafana/ path. # VITE_API_BASE_URL= # API base baked into the SPA. EMPTY (default) = same-origin via # # nginx (no CORS). Only set for a browser-direct build. -# VITE_GRAFANA_URL= # Optional Grafana dashboard link on the admin "System" page. -# VITE_APP_NAME=OpenRAG # App display name used in the UI branding. \ No newline at end of file +# VITE_GRAFANA_URL= # Optional build-time fallback for older API deployments. +# VITE_APP_NAME=OpenRAG # App display name used in the UI branding. diff --git a/docs/content/docs/documentation/env_vars.md b/docs/content/docs/documentation/env_vars.md index bced96799..7162b55e9 100644 --- a/docs/content/docs/documentation/env_vars.md +++ b/docs/content/docs/documentation/env_vars.md @@ -641,9 +641,10 @@ flowchart TD | Variable | Type | Default | Description | |----------|------|---------|-------------| | `ADMIN_UI_PORT` | `number` | `8081` | Host port the admin UI (nginx) is published on. Serves `/app/` and reverse-proxies `/auth`, `/v1`, `/chainlit`, … to the backend, so it is the OIDC front door (`OIDC_REDIRECT_URI` targets this port). Deploy-time (not a `VITE_*` build arg). | +| `GRAFANA_URL` | `string` | `""` | Runtime, browser-reachable URL for the Grafana dashboard opened from **System → Metrics**. Restart the API after changing it. When this is empty or invalid, the action explains how to configure the dashboard instead of opening it. | | `VITE_API_BASE_URL` | `string` | `""` (same-origin) | API base baked into the SPA. **Empty (default) = same-origin**: nginx reverse-proxies the API over the Docker network, so the UI works on any host/IP with no CORS. Set to an absolute URL only for a browser-direct build — then list the UI's origin in `CORS_EXTRA_ORIGINS`. | | `VITE_BASE_PATH` | `string` | `/app/` | Sub-path the SPA is served under; must match the nginx `location`. | -| `VITE_GRAFANA_URL` | `string` | `""` | Optional Grafana dashboard link shown on the admin **System** page. | +| `VITE_GRAFANA_URL` | `string` | `""` | Build-time fallback for deployments whose API does not expose `GRAFANA_URL`. New deployments should use the runtime setting instead. | | `VITE_APP_NAME` | `string` | `OpenRAG` | Application display name used in the UI branding. | | `VITE_MOCK_API` | `boolean` | `false` | Development only — serves in-browser MSW API mocks when `true`. Ignored in production builds. | @@ -712,3 +713,5 @@ Read only by the opt-in monitoring compose file (`infra/compose/monitoring.docke |----------|------|---------|-------------| | `GRAFANA_ADMIN_USER` | `str` | `admin` | Grafana admin username. | | `GRAFANA_ADMIN_PASSWORD` | `str` | _(required)_ | Grafana admin password — compose refuses to start the monitoring profile if unset. | +| `GF_SERVER_ROOT_URL` | `str` | `http://localhost:3000` | Browser-facing Grafana root URL. Set this to the admin UI's `/grafana/` URL when using its proxy. | +| `GF_SERVER_SERVE_FROM_SUB_PATH` | `bool` | `false` | Set to `true` when `GF_SERVER_ROOT_URL` includes the `/grafana/` subpath. | diff --git a/docs/content/docs/installation/docker.mdx b/docs/content/docs/installation/docker.mdx index 1bdfb5cfa..95e6556c7 100644 --- a/docs/content/docs/installation/docker.mdx +++ b/docs/content/docs/installation/docker.mdx @@ -13,3 +13,61 @@ The OpenRAG docker image is available on [DockerHub](https://hub.docker.com/r/rc OpenRAG requires several services to run, which can be orchestrated using Docker Compose. +## Grafana monitoring + +The optional monitoring stack provides ready-to-use dashboards for OpenRAG HTTP +traffic, host resources, and NVIDIA GPUs. It is separate from the default stack +so deployments that do not need monitoring keep the same footprint. + +Before starting it, set a strong `GRAFANA_ADMIN_PASSWORD` in `.env`. The +`GRAFANA_ADMIN_USER` defaults to `admin`. Create the Prometheus token file from +the same `AUTH_TOKEN` used by OpenRAG: + +```bash +cd infra/compose +compose_environment="$(docker compose --env-file .env config --environment)" +AUTH_TOKEN="$(printf '%s\n' "$compose_environment" | sed -n 's/^AUTH_TOKEN=//p')" +allow_no_auth="$( + printf '%s\n' "$compose_environment" | + sed -n 's/^ALLOW_NO_AUTH=//p' | + tr '[:upper:]' '[:lower:]' | + sed 's/^[[:space:]]*//;s/[[:space:]]*$//' +)" +if [ -z "$AUTH_TOKEN" ] && [ "$allow_no_auth" != "true" ]; then + echo "AUTH_TOKEN must be set unless ALLOW_NO_AUTH=true." >&2 + exit 1 +fi +printf '%s' "$AUTH_TOKEN" | sudo tee prometheus/openrag_token >/dev/null +sudo chown 65534:65534 prometheus/openrag_token +sudo chmod 0400 prometheus/openrag_token +``` + +Start OpenRAG with the monitoring overlay: + +```bash +export SHARED_ENV="$PWD/.env" +docker compose --env-file .env \ + -f docker-compose.yaml \ + -f monitoring.docker-compose.yaml \ + up -d +``` + +`GRAFANA_URL` controls where **System > Metrics** opens. It must be reachable +from the user's browser. For direct local access, use: + +```dotenv +GRAFANA_URL=http://localhost:3000/d/openrag-http/openrag-http-metrics +``` + +For remote deployments, expose Grafana through the Admin UI proxy and configure +its subpath: + +```dotenv +GRAFANA_URL=https://openrag.example.com/grafana/d/openrag-http/openrag-http-metrics +GF_SERVER_ROOT_URL=https://openrag.example.com/grafana/ +GF_SERVER_SERVE_FROM_SUB_PATH=true +``` + +Grafana has its own login. Its admin password is initialized when the persistent +Grafana volume is created; changing the environment variable later does not +reset an existing password. diff --git a/infra/charts/openrag-stack/values.yaml b/infra/charts/openrag-stack/values.yaml index 6e36a781d..597b1b2c2 100644 --- a/infra/charts/openrag-stack/values.yaml +++ b/infra/charts/openrag-stack/values.yaml @@ -334,6 +334,7 @@ env: ENABLE_RAY_SERVE: "true" RAY_SERVE_NUM_REPLICAS: "4" RAY_SERVE_PORT: "80" + GRAFANA_URL: "" WITH_CHAINLIT_UI: "false" SAVE_UPLOADED_FILES: "false" diff --git a/infra/compose/.env.example b/infra/compose/.env.example index 28461de32..dfc9a97bc 100644 --- a/infra/compose/.env.example +++ b/infra/compose/.env.example @@ -105,8 +105,14 @@ CHAINLIT_AUTH_SECRET=openrag-dev-secret # API/auth, so it is the OIDC front door. Zero-config otherwise (same-origin, no # CORS); VITE_* build-time options are documented in the env vars reference. # ADMIN_UI_PORT=8081 - - +# GRAFANA_ADMIN_USER=admin +# GRAFANA_ADMIN_PASSWORD=replace-with-a-strong-password +# Direct Grafana access: +# GRAFANA_URL=http://localhost:3000/d/openrag-http/openrag-http-metrics +# Or through the Admin UI proxy: +# GRAFANA_URL=http://localhost:8081/grafana/d/openrag-http/openrag-http-metrics +# GF_SERVER_ROOT_URL=http://localhost:8081/grafana/ +# GF_SERVER_SERVE_FROM_SUB_PATH=true # ── Ray (kept as-is by the compose stack; see the docs for what each does) ─── RAY_DEDUP_LOGS=0 @@ -117,4 +123,4 @@ RAY_ENABLE_UV_RUN_RUNTIME_ENV=0 # RAY_memory_monitor_refresh_ms=0 # ── Logging (DEBUG on dev, INFO on prod) ── -LOG_LEVEL=DEBUG \ No newline at end of file +LOG_LEVEL=DEBUG diff --git a/infra/compose/docker-compose.yaml b/infra/compose/docker-compose.yaml index 5ca821fb5..31e62a12e 100644 --- a/infra/compose/docker-compose.yaml +++ b/infra/compose/docker-compose.yaml @@ -125,7 +125,7 @@ services: # to CORS_EXTRA_ORIGINS on the API. VITE_API_BASE_URL: ${VITE_API_BASE_URL:-} VITE_BASE_PATH: "/app/" - # Optional Grafana dashboard link on the admin "System" page; branding name. + # Build-time Grafana fallback; prefer runtime GRAFANA_URL in .env. VITE_GRAFANA_URL: ${VITE_GRAFANA_URL:-} VITE_APP_NAME: ${VITE_APP_NAME:-OpenRAG} restart: unless-stopped diff --git a/infra/compose/grafana/dashboards/gpu-metrics.json b/infra/compose/grafana/dashboards/gpu-metrics.json deleted file mode 100644 index 89bfc528a..000000000 --- a/infra/compose/grafana/dashboards/gpu-metrics.json +++ /dev/null @@ -1,366 +0,0 @@ -{ - "uid": "gpu-metrics", - "title": "GPU Metrics", - "description": "NVIDIA GPU metrics from nvidia-gpu-exporter", - "tags": ["openrag", "gpu", "nvidia"], - "timezone": "browser", - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "links": [], - "templating": { - "list": [ - { - "current": {}, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "definition": "label_values(nvidia_gpu_duty_cycle, uuid)", - "hide": 0, - "includeAll": true, - "label": "GPU", - "multi": true, - "name": "gpu", - "options": [], - "query": { - "query": "label_values(nvidia_gpu_duty_cycle, uuid)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "skipUrlSync": false, - "sort": 1, - "type": "query" - } - ] - }, - "panels": [ - { - "id": 1, - "title": "GPU Utilization (%)", - "type": "timeseries", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisLabel": "%", - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } - }, - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 70 }, - { "color": "red", "value": 90 } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "targets": [ - { - "expr": "nvidia_gpu_duty_cycle{uuid=~\"$gpu\"}", - "legendFormat": "GPU {{ name }} ({{ uuid }})", - "refId": "A" - } - ], - "options": { - "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, - "tooltip": { "mode": "multi", "sort": "desc" } - } - }, - { - "id": 2, - "title": "GPU Memory Usage", - "type": "timeseries", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisLabel": "", - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "red", "value": 80 } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "targets": [ - { - "expr": "nvidia_gpu_memory_total_bytes{uuid=~\"$gpu\"}", - "legendFormat": "Total — {{ name }}", - "refId": "A" - }, - { - "expr": "nvidia_gpu_memory_used_bytes{uuid=~\"$gpu\"}", - "legendFormat": "Used — {{ name }}", - "refId": "B" - } - ], - "options": { - "legend": { "calcs": ["mean", "lastNotNull"], "displayMode": "table", "placement": "bottom" }, - "tooltip": { "mode": "multi", "sort": "desc" } - } - }, - { - "id": 3, - "title": "GPU Utilization", - "type": "gauge", - "gridPos": { "h": 6, "w": 6, "x": 0, "y": 8 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "thresholds" }, - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 60 }, - { "color": "orange", "value": 80 }, - { "color": "red", "value": 95 } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "targets": [ - { - "expr": "nvidia_gpu_duty_cycle{uuid=~\"$gpu\"}", - "legendFormat": "{{ name }}", - "refId": "A" - } - ], - "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "showThresholdLabels": false, - "showThresholdMarkers": true - } - }, - { - "id": 4, - "title": "GPU Memory Usage (%)", - "type": "gauge", - "gridPos": { "h": 6, "w": 6, "x": 6, "y": 8 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "thresholds" }, - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 60 }, - { "color": "orange", "value": 80 }, - { "color": "red", "value": 95 } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "targets": [ - { - "expr": "(nvidia_gpu_memory_used_bytes{uuid=~\"$gpu\"} / nvidia_gpu_memory_total_bytes{uuid=~\"$gpu\"}) * 100", - "legendFormat": "{{ name }}", - "refId": "A" - } - ], - "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "showThresholdLabels": false, - "showThresholdMarkers": true - } - }, - { - "id": 5, - "title": "GPU Temperature", - "type": "timeseries", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisLabel": "°C", - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "line+area" } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "transparent", "value": null }, - { "color": "red", "value": 85 } - ] - }, - "unit": "celsius" - }, - "overrides": [] - }, - "targets": [ - { - "expr": "nvidia_gpu_temperature_celsius{uuid=~\"$gpu\"}", - "legendFormat": "{{ name }} ({{ uuid }})", - "refId": "A" - } - ], - "options": { - "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, - "tooltip": { "mode": "multi", "sort": "desc" } - } - }, - { - "id": 6, - "title": "GPU Power Draw", - "type": "timeseries", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisLabel": "Watts", - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "none", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null } - ] - }, - "unit": "watt" - }, - "overrides": [] - }, - "targets": [ - { - "expr": "nvidia_gpu_power_draw_watts{uuid=~\"$gpu\"}", - "legendFormat": "{{ name }} ({{ uuid }})", - "refId": "A" - } - ], - "options": { - "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, - "tooltip": { "mode": "multi", "sort": "desc" } - } - }, - { - "id": 7, - "title": "GPU Fan Speed", - "type": "timeseries", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, - "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisLabel": "%", - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "lineInterpolation": "smooth", - "lineWidth": 2, - "pointSize": 5, - "showPoints": "never", - "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } - }, - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "targets": [ - { - "expr": "nvidia_gpu_fan_speed_percent{uuid=~\"$gpu\"}", - "legendFormat": "{{ name }} ({{ uuid }})", - "refId": "A" - } - ], - "options": { - "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, - "tooltip": { "mode": "multi", "sort": "desc" } - } - } - ], - "refresh": "30s", - "schemaVersion": 39, - "time": { "from": "now-1h", "to": "now" }, - "timepicker": {}, - "version": 1 -} diff --git a/infra/compose/grafana/dashboards/system-overview.json b/infra/compose/grafana/dashboards/system-overview.json index bc3c57110..7dcf65128 100644 --- a/infra/compose/grafana/dashboards/system-overview.json +++ b/infra/compose/grafana/dashboards/system-overview.json @@ -1,8 +1,8 @@ { "uid": "system-overview", - "title": "System Overview", - "description": "Host system metrics from node_exporter", - "tags": ["openrag", "system", "node-exporter"], + "title": "Infrastructure Overview", + "description": "Host CPU, memory, storage, network, and aggregate NVIDIA GPU metrics", + "tags": ["openrag", "system", "node-exporter", "gpu", "nvidia"], "timezone": "browser", "editable": true, "fiscalYearStartMonth": 0, @@ -122,7 +122,7 @@ "id": 3, "title": "Memory Usage (%)", "type": "gauge", - "gridPos": { "h": 6, "w": 6, "x": 0, "y": 8 }, + "gridPos": { "h": 8, "w": 6, "x": 0, "y": 8 }, "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { @@ -160,7 +160,7 @@ "id": 4, "title": "CPU Usage (%)", "type": "gauge", - "gridPos": { "h": 6, "w": 6, "x": 6, "y": 8 }, + "gridPos": { "h": 8, "w": 6, "x": 6, "y": 8 }, "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { @@ -256,7 +256,7 @@ "id": 6, "title": "Network I/O", "type": "timeseries", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 14 }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, "datasource": { "type": "prometheus", "uid": "prometheus" }, "fieldConfig": { "defaults": { @@ -367,6 +367,165 @@ "legend": { "calcs": ["mean", "lastNotNull"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } } + }, + { + "id": 8, + "title": "GPU Utilization (%)", + "description": "Highest compute utilization across all NVIDIA GPUs", + "type": "gauge", + "gridPos": { "h": 6, "w": 6, "x": 0, "y": 24 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "max": 100, + "min": 0, + "noValue": "N/A", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 60 }, + { "color": "orange", "value": 80 }, + { "color": "red", "value": 90 } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "targets": [ + { + "expr": "max(nvidia_smi_utilization_gpu_ratio) * 100", + "legendFormat": "GPU", + "refId": "A" + } + ], + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "showThresholdLabels": false, + "showThresholdMarkers": true + } + }, + { + "id": 9, + "title": "GPU Memory Usage (%)", + "description": "Used VRAM as a percentage of total VRAM across all NVIDIA GPUs", + "type": "gauge", + "gridPos": { "h": 6, "w": 6, "x": 6, "y": 24 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "max": 100, + "min": 0, + "noValue": "N/A", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 60 }, + { "color": "orange", "value": 80 }, + { "color": "red", "value": 90 } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "targets": [ + { + "expr": "(sum(nvidia_smi_memory_used_bytes) / sum(nvidia_smi_memory_total_bytes)) * 100", + "legendFormat": "GPU Memory", + "refId": "A" + } + ], + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "showThresholdLabels": false, + "showThresholdMarkers": true + } + }, + { + "id": 10, + "title": "Highest GPU Temperature", + "description": "Highest temperature reported by any NVIDIA GPU", + "type": "gauge", + "gridPos": { "h": 6, "w": 6, "x": 12, "y": 24 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "max": 110, + "min": 0, + "noValue": "N/A", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 70 }, + { "color": "orange", "value": 80 }, + { "color": "red", "value": 90 } + ] + }, + "unit": "celsius" + }, + "overrides": [] + }, + "targets": [ + { + "expr": "max(nvidia_smi_temperature_gpu)", + "legendFormat": "Temperature", + "refId": "A" + } + ], + "options": { + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "showThresholdLabels": false, + "showThresholdMarkers": true + } + }, + { + "id": 11, + "title": "Total GPU Power Draw", + "description": "Combined power draw reported by all NVIDIA GPUs", + "type": "stat", + "gridPos": { "h": 6, "w": 6, "x": 18, "y": 24 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "noValue": "N/A", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null } + ] + }, + "unit": "watt" + }, + "overrides": [] + }, + "targets": [ + { + "expr": "sum(nvidia_smi_power_draw_watts)", + "legendFormat": "Power", + "refId": "A" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto", + "wideLayout": true + } } ], "refresh": "30s", diff --git a/infra/compose/monitoring.docker-compose.yaml b/infra/compose/monitoring.docker-compose.yaml index 1e85a6b3b..41315a156 100644 --- a/infra/compose/monitoring.docker-compose.yaml +++ b/infra/compose/monitoring.docker-compose.yaml @@ -22,11 +22,13 @@ services: image: grafana/grafana:11.2.2 container_name: openrag-grafana ports: - - "3000:3000" + - "127.0.0.1:3000:3000" environment: - GF_SECURITY_ADMIN_USER=${GRAFANA_ADMIN_USER:-admin} - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD:?Set GRAFANA_ADMIN_PASSWORD in .env} - GF_USERS_ALLOW_SIGN_UP=false + - GF_SERVER_ROOT_URL=${GF_SERVER_ROOT_URL:-http://localhost:3000} + - GF_SERVER_SERVE_FROM_SUB_PATH=${GF_SERVER_SERVE_FROM_SUB_PATH:-false} volumes: - ./grafana/provisioning:/etc/grafana/provisioning:ro - ./grafana/dashboards:/var/lib/grafana/dashboards:ro @@ -55,15 +57,13 @@ services: restart: unless-stopped nvidia-gpu-exporter: - image: utkuozdemir/nvidia_gpu_exporter:1.2.0 + image: utkuozdemir/nvidia_gpu_exporter:1.13.1 container_name: openrag-nvidia-gpu-exporter ports: # Localhost only: scraped over the compose network by name. - "127.0.0.1:9835:9835" - volumes: - - /usr/lib/x86_64-linux-gnu/libnvidia-ml.so:/usr/lib/x86_64-linux-gnu/libnvidia-ml.so:ro - - /usr/lib/x86_64-linux-gnu/libnvidia-ml.so.1:/usr/lib/x86_64-linux-gnu/libnvidia-ml.so.1:ro - - /usr/bin/nvidia-smi:/usr/bin/nvidia-smi:ro + environment: + - NVIDIA_DRIVER_CAPABILITIES=utility deploy: resources: reservations: diff --git a/infra/compose/nginx/openrag-admin.conf b/infra/compose/nginx/openrag-admin.conf index be520fe68..e1e47223a 100644 --- a/infra/compose/nginx/openrag-admin.conf +++ b/infra/compose/nginx/openrag-admin.conf @@ -58,6 +58,31 @@ server { try_files $uri $uri/ /app/index.html; } + # Keep Grafana on the same browser-facing origin as the admin UI. + location = /grafana { + return 301 /grafana/; + } + + location /grafana/ { + resolver 127.0.0.11 valid=30s ipv6=off; + set $grafana_upstream grafana; + + proxy_pass http://$grafana_upstream:3000; + + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Forwarded-Proto $forwarded_scheme; + proxy_set_header X-Forwarded-Prefix /grafana; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + + proxy_read_timeout 300s; + } + # Everything else is the OpenRag API (and /chainlit, /auth, /v1, …). location / { # Resolve the backend at request time via Docker's embedded DNS so nginx diff --git a/infra/compose/prometheus/prometheus.yml b/infra/compose/prometheus/prometheus.yml index a2c65015a..2661aaf6a 100644 --- a/infra/compose/prometheus/prometheus.yml +++ b/infra/compose/prometheus/prometheus.yml @@ -21,8 +21,4 @@ scrape_configs: type: Bearer credentials_file: /etc/prometheus/openrag_token static_configs: - - targets: ["host.docker.internal:8000"] - - - job_name: "ray" - static_configs: - - targets: ["host.docker.internal:8080"] + - targets: ["openrag:8080"] diff --git a/infra/docker/api.Dockerfile b/infra/docker/api.Dockerfile index 0e3669f95..d1388c27a 100644 --- a/infra/docker/api.Dockerfile +++ b/infra/docker/api.Dockerfile @@ -94,7 +94,7 @@ RUN useradd --uid ${APP_UID} --gid 0 --no-log-init --no-create-home \ /app/.venv /app/openrag.egg-info /opt/uv/cache \ /app/openrag/.files /app/openrag/.chainlit \ && chgrp -R 0 /app /opt/uv \ - && chmod -R g-w /app /opt/uv \ + && chmod -R g+rX,g-w /app /opt/uv \ && chmod -R g=u /app/home /app/data /app/db /app/logs /app/model_weights \ /app/.venv /app/openrag.egg-info /opt/uv/cache \ /app/openrag/.files /app/openrag/.chainlit diff --git a/openrag/api/main.py b/openrag/api/main.py index e3255a5d8..471c57c7e 100644 --- a/openrag/api/main.py +++ b/openrag/api/main.py @@ -57,6 +57,7 @@ from api.routers.user.health import router as health_router from api.routers.user.search import router as search_router from api.runtime_flags import WITH_CHAINLIT_UI, WITH_OPENAI_API +from api.runtime_ui import get_grafana_url from core.config import load_config from core.utils.banner import print_startup_banner from core.utils.logging import get_logger @@ -340,6 +341,7 @@ def get_config(): **redact_secrets(jsonable_encoder(settings)), "super_admin_mode": SUPER_ADMIN_MODE, "chainlit_enabled": WITH_CHAINLIT_UI, + "grafana_url": get_grafana_url(), } diff --git a/openrag/api/runtime_ui.py b/openrag/api/runtime_ui.py new file mode 100644 index 000000000..4a2de0299 --- /dev/null +++ b/openrag/api/runtime_ui.py @@ -0,0 +1,28 @@ +"""Runtime configuration exposed to the browser-facing Admin UI.""" + +from __future__ import annotations + +import os +from urllib.parse import urlparse + + +def get_grafana_url() -> str | None: + """Return a safe Grafana destination configured for this deployment.""" + value = os.getenv("GRAFANA_URL", "").strip() + if not value or "\\" in value: + return None + + if value.startswith("/") and not value.startswith("//"): + return value + + try: + parsed = urlparse(value) + hostname = parsed.hostname + _ = parsed.port + except ValueError: + return None + + if parsed.scheme in {"http", "https"} and hostname: + return value + + return None diff --git a/tests/unit/api/test_runtime_ui.py b/tests/unit/api/test_runtime_ui.py new file mode 100644 index 000000000..041f73b5f --- /dev/null +++ b/tests/unit/api/test_runtime_ui.py @@ -0,0 +1,39 @@ +"""Tests for deployment-provided Admin UI destinations.""" + +import pytest +from api.runtime_ui import get_grafana_url + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + (None, None), + ("", None), + (" ", None), + ( + "https://grafana.example/d/openrag-http/openrag-http-metrics", + "https://grafana.example/d/openrag-http/openrag-http-metrics", + ), + ( + " http://localhost:3000/d/openrag-http/openrag-http-metrics ", + "http://localhost:3000/d/openrag-http/openrag-http-metrics", + ), + ("/grafana/d/openrag-http/openrag-http-metrics", "/grafana/d/openrag-http/openrag-http-metrics"), + ("/\\evil.example/dashboard", None), + ("https://grafana.example\\@evil.example/dashboard", None), + ("//untrusted.example/dashboard", None), + ("http://[", None), + ("http://[::1", None), + ("http://:3000/dashboard", None), + ("http://grafana.example:99999/dashboard", None), + ("javascript:alert(1)", None), + ("grafana.example/dashboard", None), + ], +) +def test_grafana_url_accepts_only_browser_safe_destinations(monkeypatch, value, expected): + if value is None: + monkeypatch.delenv("GRAFANA_URL", raising=False) + else: + monkeypatch.setenv("GRAFANA_URL", value) + + assert get_grafana_url() == expected diff --git a/ui/.env.example b/ui/.env.example index 6d7488dfc..f5546a26e 100644 --- a/ui/.env.example +++ b/ui/.env.example @@ -1,2 +1,3 @@ VITE_API_BASE_URL=http://localhost:8000 +# Build-time fallback for frontend-only development. Production uses GRAFANA_URL on the API. VITE_GRAFANA_URL=http://localhost:3000/d/system-overview diff --git a/ui/README.md b/ui/README.md index 23589d190..542a67bd7 100644 --- a/ui/README.md +++ b/ui/README.md @@ -30,13 +30,15 @@ npm run lint ## Environment -All settings are build-time (Vite inlines `import.meta.env.*` — rebuild after changing). +All `VITE_*` settings are build-time (Vite inlines `import.meta.env.*` — rebuild after changing). The UI discovers the auth mode (`token` / `oidc`) from the backend at runtime, not from a build var. | Var | Purpose | |-----|---------| | `VITE_API_BASE_URL` | OpenRAG API base (empty = same origin) | | `VITE_BASE_PATH` | base path when served under a sub-path (default `/`) | -| `VITE_GRAFANA_URL` | Grafana dashboard link shown on the admin System page | +| `VITE_GRAFANA_URL` | Build-time fallback for the Grafana dashboard link | | `VITE_APP_NAME` | app display name / branding (default `OpenRAG`) | | `VITE_MOCK_API` | `true` to serve MSW mocks in dev | + +Production deployments should set `GRAFANA_URL` on the OpenRAG API. The Admin UI reads it at runtime, so the same prebuilt UI image can point to different dashboards. diff --git a/ui/src/lib/api/system.ts b/ui/src/lib/api/system.ts index f9eaa29e8..7d8e5ba9f 100644 --- a/ui/src/lib/api/system.ts +++ b/ui/src/lib/api/system.ts @@ -12,6 +12,12 @@ export interface VersionResponse { version: string; } +export interface SystemConfig extends Record { + chainlit_enabled?: boolean; + grafana_url?: string | null; + super_admin_mode?: boolean; +} + export interface RayActor { actor_id: string; name: string; @@ -29,7 +35,7 @@ export function getVersion() { } export function getConfig() { - return request>("/config"); + return request("/config"); } export function getMetrics() { diff --git a/ui/src/mocks/handlers.ts b/ui/src/mocks/handlers.ts index 39c434110..69e9e1d52 100644 --- a/ui/src/mocks/handlers.ts +++ b/ui/src/mocks/handlers.ts @@ -664,6 +664,7 @@ export const handlers = [ // not nested under auth — the permission layer reads config.super_admin_mode. super_admin_mode: true, chainlit_enabled: true, + grafana_url: "http://localhost:3000/d/openrag-http/openrag-http-metrics", auth: { auth_mode: "oidc" }, milvus: { host: "milvus", port: 19530, database: "openrag" }, ray: { address: "ray://ray-head:10001", pool_size: 2, max_tasks_per_worker: 2 }, diff --git a/ui/src/pages/admin/system.test.tsx b/ui/src/pages/admin/system.test.tsx new file mode 100644 index 000000000..b2feb4a30 --- /dev/null +++ b/ui/src/pages/admin/system.test.tsx @@ -0,0 +1,128 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import SystemPage from "./system"; + +const systemConfig = vi.hoisted(() => ({ + grafanaUrl: null as string | null, + metricsLoading: false, + refetchOnMount: undefined as unknown, +})); + +vi.mock("@tanstack/react-query", () => ({ + useQuery: ({ + queryKey, + refetchOnMount, + }: { + queryKey: unknown[]; + refetchOnMount?: unknown; + }) => { + const key = queryKey[0]; + if (key === "system-config") { + systemConfig.refetchOnMount = refetchOnMount; + return { + data: { grafana_url: systemConfig.grafanaUrl }, + error: null, + isLoading: false, + }; + } + if (key === "system-health") { + return { + isLoading: false, + isSuccess: true, + refetch: vi.fn(), + }; + } + if (key === "system-version") { + return { + data: { version: "2.0.2" }, + isLoading: false, + }; + } + if (key === "system-actors") { + return { + data: { actors: [] }, + isLoading: false, + }; + } + if (key === "system-metrics") { + return { + data: "openrag_requests_total 1", + isLoading: systemConfig.metricsLoading, + }; + } + return { + data: undefined, + isLoading: false, + }; + }, + useMutation: () => ({ + isPending: false, + mutate: vi.fn(), + }), + useQueryClient: () => ({ + invalidateQueries: vi.fn(), + }), +})); + +describe("SystemPage Grafana action", () => { + beforeEach(() => { + systemConfig.grafanaUrl = null; + systemConfig.metricsLoading = false; + systemConfig.refetchOnMount = undefined; + }); + + it("refetches runtime configuration whenever the page mounts", () => { + render(); + + expect(systemConfig.refetchOnMount).toBe("always"); + }); + + it("opens the runtime-configured dashboard from the Metrics tab", async () => { + systemConfig.grafanaUrl = + "https://grafana.example/d/openrag-http/openrag-http-metrics"; + + render(); + + await userEvent.click(screen.getByRole("tab", { name: "Metrics" })); + + const link = screen.getByRole("link", { + name: "Open metrics dashboard in Grafana (opens in a new tab)", + }); + expect(link.getAttribute("href")).toBe( + "https://grafana.example/d/openrag-http/openrag-http-metrics", + ); + expect(link.getAttribute("target")).toBe("_blank"); + expect(link.getAttribute("rel")).toBe("noopener noreferrer"); + }); + + it("keeps the Grafana action visible while metrics are loading", async () => { + systemConfig.grafanaUrl = + "https://grafana.example/d/openrag-http/openrag-http-metrics"; + systemConfig.metricsLoading = true; + + render(); + + await userEvent.click(screen.getByRole("tab", { name: "Metrics" })); + + expect( + screen.getByRole("link", { + name: "Open metrics dashboard in Grafana (opens in a new tab)", + }), + ).not.toBeNull(); + }); + + it("explains how to configure Grafana when the dashboard URL is missing", async () => { + render(); + + expect(screen.queryByRole("link", { name: /grafana/i })).toBeNull(); + + await userEvent.click(screen.getByRole("tab", { name: "Metrics" })); + await userEvent.click(screen.getByRole("button", { name: "Open in Grafana" })); + + expect(screen.getByRole("dialog")).not.toBeNull(); + expect(screen.getByRole("heading", { name: "Grafana is not configured" })).not.toBeNull(); + expect(screen.getByText("GRAFANA_URL")).not.toBeNull(); + }); +}); diff --git a/ui/src/pages/admin/system.tsx b/ui/src/pages/admin/system.tsx index 67de38263..7068f87f2 100644 --- a/ui/src/pages/admin/system.tsx +++ b/ui/src/pages/admin/system.tsx @@ -16,8 +16,17 @@ import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; import { Badge } from "@/components/ui/badge"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { Skeleton } from "@/components/ui/skeleton"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, + DialogTrigger, +} from "@/components/ui/dialog"; -const GRAFANA_URL = import.meta.env.VITE_GRAFANA_URL || ""; +const BUILD_TIME_GRAFANA_URL = import.meta.env.VITE_GRAFANA_URL || ""; function actorStateColor(state: string): string { const s = state.toUpperCase(); @@ -27,21 +36,21 @@ function actorStateColor(state: string): string { } export default function SystemPage() { + const { data: config } = useQuery({ + queryKey: ["system-config"], + queryFn: getConfig, + staleTime: Infinity, + refetchOnMount: "always", + }); + const runtimeGrafanaUrl = + typeof config?.grafana_url === "string" ? config.grafana_url.trim() : ""; + const grafanaUrl = runtimeGrafanaUrl || BUILD_TIME_GRAFANA_URL; + return (
- - - View in Grafana - - - ) : undefined - } /> @@ -59,7 +68,7 @@ export default function SystemPage() { - + @@ -174,15 +183,13 @@ function ActorsTab() { ); } -function MetricsTab() { +function MetricsTab({ grafanaUrl }: { grafanaUrl: string }) { const { data, isLoading } = useQuery({ queryKey: ["system-metrics"], queryFn: getMetrics, refetchInterval: 10000, }); - if (isLoading) return ; - // Parse Prometheus text format into simple metric entries. const lines = (data || "").split("\n").filter((l) => l && !l.startsWith("#")); const parsed = lines.map((line) => { @@ -196,17 +203,44 @@ function MetricsTab() { Prometheus Metrics - {GRAFANA_URL && ( + {grafanaUrl ? ( + ) : ( + + + + + + + Grafana is not configured + + Set GRAFANA_URL to the + browser-reachable OpenRAG dashboard URL, then restart the API. This action will + open that dashboard without rebuilding the Admin UI. + + + + + )} - {parsed.length === 0 ? ( + {isLoading ? ( + + ) : parsed.length === 0 ? (

No metrics available

) : (