AZ-Intec-GmbH/AZ-NIX-SERVERS
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: litellm -> grafana

Browse Source
This commit is contained in:
Sascha König
2026-07-13 10:32:34 +02:00
parent 8d57aa4bc0
commit d58173760d
8 changed files with 849 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hosts/AZ-PRM-1/secrets.nix
+9
View File
@@ -1,5 +1,6 @@
{lib, ...}: let
ntfyGrafanaWebhookSecret = ../../secrets/ntfy-grafana-webhook.age;
litellmPrometheusBearerSecret = ../../secrets/litellm-prometheus-bearer.age;
in {
age = {
secrets =
@@ -64,6 +65,14 @@ in {
file = ntfyGrafanaWebhookSecret;
mode = "0400";
};
}
// lib.optionalAttrs (builtins.pathExists litellmPrometheusBearerSecret) {
litellm-prometheus-bearer = {
file = litellmPrometheusBearerSecret;
owner = "prometheus";
group = "prometheus";
mode = "0400";
};
};
};
}
hosts/AZ-PRM-1/services/monitoring/alerting.nix
+41
View File
@@ -195,6 +195,47 @@ in {
};
annotations.description = "A Kestra flow execution ended with FAILED, WARNING, or KILLED.";
})
(mkAlertRule {
uid = "az_litellm_down";
title = "LiteLLM metrics unreachable";
expr = ''up{job="litellm"}'';
for = "2m";
evaluatorType = "lt";
evaluatorParams = [1 0];
labels = {
service = "litellm";
severity = "critical";
};
annotations.description = "Prometheus cannot scrape LiteLLM /metrics on AZ-CLD-1 over Netbird.";
})
(mkAlertRule {
uid = "az_litellm_proxy_errors";
title = "LiteLLM proxy errors";
expr = ''sum(increase(litellm_proxy_failed_requests_metric_total{job="litellm"}[5m]))'';
for = "1m";
noDataState = "OK";
evaluatorType = "gt";
evaluatorParams = [0 0];
labels = {
service = "litellm";
severity = "warning";
};
annotations.description = "LiteLLM reported one or more failed proxy responses in the last 5 minutes.";
})
(mkAlertRule {
uid = "az_litellm_latency_high";
title = "LiteLLM latency high";
expr = ''histogram_quantile(0.95, sum(rate(litellm_request_total_latency_metric_bucket{job="litellm"}[5m])) by (le))'';
for = "10m";
noDataState = "OK";
evaluatorType = "gt";
evaluatorParams = [30 0];
labels = {
service = "litellm";
severity = "warning";
};
annotations.description = "LiteLLM p95 total request latency has been above 30 seconds for 10 minutes.";
})
(mkAlertRule {
uid = "az_http_probe_failed";
title = "HTTP probe failed";
hosts/AZ-PRM-1/services/monitoring/cld-scrape.nix
+34 -3
View File
@@ -1,7 +1,23 @@
{config, ...}: let
{
config,
lib,
...
}: let
nodeExporterPort = config.m3ta.ports.get "node-exporter";
litellmPort = config.m3ta.ports.get "litellm";
cldNetbirdIP = "100.91.203.184";
litellmPrometheusBearerSecretAvailable = builtins.hasAttr "litellm-prometheus-bearer" config.age.secrets;
litellmPrometheusAuthorization = lib.optionalAttrs litellmPrometheusBearerSecretAvailable {
authorization = {
type = "Bearer";
credentials_file = config.age.secrets."litellm-prometheus-bearer".path;
};
};
in {
# Prometheus' config checker validates credentials_file paths at build time,
# but agenix secrets only exist at runtime under /run/agenix.
services.prometheus.checkConfig = lib.mkIf litellmPrometheusBearerSecretAvailable false;
services.prometheus.scrapeConfigs = [
{
job_name = "node-cld";
@@ -14,11 +30,26 @@ in {
}
];
}
({
job_name = "litellm";
metrics_path = "/metrics/";
scrape_interval = "15s";
static_configs = [
{
targets = ["${cldNetbirdIP}:${toString litellmPort}"];
labels = {
instance = "AZ-CLD-1";
service = "litellm";
};
}
];
}
// litellmPrometheusAuthorization)
];
# Allow CLD to reach PRM prometheus scrapes (prometheus initiates connection TO CLD exporter)
# Allow CLD to reach PRM prometheus scrapes (prometheus initiates connection TO CLD exporters)
networking.firewall.extraCommands = ''
# No PRM-side firewall change needed: PRM scrapes outbound to CLD:9100.
# No PRM-side firewall change needed: PRM scrapes outbound to CLD:9100/4000.
# CLD-side must allow inbound from 100.91.49.26 (PRM netbird IP) — see CLD config.
'';
}
hosts/AZ-PRM-1/services/monitoring/dashboards/az-litellm-observability.json
+568
View File
@@ -0,0 +1,568 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"mappings": [
{
"options": {
"0": {
"color": "red",
"index": 1,
"text": "down"
},
"1": {
"color": "green",
"index": 0,
"text": "up"
}
},
"type": "value"
}
],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 1
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 6,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"colorMode": "background",
"graphMode": "none",
"justifyMode": "center",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "11.0.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code",
"expr": "up{job=\"litellm\", instance=~\"$instance\"}",
"legendFormat": "{{instance}}",
"range": true,
"refId": "A"
}
],
"title": "Scrape status",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"decimals": 3,
"unit": "reqps"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 6,
"x": 6,
"y": 0
},
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "11.0.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code",
"expr": "sum(rate(litellm_proxy_total_requests_metric_total{job=\"litellm\", instance=~\"$instance\"}[$__rate_interval]))",
"legendFormat": "requests/s",
"range": true,
"refId": "A"
}
],
"title": "Proxy request rate",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"decimals": 3,
"unit": "reqps"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 6,
"x": 12,
"y": 0
},
"id": 3,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "11.0.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code",
"expr": "sum(rate(litellm_proxy_failed_requests_metric_total{job=\"litellm\", instance=~\"$instance\"}[$__rate_interval])) or vector(0)",
"legendFormat": "errors/s",
"range": true,
"refId": "A"
}
],
"title": "Proxy error rate",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"decimals": 2,
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 6,
"x": 18,
"y": 0
},
"id": 4,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "11.0.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code",
"expr": "histogram_quantile(0.95, sum(rate(litellm_request_total_latency_metric_bucket{job=\"litellm\", instance=~\"$instance\"}[$__rate_interval])) by (le))",
"legendFormat": "p95",
"range": true,
"refId": "A"
}
],
"title": "p95 total latency",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineInterpolation": "linear",
"lineWidth": 2,
"showPoints": "never",
"spanNulls": false
},
"unit": "reqps"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 4
},
"id": 5,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code",
"expr": "sum by (requested_model) (rate(litellm_proxy_total_requests_metric_total{job=\"litellm\", instance=~\"$instance\"}[$__rate_interval]))",
"legendFormat": "{{requested_model}}",
"range": true,
"refId": "A"
}
],
"title": "Requests by requested model",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineInterpolation": "linear",
"lineWidth": 2,
"showPoints": "never",
"spanNulls": false
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 4
},
"id": 6,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code",
"expr": "sum by (model) (rate(litellm_total_tokens_metric_total{job=\"litellm\", instance=~\"$instance\"}[$__rate_interval]))",
"legendFormat": "{{model}}",
"range": true,
"refId": "A"
}
],
"title": "Token rate by model",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "currencyUSD"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 12
},
"id": 7,
"options": {
"displayMode": "gradient",
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": true
},
"showUnfilled": true,
"valueMode": "color"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code",
"expr": "topk(10, sum by (model) (increase(litellm_spend_metric_total{job=\"litellm\", instance=~\"$instance\"}[$__range])))",
"legendFormat": "{{model}}",
"range": true,
"refId": "A"
}
],
"title": "Spend by model in selected range",
"type": "bargauge"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineInterpolation": "linear",
"lineWidth": 2,
"showPoints": "never",
"spanNulls": false
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 12
},
"id": 8,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code",
"expr": "histogram_quantile(0.95, sum by (le, model) (rate(litellm_llm_api_latency_metric_bucket{job=\"litellm\", instance=~\"$instance\"}[$__rate_interval])))",
"legendFormat": "{{model}} p95",
"range": true,
"refId": "A"
}
],
"title": "LLM API latency by model",
"type": "timeseries"
},
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 20
},
"id": 9,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"editorMode": "code",
"expr": "{host=\"AZ-CLD-1\", unit=~\"podman-litellm.service|docker-litellm.service|litellm.service\"}",
"queryType": "range",
"refId": "A"
}
],
"title": "LiteLLM logs from Loki",
"type": "logs"
}
],
"refresh": "30s",
"schemaVersion": 39,
"style": "dark",
"tags": [
"az",
"litellm",
"prometheus",
"loki",
"provisioned"
],
"templating": {
"list": [
{
"current": {
"selected": true,
"text": "All",
"value": "$__all"
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"definition": "label_values(up{job=\"litellm\"}, instance)",
"hide": 0,
"includeAll": true,
"label": "Instance",
"multi": true,
"name": "instance",
"options": [],
"query": {
"query": "label_values(up{job=\"litellm\"}, instance)",
"refId": "PrometheusVariableQueryEditor-VariableQuery"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"type": "query"
}
]
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "AZ LiteLLM Observability",
"uid": "az-litellm-observability",
"version": 1,
"weekStart": ""
}