AZ-Intec-GmbH/AZ-NIX-SERVERS
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: litellm -> grafana

Browse Source
This commit is contained in:
Sascha König
2026-07-13 10:32:34 +02:00
parent 8d57aa4bc0
commit d58173760d
8 changed files with 849 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hosts/AZ-CLD-1/services/containers/litellm.nix
+104 -5
View File
@@ -1,20 +1,97 @@
{config, ...}: let
{
config,
pkgs,
...
}: let
serviceName = "litellm";
servicePort = config.m3ta.ports.get serviceName;
prmNetbirdIP = "100.91.49.26";
cldNetbirdIP = "100.91.203.184";
python = pkgs.python3.withPackages (ps: [ps.pyyaml]);
litellmConfigGenerator = pkgs.writeText "litellm-config-generator.py" ''
import sys
from pathlib import Path
import yaml
base_path = Path(sys.argv[1])
target_path = Path(sys.argv[2])
with base_path.open("r", encoding="utf-8") as fh:
config = yaml.safe_load(fh) or {}
settings = config.get("litellm_settings")
if not isinstance(settings, dict):
settings = {}
config["litellm_settings"] = settings
def ensure_list(value):
if value is None:
return []
if isinstance(value, list):
return value
return [value]
callbacks = ensure_list(settings.get("callbacks"))
if "prometheus" not in callbacks:
callbacks.append("prometheus")
settings["callbacks"] = callbacks
service_callbacks = ensure_list(settings.get("service_callback"))
if "prometheus_system" not in service_callbacks:
service_callbacks.append("prometheus_system")
settings["service_callback"] = service_callbacks
# LiteLLM >= 1.85 protects /metrics by default. Keep auth enabled;
# Prometheus scrapes with a bearer token from an agenix secret on AZ-PRM-1.
settings["require_auth_for_metrics_endpoint"] = True
target_path.parent.mkdir(parents=True, exist_ok=True)
with target_path.open("w", encoding="utf-8") as fh:
yaml.safe_dump(config, fh, sort_keys=False)
'';
in {
systemd.services."podman-${serviceName}".preStart = ''
set -euo pipefail
base_config="/var/lib/${serviceName}/config.yaml"
target_config="/run/${serviceName}/config.yaml"
multiproc_dir="/var/lib/${serviceName}/prometheus-multiproc"
if [ ! -f "$base_config" ]; then
echo "Missing LiteLLM base config: $base_config" >&2
exit 1
fi
mkdir -p "$(dirname "$target_config")" "$multiproc_dir"
rm -f "$multiproc_dir"/*
chmod 0777 "$multiproc_dir"
${python}/bin/python ${litellmConfigGenerator} "$base_config" "$target_config"
chmod 0644 "$target_config"
'';
virtualisation.oci-containers.containers.${serviceName} = {
#image = "ghcr.io/berriai/litellm:v1.78.5-stable";
image = "docker.litellm.ai/berriai/litellm:1.89.0";
ports = ["127.0.0.1:${toString servicePort}:4000"];
image = "docker.litellm.ai/berriai/litellm:1.92.0";
ports = [
"127.0.0.1:${toString servicePort}:4000"
"${cldNetbirdIP}:${toString servicePort}:4000"
];
cmd = ["--config" "/app/config.yaml" "--port" "4000"];
environmentFiles = [config.age.secrets.litellm-env.path];
environment = {
ANONYMIZED_TELEMETRY = "False";
DO_NOT_TRACK = "True";
SCARF_NO_ANALYTICS = "True";
STORE_MODEL_IN_DB = "True";
LITELLM_LOG = "json";
PROMETHEUS_MULTIPROC_DIR = "/prometheus_multiproc";
};
volumes = ["/var/lib/litellm/config.yaml:/app/config.yaml"];
volumes = [
"/run/${serviceName}/config.yaml:/app/config.yaml:ro"
"/var/lib/${serviceName}/prometheus-multiproc:/prometheus_multiproc"
];
extraOptions = ["--add-host=postgres:10.89.0.1" "--ip=10.89.0.30" "--network=web"];
};
@@ -26,6 +103,23 @@ in {
}
];
middlewares."${serviceName}-metrics-local-only".ipAllowList.sourceRange = [
"127.0.0.1/32"
"::1/128"
"${prmNetbirdIP}/32"
];
routers."${serviceName}-metrics" = {
rule = "Host(`llm.az-gruppe.com`) && PathPrefix(`/metrics`)";
tls = {
certResolver = "ionos";
};
service = serviceName;
entrypoints = "websecure";
middlewares = ["${serviceName}-metrics-local-only"];
priority = 200;
};
routers.${serviceName} = {
rule = "Host(`llm.az-gruppe.com`)";
tls = {
@@ -33,6 +127,11 @@ in {
};
service = serviceName;
entrypoints = "websecure";
priority = 1;
};
};
networking.firewall.extraCommands = ''
iptables -A INPUT -p tcp -s ${prmNetbirdIP} --dport ${toString servicePort} -j ACCEPT
'';
}
hosts/AZ-PRM-1/secrets.nix
+9
View File
@@ -1,5 +1,6 @@
{lib, ...}: let
ntfyGrafanaWebhookSecret = ../../secrets/ntfy-grafana-webhook.age;
litellmPrometheusBearerSecret = ../../secrets/litellm-prometheus-bearer.age;
in {
age = {
secrets =
@@ -64,6 +65,14 @@ in {
file = ntfyGrafanaWebhookSecret;
mode = "0400";
};
}
// lib.optionalAttrs (builtins.pathExists litellmPrometheusBearerSecret) {
litellm-prometheus-bearer = {
file = litellmPrometheusBearerSecret;
owner = "prometheus";
group = "prometheus";
mode = "0400";
};
};
};
}
hosts/AZ-PRM-1/services/monitoring/alerting.nix
+41
View File
@@ -195,6 +195,47 @@ in {
};
annotations.description = "A Kestra flow execution ended with FAILED, WARNING, or KILLED.";
})
(mkAlertRule {
uid = "az_litellm_down";
title = "LiteLLM metrics unreachable";
expr = ''up{job="litellm"}'';
for = "2m";
evaluatorType = "lt";
evaluatorParams = [1 0];
labels = {
service = "litellm";
severity = "critical";
};
annotations.description = "Prometheus cannot scrape LiteLLM /metrics on AZ-CLD-1 over Netbird.";
})
(mkAlertRule {
uid = "az_litellm_proxy_errors";
title = "LiteLLM proxy errors";
expr = ''sum(increase(litellm_proxy_failed_requests_metric_total{job="litellm"}[5m]))'';
for = "1m";
noDataState = "OK";
evaluatorType = "gt";
evaluatorParams = [0 0];
labels = {
service = "litellm";
severity = "warning";
};
annotations.description = "LiteLLM reported one or more failed proxy responses in the last 5 minutes.";
})
(mkAlertRule {
uid = "az_litellm_latency_high";
title = "LiteLLM latency high";
expr = ''histogram_quantile(0.95, sum(rate(litellm_request_total_latency_metric_bucket{job="litellm"}[5m])) by (le))'';
for = "10m";
noDataState = "OK";
evaluatorType = "gt";
evaluatorParams = [30 0];
labels = {
service = "litellm";
severity = "warning";
};
annotations.description = "LiteLLM p95 total request latency has been above 30 seconds for 10 minutes.";
})
(mkAlertRule {
uid = "az_http_probe_failed";
title = "HTTP probe failed";
hosts/AZ-PRM-1/services/monitoring/cld-scrape.nix
+34 -3
View File
@@ -1,7 +1,23 @@
{config, ...}: let
{
config,
lib,
...
}: let
nodeExporterPort = config.m3ta.ports.get "node-exporter";
litellmPort = config.m3ta.ports.get "litellm";
cldNetbirdIP = "100.91.203.184";
litellmPrometheusBearerSecretAvailable = builtins.hasAttr "litellm-prometheus-bearer" config.age.secrets;
litellmPrometheusAuthorization = lib.optionalAttrs litellmPrometheusBearerSecretAvailable {
authorization = {
type = "Bearer";
credentials_file = config.age.secrets."litellm-prometheus-bearer".path;
};
};
in {
# Prometheus' config checker validates credentials_file paths at build time,
# but agenix secrets only exist at runtime under /run/agenix.
services.prometheus.checkConfig = lib.mkIf litellmPrometheusBearerSecretAvailable false;
services.prometheus.scrapeConfigs = [
{
job_name = "node-cld";
@@ -14,11 +30,26 @@ in {
}
];
}
({
job_name = "litellm";
metrics_path = "/metrics/";
scrape_interval = "15s";
static_configs = [
{
targets = ["${cldNetbirdIP}:${toString litellmPort}"];
labels = {
instance = "AZ-CLD-1";
service = "litellm";
};
}
];
}
// litellmPrometheusAuthorization)
];
# Allow CLD to reach PRM prometheus scrapes (prometheus initiates connection TO CLD exporter)
# Allow CLD to reach PRM prometheus scrapes (prometheus initiates connection TO CLD exporters)
networking.firewall.extraCommands = ''
# No PRM-side firewall change needed: PRM scrapes outbound to CLD:9100.
# No PRM-side firewall change needed: PRM scrapes outbound to CLD:9100/4000.
# CLD-side must allow inbound from 100.91.49.26 (PRM netbird IP) — see CLD config.
'';
}
hosts/AZ-PRM-1/services/monitoring/dashboards/az-litellm-observability.json
+568
View File
@@ -0,0 +1,568 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"mappings": [
{
"options": {
"0": {
"color": "red",
"index": 1,
"text": "down"
},
"1": {
"color": "green",
"index": 0,
"text": "up"
}
},
"type": "value"
}
],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 1
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 6,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"colorMode": "background",
"graphMode": "none",
"justifyMode": "center",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "11.0.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code",
"expr": "up{job=\"litellm\", instance=~\"$instance\"}",
"legendFormat": "{{instance}}",
"range": true,
"refId": "A"
}
],
"title": "Scrape status",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"decimals": 3,
"unit": "reqps"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 6,
"x": 6,
"y": 0
},
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "11.0.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code",
"expr": "sum(rate(litellm_proxy_total_requests_metric_total{job=\"litellm\", instance=~\"$instance\"}[$__rate_interval]))",
"legendFormat": "requests/s",
"range": true,
"refId": "A"
}
],
"title": "Proxy request rate",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"decimals": 3,
"unit": "reqps"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 6,
"x": 12,
"y": 0
},
"id": 3,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "11.0.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code",
"expr": "sum(rate(litellm_proxy_failed_requests_metric_total{job=\"litellm\", instance=~\"$instance\"}[$__rate_interval])) or vector(0)",
"legendFormat": "errors/s",
"range": true,
"refId": "A"
}
],
"title": "Proxy error rate",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"decimals": 2,
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 6,
"x": 18,
"y": 0
},
"id": 4,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"pluginVersion": "11.0.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code",
"expr": "histogram_quantile(0.95, sum(rate(litellm_request_total_latency_metric_bucket{job=\"litellm\", instance=~\"$instance\"}[$__rate_interval])) by (le))",
"legendFormat": "p95",
"range": true,
"refId": "A"
}
],
"title": "p95 total latency",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineInterpolation": "linear",
"lineWidth": 2,
"showPoints": "never",
"spanNulls": false
},
"unit": "reqps"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 4
},
"id": 5,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code",
"expr": "sum by (requested_model) (rate(litellm_proxy_total_requests_metric_total{job=\"litellm\", instance=~\"$instance\"}[$__rate_interval]))",
"legendFormat": "{{requested_model}}",
"range": true,
"refId": "A"
}
],
"title": "Requests by requested model",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineInterpolation": "linear",
"lineWidth": 2,
"showPoints": "never",
"spanNulls": false
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 4
},
"id": 6,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code",
"expr": "sum by (model) (rate(litellm_total_tokens_metric_total{job=\"litellm\", instance=~\"$instance\"}[$__rate_interval]))",
"legendFormat": "{{model}}",
"range": true,
"refId": "A"
}
],
"title": "Token rate by model",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "currencyUSD"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 12
},
"id": 7,
"options": {
"displayMode": "gradient",
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": true
},
"showUnfilled": true,
"valueMode": "color"
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code",
"expr": "topk(10, sum by (model) (increase(litellm_spend_metric_total{job=\"litellm\", instance=~\"$instance\"}[$__range])))",
"legendFormat": "{{model}}",
"range": true,
"refId": "A"
}
],
"title": "Spend by model in selected range",
"type": "bargauge"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"custom": {
"drawStyle": "line",
"fillOpacity": 10,
"lineInterpolation": "linear",
"lineWidth": 2,
"showPoints": "never",
"spanNulls": false
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 12
},
"id": 8,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"editorMode": "code",
"expr": "histogram_quantile(0.95, sum by (le, model) (rate(litellm_llm_api_latency_metric_bucket{job=\"litellm\", instance=~\"$instance\"}[$__rate_interval])))",
"legendFormat": "{{model}} p95",
"range": true,
"refId": "A"
}
],
"title": "LLM API latency by model",
"type": "timeseries"
},
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 20
},
"id": 9,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"editorMode": "code",
"expr": "{host=\"AZ-CLD-1\", unit=~\"podman-litellm.service|docker-litellm.service|litellm.service\"}",
"queryType": "range",
"refId": "A"
}
],
"title": "LiteLLM logs from Loki",
"type": "logs"
}
],
"refresh": "30s",
"schemaVersion": 39,
"style": "dark",
"tags": [
"az",
"litellm",
"prometheus",
"loki",
"provisioned"
],
"templating": {
"list": [
{
"current": {
"selected": true,
"text": "All",
"value": "$__all"
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"definition": "label_values(up{job=\"litellm\"}, instance)",
"hide": 0,
"includeAll": true,
"label": "Instance",
"multi": true,
"name": "instance",
"options": [],
"query": {
"query": "label_values(up{job=\"litellm\"}, instance)",
"refId": "PrometheusVariableQueryEditor-VariableQuery"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"type": "query"
}
]
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "AZ LiteLLM Observability",
"uid": "az-litellm-observability",
"version": 1,
"weekStart": ""
}