From d58173760dd4450db2a9b28e02787bcce5aa4afd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sascha=20K=C3=B6nig?= Date: Mon, 13 Jul 2026 10:32:34 +0200 Subject: [PATCH] fix: litellm -> grafana --- flake.lock | 168 +++--- .../AZ-CLD-1/services/containers/litellm.nix | 109 +++- hosts/AZ-PRM-1/secrets.nix | 9 + .../AZ-PRM-1/services/monitoring/alerting.nix | 41 ++ .../services/monitoring/cld-scrape.nix | 37 +- .../dashboards/az-litellm-observability.json | 568 ++++++++++++++++++ secrets.nix | 1 + secrets/litellm-prometheus-bearer.age | 11 + 8 files changed, 849 insertions(+), 95 deletions(-) create mode 100644 hosts/AZ-PRM-1/services/monitoring/dashboards/az-litellm-observability.json create mode 100644 secrets/litellm-prometheus-bearer.age diff --git a/flake.lock b/flake.lock index 5c57508..fe2a687 100644 --- a/flake.lock +++ b/flake.lock @@ -66,11 +66,11 @@ "agents": { "flake": false, "locked": { - "lastModified": 1780133320, - "narHash": "sha256-8AiN9tV9PBb5xblJiPlhumBbKj61qLjzqXXFtkj3vvY=", + "lastModified": 1783016143, + "narHash": "sha256-3qejDAo4g2XfzTyWCaX8RVKH17JXnw/fujsbXQA+Mzc=", "ref": "refs/heads/master", - "rev": "920c00313ae242bd93275c30131b9ab1e52ee2fb", - "revCount": 88, + "rev": "83284d752d2ac325d4bf3ab7b38acad1c2fffe3a", + "revCount": 98, "type": "git", "url": "https://code.m3ta.dev/m3tam3re/AGENTS" }, @@ -84,11 +84,11 @@ "nixpkgs": "nixpkgs_4" }, "locked": { - "lastModified": 1782789853, - "narHash": "sha256-LEmctqYRQRq0wB1MoS+IVr/0/uu/0nKahqNeJBGTjJ8=", + "lastModified": 1783016143, + "narHash": "sha256-3qejDAo4g2XfzTyWCaX8RVKH17JXnw/fujsbXQA+Mzc=", "ref": "refs/heads/master", - "rev": "cd36a91440b971582fcf2d4eedf9a46fb755f6e1", - "revCount": 92, + "rev": "83284d752d2ac325d4bf3ab7b38acad1c2fffe3a", + "revCount": 98, "type": "git", "url": "ssh://gitea@code.m3ta.dev/m3tam3re/AGENTS" }, @@ -409,18 +409,18 @@ }, "dms": { "inputs": { + "flake-compat": "flake-compat", "nixpkgs": [ "m3ta-home", "nixpkgs" - ], - "quickshell": "quickshell" + ] }, "locked": { - "lastModified": 1777431599, - "narHash": "sha256-g6r/Gx8PTDzO3jCNzzySA+Ff1lmLF9nDlMCNyyoQjoE=", + "lastModified": 1783529634, + "narHash": "sha256-zdHsPGPE5MVi/y+uIt548XScTfZjQzdF21dME7ISEJM=", "owner": "AvengeMedia", "repo": "DankMaterialShell", - "rev": "eb5afcdc40ea5446c27e18552ff4a19f9daf9484", + "rev": "bdfd565b72cbb416441a9ab828efcac4fd516c70", "type": "github" }, "original": { @@ -438,11 +438,11 @@ ] }, "locked": { - "lastModified": 1782780764, - "narHash": "sha256-Q2wPFsuDo1y6neZ3ttxxOOItupg/4+mPkyAZbC0+wfw=", + "lastModified": 1783591397, + "narHash": "sha256-e4DDmVAas6IPwwY+gfSDKh3PMzW/FXLCbjyWXOnnnbM=", "owner": "AvengeMedia", "repo": "dms-plugin-registry", - "rev": "9fa716427ab5905845b2a10a01d67bee0b1e4c4b", + "rev": "8b61e8fca730986e673737529fa56802dc9326bb", "type": "github" }, "original": { @@ -451,6 +451,22 @@ "type": "github" } }, + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1767039857, + "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=", + "owner": "NixOS", + "repo": "flake-compat", + "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "flake-compat", + "type": "github" + } + }, "flake-parts": { "inputs": { "nixpkgs-lib": [ @@ -568,15 +584,16 @@ "uv2nix": "uv2nix_2" }, "locked": { - "lastModified": 1782794050, - "narHash": "sha256-+qKEwIhTkml3DNSmIvGFJyssW9ZXTp78KGclFdcDvX8=", + "lastModified": 1782936460, + "narHash": "sha256-Wt72AQtA6Eizi7Ubj23JBhwZ7GKYcjY4mcV6upqHOaU=", "owner": "NousResearch", "repo": "hermes-agent", - "rev": "972b1620906a1b80772c2f67492ad50b3c83f048", + "rev": "7c1a029553d87c43ecff8a3821336bc95872213b", "type": "github" }, "original": { "owner": "NousResearch", + "ref": "v2026.7.1", "repo": "hermes-agent", "type": "github" } @@ -609,11 +626,11 @@ ] }, "locked": { - "lastModified": 1781981105, - "narHash": "sha256-/1nNBbA7PrSQpTc9Qazkhl4kIPg+TNl0CjxS3UQJKlw=", + "lastModified": 1783221248, + "narHash": "sha256-ESQnuNHEDChsB4IxoLRhscVahqkDWkTb+qdIz8euYt4=", "owner": "nix-community", "repo": "home-manager", - "rev": "7bfff44b465909f69a442701293bc0badcf476dc", + "rev": "af2beae5f0fae0a4310cc0e6aef2572f56090353", "type": "github" }, "original": { @@ -653,11 +670,11 @@ ] }, "locked": { - "lastModified": 1782749631, - "narHash": "sha256-slFTUgDy0KTPA4LBAmC/9SngDq8GCPdX+ZR0yQHHN1E=", + "lastModified": 1783550008, + "narHash": "sha256-qbbZUk9IG9dLNwCPwdPBs6I5clxwugRpP+1YU+GPK20=", "owner": "nix-community", "repo": "home-manager", - "rev": "5d72a29fc36ac21adae6ae35568fe5ee6700850f", + "rev": "f4d01c1d87c7c2ec909549165d5a8338f1bd3315", "type": "github" }, "original": { @@ -676,11 +693,11 @@ "treefmt-nix": "treefmt-nix" }, "locked": { - "lastModified": 1783567605, - "narHash": "sha256-LyF84yqWppXAAEOAkL325Lt+DrVmkbbeXGzDgX9zMzI=", + "lastModified": 1783669679, + "narHash": "sha256-WDGDMsPXxceC0k1ktqcobmwE4GCTrzzULUKlp1CzSKQ=", "owner": "numtide", "repo": "llm-agents.nix", - "rev": "a4c847460f0e773d02d0655ce28dc6b532dd65d6", + "rev": "6e9f6664e72433966557ab83a7c3e8c0bbf64a44", "type": "github" }, "original": { @@ -706,11 +723,11 @@ "nur": "nur" }, "locked": { - "lastModified": 1783536074, - "narHash": "sha256-oftWmLYtGzP81WdaCOPN0KxPp5rWdsnYcXfjh2h24AM=", + "lastModified": 1783689486, + "narHash": "sha256-dXHzh9nJTR3sp3le+5GCZVMsrKP4eKr4yDgJUi7+sD4=", "ref": "refs/heads/master", - "rev": "32fdaddf7ca5018154bb97813c3492101ed9aa98", - "revCount": 88, + "rev": "c85600a3b073832d8edf83fd27b35811e36e4688", + "revCount": 89, "type": "git", "url": "ssh://gitea@code.m3ta.dev/m3tam3re/m3ta-home" }, @@ -731,11 +748,11 @@ "openspec": "openspec" }, "locked": { - "lastModified": 1783016129, - "narHash": "sha256-Uxie6uBNLUFJ8Tts7e6AYk5cOER5/EnCqKNL77Tzgc4=", + "lastModified": 1783269665, + "narHash": "sha256-bA4PooGV1x3vtmhs6NEprZbWErwaEJdgxoKfAWaH0do=", "ref": "refs/heads/master", - "rev": "fc5092d72dda7db13f81e5910a159de5108590d5", - "revCount": 350, + "rev": "f279ae9c89d59f3e0deb0fd6307800f4cb993815", + "revCount": 356, "type": "git", "url": "ssh://gitea@code.m3ta.dev/m3tam3re/nixpkgs" }, @@ -753,11 +770,11 @@ "openspec": "openspec_2" }, "locked": { - "lastModified": 1783269665, - "narHash": "sha256-bA4PooGV1x3vtmhs6NEprZbWErwaEJdgxoKfAWaH0do=", + "lastModified": 1783682547, + "narHash": "sha256-usY1tQB2srsjjFPoKA6BRwoRTzP48ChiM2qu3vju85w=", "ref": "refs/heads/master", - "rev": "f279ae9c89d59f3e0deb0fd6307800f4cb993815", - "revCount": 356, + "rev": "01bede8826fb0b35a0e432d788ce0a8fad2453f1", + "revCount": 358, "type": "git", "url": "https://code.m3ta.dev/m3tam3re/nixpkgs" }, @@ -857,11 +874,11 @@ "treefmt-nix": "treefmt-nix_2" }, "locked": { - "lastModified": 1781681115, - "narHash": "sha256-z8bD7qctIVWSv42xIvdPaw8IyC5hVTsoz4Nr6wAt46o=", + "lastModified": 1782890500, + "narHash": "sha256-UwamW3kYf/7vSsYq2VRb3VBq8A1hvOiLzXVnJeKVS38=", "owner": "nix-community", "repo": "nixos-anywhere", - "rev": "13d4a2c51a896417289acc0664ed41936c6897e2", + "rev": "4dfb813db065afb0aba1f61658ef77993d382db1", "type": "github" }, "original": { @@ -882,11 +899,11 @@ ] }, "locked": { - "lastModified": 1781173814, - "narHash": "sha256-lCoAwHfFXM0bhy7bY3QkN3XPW5HEyCwVOAgWyO3jl/w=", + "lastModified": 1782383811, + "narHash": "sha256-UFSWbDEltShkthOfvlAyxDq4L7dLHcsuJzHbgu79lHM=", "owner": "nix-community", "repo": "nixos-images", - "rev": "bf315e58d33dcc5a04df17304166f64bc550983f", + "rev": "45c188c452d274e003c9acc6b43fab911fa6cfa5", "type": "github" }, "original": { @@ -897,11 +914,11 @@ }, "nixos-stable": { "locked": { - "lastModified": 1781216227, - "narHash": "sha256-9mUW6gNwoN2SWc/l0fW4svPNOulXLl8ijqKyeSOGgJE=", + "lastModified": 1782691344, + "narHash": "sha256-i5nw9BYYsMDAaOC4J+JmTof6b2GhlyH076awYRNrTV8=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "a0374025a863d007d98e3297f6aa46cc3141c2f0", + "rev": "1f01958ffb5b3545c96d9ef2f4e24c5e5e1eb846", "type": "github" }, "original": { @@ -1007,11 +1024,11 @@ }, "nixpkgs-unstable": { "locked": { - "lastModified": 1781577229, - "narHash": "sha256-lrp67w8AulE9Ks53n27I45ADSzbOCn4H+CNW1Ck8B+8=", + "lastModified": 1783522502, + "narHash": "sha256-iffAls3iaNTyJC2faYcUXSI+Gp02cDjYl+MygxKl2GI=", "owner": "nixos", "repo": "nixpkgs", - "rev": "567a49d1913ce81ac6e9582e3553dd90a955875f", + "rev": "0bb7ec54c8483066ec9d7720e780a5caa71f8612", "type": "github" }, "original": { @@ -1071,11 +1088,11 @@ }, "nixpkgs_5": { "locked": { - "lastModified": 1782723713, - "narHash": "sha256-oPXCU/SSUokcGaJREHibG1CBX3+s/W7orDWQOZDsEeQ=", + "lastModified": 1783224372, + "narHash": "sha256-8i/87eeoqiGE4yOTjwSA3Eh/ziJRQEmd/unYU+K27sk=", "owner": "nixos", "repo": "nixpkgs", - "rev": "b5aa0fbd538984f6e3d201be0005b4463d8b09f8", + "rev": "d407951447dcd00442e97087bf374aad70c04cea", "type": "github" }, "original": { @@ -1103,11 +1120,11 @@ }, "nixpkgs_7": { "locked": { - "lastModified": 1781216227, - "narHash": "sha256-9mUW6gNwoN2SWc/l0fW4svPNOulXLl8ijqKyeSOGgJE=", + "lastModified": 1783389287, + "narHash": "sha256-0xIy4dVLqq47rA+mRy0hXDfjhQd4E5PoIns/RmB7nR4=", "owner": "nixos", "repo": "nixpkgs", - "rev": "a0374025a863d007d98e3297f6aa46cc3141c2f0", + "rev": "0ad6f47ea4fe188f4bc8f0380f93ae8523337c6c", "type": "github" }, "original": { @@ -1177,11 +1194,11 @@ "nixpkgs": "nixpkgs_5" }, "locked": { - "lastModified": 1782796354, - "narHash": "sha256-zHZEX+twYRO4n369Nrk21DBb519JZCGJLHBo/trhHO0=", + "lastModified": 1783593872, + "narHash": "sha256-KvFRxsi81gBoK7JOTHD2LoR1y2sWobHkrHGPhM3v/NY=", "owner": "nix-community", "repo": "NUR", - "rev": "df4f4323fe161a8bab3aafa2d067ff9d7e223a67", + "rev": "73078fa6f0d34686ad26808f90cc74e81039ba9f", "type": "github" }, "original": { @@ -1198,11 +1215,11 @@ ] }, "locked": { - "lastModified": 1782104183, - "narHash": "sha256-BZD9AAFrbIJ+sIfUxI4wKs9B+J8e7gfOmewdcAgi/HY=", + "lastModified": 1783688207, + "narHash": "sha256-003aVBGJLPSn/kjlkDAMUaHTz0hBBN7QK1qjSJpCsIQ=", "owner": "nix-community", "repo": "NUR", - "rev": "adda660080fa2677e1b3f368024956b15b066439", + "rev": "bea54b7713e4a22b8c1c9d4a12c706d31d235462", "type": "github" }, "original": { @@ -1369,29 +1386,6 @@ "type": "github" } }, - "quickshell": { - "inputs": { - "nixpkgs": [ - "m3ta-home", - "dms", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1776854048, - "narHash": "sha256-lLbV66V3RMNp1l8/UelmR4YzoJ5ONtgvEtiUMJATH/o=", - "ref": "refs/heads/master", - "rev": "783c953987dc56ff0601abe6845ed96f1d00495a", - "revCount": 806, - "type": "git", - "url": "https://git.outfoxxed.me/quickshell/quickshell" - }, - "original": { - "rev": "783c953987dc56ff0601abe6845ed96f1d00495a", - "type": "git", - "url": "https://git.outfoxxed.me/quickshell/quickshell" - } - }, "root": { "inputs": { "agenix": "agenix", diff --git a/hosts/AZ-CLD-1/services/containers/litellm.nix b/hosts/AZ-CLD-1/services/containers/litellm.nix index 69c0796..37ca763 100644 --- a/hosts/AZ-CLD-1/services/containers/litellm.nix +++ b/hosts/AZ-CLD-1/services/containers/litellm.nix @@ -1,20 +1,97 @@ -{config, ...}: let +{ + config, + pkgs, + ... +}: let serviceName = "litellm"; servicePort = config.m3ta.ports.get serviceName; + prmNetbirdIP = "100.91.49.26"; + cldNetbirdIP = "100.91.203.184"; + + python = pkgs.python3.withPackages (ps: [ps.pyyaml]); + litellmConfigGenerator = pkgs.writeText "litellm-config-generator.py" '' + import sys + from pathlib import Path + + import yaml + + base_path = Path(sys.argv[1]) + target_path = Path(sys.argv[2]) + + with base_path.open("r", encoding="utf-8") as fh: + config = yaml.safe_load(fh) or {} + + settings = config.get("litellm_settings") + if not isinstance(settings, dict): + settings = {} + config["litellm_settings"] = settings + + def ensure_list(value): + if value is None: + return [] + if isinstance(value, list): + return value + return [value] + + callbacks = ensure_list(settings.get("callbacks")) + if "prometheus" not in callbacks: + callbacks.append("prometheus") + settings["callbacks"] = callbacks + + service_callbacks = ensure_list(settings.get("service_callback")) + if "prometheus_system" not in service_callbacks: + service_callbacks.append("prometheus_system") + settings["service_callback"] = service_callbacks + + # LiteLLM >= 1.85 protects /metrics by default. Keep auth enabled; + # Prometheus scrapes with a bearer token from an agenix secret on AZ-PRM-1. + settings["require_auth_for_metrics_endpoint"] = True + + target_path.parent.mkdir(parents=True, exist_ok=True) + with target_path.open("w", encoding="utf-8") as fh: + yaml.safe_dump(config, fh, sort_keys=False) + ''; in { + systemd.services."podman-${serviceName}".preStart = '' + set -euo pipefail + + base_config="/var/lib/${serviceName}/config.yaml" + target_config="/run/${serviceName}/config.yaml" + multiproc_dir="/var/lib/${serviceName}/prometheus-multiproc" + + if [ ! -f "$base_config" ]; then + echo "Missing LiteLLM base config: $base_config" >&2 + exit 1 + fi + + mkdir -p "$(dirname "$target_config")" "$multiproc_dir" + rm -f "$multiproc_dir"/* + chmod 0777 "$multiproc_dir" + + ${python}/bin/python ${litellmConfigGenerator} "$base_config" "$target_config" + chmod 0644 "$target_config" + ''; + virtualisation.oci-containers.containers.${serviceName} = { #image = "ghcr.io/berriai/litellm:v1.78.5-stable"; - image = "docker.litellm.ai/berriai/litellm:1.89.0"; - ports = ["127.0.0.1:${toString servicePort}:4000"]; + image = "docker.litellm.ai/berriai/litellm:1.92.0"; + ports = [ + "127.0.0.1:${toString servicePort}:4000" + "${cldNetbirdIP}:${toString servicePort}:4000" + ]; + cmd = ["--config" "/app/config.yaml" "--port" "4000"]; environmentFiles = [config.age.secrets.litellm-env.path]; environment = { ANONYMIZED_TELEMETRY = "False"; DO_NOT_TRACK = "True"; SCARF_NO_ANALYTICS = "True"; STORE_MODEL_IN_DB = "True"; - LITELLM_LOG = "json"; + PROMETHEUS_MULTIPROC_DIR = "/prometheus_multiproc"; }; - volumes = ["/var/lib/litellm/config.yaml:/app/config.yaml"]; + volumes = [ + "/run/${serviceName}/config.yaml:/app/config.yaml:ro" + "/var/lib/${serviceName}/prometheus-multiproc:/prometheus_multiproc" + ]; extraOptions = ["--add-host=postgres:10.89.0.1" "--ip=10.89.0.30" "--network=web"]; }; @@ -26,6 +103,23 @@ in { } ]; + middlewares."${serviceName}-metrics-local-only".ipAllowList.sourceRange = [ + "127.0.0.1/32" + "::1/128" + "${prmNetbirdIP}/32" + ]; + + routers."${serviceName}-metrics" = { + rule = "Host(`llm.az-gruppe.com`) && PathPrefix(`/metrics`)"; + tls = { + certResolver = "ionos"; + }; + service = serviceName; + entrypoints = "websecure"; + middlewares = ["${serviceName}-metrics-local-only"]; + priority = 200; + }; + routers.${serviceName} = { rule = "Host(`llm.az-gruppe.com`)"; tls = { @@ -33,6 +127,11 @@ in { }; service = serviceName; entrypoints = "websecure"; + priority = 1; }; }; + + networking.firewall.extraCommands = '' + iptables -A INPUT -p tcp -s ${prmNetbirdIP} --dport ${toString servicePort} -j ACCEPT + ''; } diff --git a/hosts/AZ-PRM-1/secrets.nix b/hosts/AZ-PRM-1/secrets.nix index ee3b2a2..736698b 100644 --- a/hosts/AZ-PRM-1/secrets.nix +++ b/hosts/AZ-PRM-1/secrets.nix @@ -1,5 +1,6 @@ {lib, ...}: let ntfyGrafanaWebhookSecret = ../../secrets/ntfy-grafana-webhook.age; + litellmPrometheusBearerSecret = ../../secrets/litellm-prometheus-bearer.age; in { age = { secrets = @@ -64,6 +65,14 @@ in { file = ntfyGrafanaWebhookSecret; mode = "0400"; }; + } + // lib.optionalAttrs (builtins.pathExists litellmPrometheusBearerSecret) { + litellm-prometheus-bearer = { + file = litellmPrometheusBearerSecret; + owner = "prometheus"; + group = "prometheus"; + mode = "0400"; + }; }; }; } diff --git a/hosts/AZ-PRM-1/services/monitoring/alerting.nix b/hosts/AZ-PRM-1/services/monitoring/alerting.nix index 4c6459c..a49d6cb 100644 --- a/hosts/AZ-PRM-1/services/monitoring/alerting.nix +++ b/hosts/AZ-PRM-1/services/monitoring/alerting.nix @@ -195,6 +195,47 @@ in { }; annotations.description = "A Kestra flow execution ended with FAILED, WARNING, or KILLED."; }) + (mkAlertRule { + uid = "az_litellm_down"; + title = "LiteLLM metrics unreachable"; + expr = ''up{job="litellm"}''; + for = "2m"; + evaluatorType = "lt"; + evaluatorParams = [1 0]; + labels = { + service = "litellm"; + severity = "critical"; + }; + annotations.description = "Prometheus cannot scrape LiteLLM /metrics on AZ-CLD-1 over Netbird."; + }) + (mkAlertRule { + uid = "az_litellm_proxy_errors"; + title = "LiteLLM proxy errors"; + expr = ''sum(increase(litellm_proxy_failed_requests_metric_total{job="litellm"}[5m]))''; + for = "1m"; + noDataState = "OK"; + evaluatorType = "gt"; + evaluatorParams = [0 0]; + labels = { + service = "litellm"; + severity = "warning"; + }; + annotations.description = "LiteLLM reported one or more failed proxy responses in the last 5 minutes."; + }) + (mkAlertRule { + uid = "az_litellm_latency_high"; + title = "LiteLLM latency high"; + expr = ''histogram_quantile(0.95, sum(rate(litellm_request_total_latency_metric_bucket{job="litellm"}[5m])) by (le))''; + for = "10m"; + noDataState = "OK"; + evaluatorType = "gt"; + evaluatorParams = [30 0]; + labels = { + service = "litellm"; + severity = "warning"; + }; + annotations.description = "LiteLLM p95 total request latency has been above 30 seconds for 10 minutes."; + }) (mkAlertRule { uid = "az_http_probe_failed"; title = "HTTP probe failed"; diff --git a/hosts/AZ-PRM-1/services/monitoring/cld-scrape.nix b/hosts/AZ-PRM-1/services/monitoring/cld-scrape.nix index 1af7da0..4da65db 100644 --- a/hosts/AZ-PRM-1/services/monitoring/cld-scrape.nix +++ b/hosts/AZ-PRM-1/services/monitoring/cld-scrape.nix @@ -1,7 +1,23 @@ -{config, ...}: let +{ + config, + lib, + ... +}: let nodeExporterPort = config.m3ta.ports.get "node-exporter"; + litellmPort = config.m3ta.ports.get "litellm"; cldNetbirdIP = "100.91.203.184"; + litellmPrometheusBearerSecretAvailable = builtins.hasAttr "litellm-prometheus-bearer" config.age.secrets; + litellmPrometheusAuthorization = lib.optionalAttrs litellmPrometheusBearerSecretAvailable { + authorization = { + type = "Bearer"; + credentials_file = config.age.secrets."litellm-prometheus-bearer".path; + }; + }; in { + # Prometheus' config checker validates credentials_file paths at build time, + # but agenix secrets only exist at runtime under /run/agenix. + services.prometheus.checkConfig = lib.mkIf litellmPrometheusBearerSecretAvailable false; + services.prometheus.scrapeConfigs = [ { job_name = "node-cld"; @@ -14,11 +30,26 @@ in { } ]; } + ({ + job_name = "litellm"; + metrics_path = "/metrics/"; + scrape_interval = "15s"; + static_configs = [ + { + targets = ["${cldNetbirdIP}:${toString litellmPort}"]; + labels = { + instance = "AZ-CLD-1"; + service = "litellm"; + }; + } + ]; + } + // litellmPrometheusAuthorization) ]; - # Allow CLD to reach PRM prometheus scrapes (prometheus initiates connection TO CLD exporter) + # Allow CLD to reach PRM prometheus scrapes (prometheus initiates connection TO CLD exporters) networking.firewall.extraCommands = '' - # No PRM-side firewall change needed: PRM scrapes outbound to CLD:9100. + # No PRM-side firewall change needed: PRM scrapes outbound to CLD:9100/4000. # CLD-side must allow inbound from 100.91.49.26 (PRM netbird IP) — see CLD config. ''; } diff --git a/hosts/AZ-PRM-1/services/monitoring/dashboards/az-litellm-observability.json b/hosts/AZ-PRM-1/services/monitoring/dashboards/az-litellm-observability.json new file mode 100644 index 0000000..e4cf14d --- /dev/null +++ b/hosts/AZ-PRM-1/services/monitoring/dashboards/az-litellm-observability.json @@ -0,0 +1,568 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "down" + }, + "1": { + "color": "green", + "index": 0, + "text": "up" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "up{job=\"litellm\", instance=~\"$instance\"}", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Scrape status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "decimals": 3, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(litellm_proxy_total_requests_metric_total{job=\"litellm\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "requests/s", + "range": true, + "refId": "A" + } + ], + "title": "Proxy request rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "decimals": 3, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(litellm_proxy_failed_requests_metric_total{job=\"litellm\", instance=~\"$instance\"}[$__rate_interval])) or vector(0)", + "legendFormat": "errors/s", + "range": true, + "refId": "A" + } + ], + "title": "Proxy error rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "decimals": 2, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(litellm_request_total_latency_metric_bucket{job=\"litellm\", instance=~\"$instance\"}[$__rate_interval])) by (le))", + "legendFormat": "p95", + "range": true, + "refId": "A" + } + ], + "title": "p95 total latency", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 4 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (requested_model) (rate(litellm_proxy_total_requests_metric_total{job=\"litellm\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "{{requested_model}}", + "range": true, + "refId": "A" + } + ], + "title": "Requests by requested model", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 4 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (model) (rate(litellm_total_tokens_metric_total{job=\"litellm\", instance=~\"$instance\"}[$__rate_interval]))", + "legendFormat": "{{model}}", + "range": true, + "refId": "A" + } + ], + "title": "Token rate by model", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "currencyUSD" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 7, + "options": { + "displayMode": "gradient", + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "showUnfilled": true, + "valueMode": "color" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "topk(10, sum by (model) (increase(litellm_spend_metric_total{job=\"litellm\", instance=~\"$instance\"}[$__range])))", + "legendFormat": "{{model}}", + "range": true, + "refId": "A" + } + ], + "title": "Spend by model in selected range", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "linear", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": false + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le, model) (rate(litellm_llm_api_latency_metric_bucket{job=\"litellm\", instance=~\"$instance\"}[$__rate_interval])))", + "legendFormat": "{{model}} p95", + "range": true, + "refId": "A" + } + ], + "title": "LLM API latency by model", + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 9, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "code", + "expr": "{host=\"AZ-CLD-1\", unit=~\"podman-litellm.service|docker-litellm.service|litellm.service\"}", + "queryType": "range", + "refId": "A" + } + ], + "title": "LiteLLM logs from Loki", + "type": "logs" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "style": "dark", + "tags": [ + "az", + "litellm", + "prometheus", + "loki", + "provisioned" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "definition": "label_values(up{job=\"litellm\"}, instance)", + "hide": 0, + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": { + "query": "label_values(up{job=\"litellm\"}, instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "AZ LiteLLM Observability", + "uid": "az-litellm-observability", + "version": 1, + "weekStart": "" +} diff --git a/secrets.nix b/secrets.nix index 56cdd34..04b9bf5 100644 --- a/secrets.nix +++ b/secrets.nix @@ -26,6 +26,7 @@ in { "secrets/librechat-env-prod.age".publicKeys = systems ++ users; "secrets/librechat-env-dev.age".publicKeys = systems ++ users; "secrets/litellm-env.age".publicKeys = systems ++ users; + "secrets/litellm-prometheus-bearer.age".publicKeys = [AZ-PRM-1] ++ users; "secrets/metabase-env.age".publicKeys = systems ++ users; "secrets/n8n-env.age".publicKeys = systems ++ users; "secrets/n8n-env-prm.age".publicKeys = systems ++ users; diff --git a/secrets/litellm-prometheus-bearer.age b/secrets/litellm-prometheus-bearer.age new file mode 100644 index 0000000..acaa2ef --- /dev/null +++ b/secrets/litellm-prometheus-bearer.age @@ -0,0 +1,11 @@ +-----BEGIN AGE ENCRYPTED FILE----- +YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHNzaC1lZDI1NTE5IGxHcVZaZyBJZkVq +aWU5RTljeXR1ZTdWcmQrL21aMDB1aTQ0Q1hrVWJWVzFSUmorLzBvCmhRWUxjaWsx +aHczT0h0Z2NhbGhuMjZqRHdMdGRGeU9ueDd1eHJwdk90Y0EKLT4gc3NoLWVkMjU1 +MTkgQ1NNeWhnIEpMbUlxZ2xLR3NQOXhlZzFlWGdrNkd4N3pKWWVJN1I1bjZ5clJZ +WTR0amsKZW5peVlnZzZvYUxtdU5COGNqVFRoRVQrQ2ludU9XTjc2V2tQak0zL0xP +dwotPiBiWTt8OyUlZi1ncmVhc2UgRjEyIGUqUykgNV1WVykgdyFTe0wiYwp5aGZw +bjRPeitGTlhzNVFpbUNuSEw3T1V2YVJlT2IxUkJRCi0tLSA3amRwTWtNOVh1cm1R +WElER3NYZVZGeXJNdFQwTno5cnFXQWVraUNDOFVVCjQsmAl0ywy880NBAG+Zv2Wz +jAOi0Sm6BNRdDkkfUGq1i63qy4ZnFxVGlAwmsmIxseicJ2sqteUmlqN2eqLkC3jO +-----END AGE ENCRYPTED FILE-----