fix: litellm -> grafana

flake.lock

@@ -66,11 +66,11 @@
     "agents": {
       "flake": false,
       "locked": {
-        "lastModified": 1780133320,
+        "lastModified": 1783016143,
-        "narHash": "sha256-8AiN9tV9PBb5xblJiPlhumBbKj61qLjzqXXFtkj3vvY=",
+        "narHash": "sha256-3qejDAo4g2XfzTyWCaX8RVKH17JXnw/fujsbXQA+Mzc=",
         "ref": "refs/heads/master",
-        "rev": "920c00313ae242bd93275c30131b9ab1e52ee2fb",
+        "rev": "83284d752d2ac325d4bf3ab7b38acad1c2fffe3a",
-        "revCount": 88,
+        "revCount": 98,
         "type": "git",
         "url": "https://code.m3ta.dev/m3tam3re/AGENTS"
       },
@@ -84,11 +84,11 @@
         "nixpkgs": "nixpkgs_4"
       },
       "locked": {
-        "lastModified": 1782789853,
+        "lastModified": 1783016143,
-        "narHash": "sha256-LEmctqYRQRq0wB1MoS+IVr/0/uu/0nKahqNeJBGTjJ8=",
+        "narHash": "sha256-3qejDAo4g2XfzTyWCaX8RVKH17JXnw/fujsbXQA+Mzc=",
         "ref": "refs/heads/master",
-        "rev": "cd36a91440b971582fcf2d4eedf9a46fb755f6e1",
+        "rev": "83284d752d2ac325d4bf3ab7b38acad1c2fffe3a",
-        "revCount": 92,
+        "revCount": 98,
         "type": "git",
         "url": "ssh://gitea@code.m3ta.dev/m3tam3re/AGENTS"
       },
@@ -409,18 +409,18 @@
     },
     "dms": {
       "inputs": {
+        "flake-compat": "flake-compat",
         "nixpkgs": [
           "m3ta-home",
           "nixpkgs"
-        ],
+        ]
-        "quickshell": "quickshell"
       },
       "locked": {
-        "lastModified": 1777431599,
+        "lastModified": 1783529634,
-        "narHash": "sha256-g6r/Gx8PTDzO3jCNzzySA+Ff1lmLF9nDlMCNyyoQjoE=",
+        "narHash": "sha256-zdHsPGPE5MVi/y+uIt548XScTfZjQzdF21dME7ISEJM=",
         "owner": "AvengeMedia",
         "repo": "DankMaterialShell",
-        "rev": "eb5afcdc40ea5446c27e18552ff4a19f9daf9484",
+        "rev": "bdfd565b72cbb416441a9ab828efcac4fd516c70",
         "type": "github"
       },
       "original": {
@@ -438,11 +438,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1782780764,
+        "lastModified": 1783591397,
-        "narHash": "sha256-Q2wPFsuDo1y6neZ3ttxxOOItupg/4+mPkyAZbC0+wfw=",
+        "narHash": "sha256-e4DDmVAas6IPwwY+gfSDKh3PMzW/FXLCbjyWXOnnnbM=",
         "owner": "AvengeMedia",
         "repo": "dms-plugin-registry",
-        "rev": "9fa716427ab5905845b2a10a01d67bee0b1e4c4b",
+        "rev": "8b61e8fca730986e673737529fa56802dc9326bb",
         "type": "github"
       },
       "original": {
@@ -451,6 +451,22 @@
         "type": "github"
       }
     },
+    "flake-compat": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1767039857,
+        "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=",
+        "owner": "NixOS",
+        "repo": "flake-compat",
+        "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
     "flake-parts": {
       "inputs": {
         "nixpkgs-lib": [
@@ -568,15 +584,16 @@
         "uv2nix": "uv2nix_2"
       },
       "locked": {
-        "lastModified": 1782794050,
+        "lastModified": 1782936460,
-        "narHash": "sha256-+qKEwIhTkml3DNSmIvGFJyssW9ZXTp78KGclFdcDvX8=",
+        "narHash": "sha256-Wt72AQtA6Eizi7Ubj23JBhwZ7GKYcjY4mcV6upqHOaU=",
         "owner": "NousResearch",
         "repo": "hermes-agent",
-        "rev": "972b1620906a1b80772c2f67492ad50b3c83f048",
+        "rev": "7c1a029553d87c43ecff8a3821336bc95872213b",
         "type": "github"
       },
       "original": {
         "owner": "NousResearch",
+        "ref": "v2026.7.1",
         "repo": "hermes-agent",
         "type": "github"
       }
@@ -609,11 +626,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1781981105,
+        "lastModified": 1783221248,
-        "narHash": "sha256-/1nNBbA7PrSQpTc9Qazkhl4kIPg+TNl0CjxS3UQJKlw=",
+        "narHash": "sha256-ESQnuNHEDChsB4IxoLRhscVahqkDWkTb+qdIz8euYt4=",
         "owner": "nix-community",
         "repo": "home-manager",
-        "rev": "7bfff44b465909f69a442701293bc0badcf476dc",
+        "rev": "af2beae5f0fae0a4310cc0e6aef2572f56090353",
         "type": "github"
       },
       "original": {
@@ -653,11 +670,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1782749631,
+        "lastModified": 1783550008,
-        "narHash": "sha256-slFTUgDy0KTPA4LBAmC/9SngDq8GCPdX+ZR0yQHHN1E=",
+        "narHash": "sha256-qbbZUk9IG9dLNwCPwdPBs6I5clxwugRpP+1YU+GPK20=",
         "owner": "nix-community",
         "repo": "home-manager",
-        "rev": "5d72a29fc36ac21adae6ae35568fe5ee6700850f",
+        "rev": "f4d01c1d87c7c2ec909549165d5a8338f1bd3315",
         "type": "github"
       },
       "original": {
@@ -676,11 +693,11 @@
         "treefmt-nix": "treefmt-nix"
       },
       "locked": {
-        "lastModified": 1783567605,
+        "lastModified": 1783669679,
-        "narHash": "sha256-LyF84yqWppXAAEOAkL325Lt+DrVmkbbeXGzDgX9zMzI=",
+        "narHash": "sha256-WDGDMsPXxceC0k1ktqcobmwE4GCTrzzULUKlp1CzSKQ=",
         "owner": "numtide",
         "repo": "llm-agents.nix",
-        "rev": "a4c847460f0e773d02d0655ce28dc6b532dd65d6",
+        "rev": "6e9f6664e72433966557ab83a7c3e8c0bbf64a44",
         "type": "github"
       },
       "original": {
@@ -706,11 +723,11 @@
         "nur": "nur"
       },
       "locked": {
-        "lastModified": 1783536074,
+        "lastModified": 1783689486,
-        "narHash": "sha256-oftWmLYtGzP81WdaCOPN0KxPp5rWdsnYcXfjh2h24AM=",
+        "narHash": "sha256-dXHzh9nJTR3sp3le+5GCZVMsrKP4eKr4yDgJUi7+sD4=",
         "ref": "refs/heads/master",
-        "rev": "32fdaddf7ca5018154bb97813c3492101ed9aa98",
+        "rev": "c85600a3b073832d8edf83fd27b35811e36e4688",
-        "revCount": 88,
+        "revCount": 89,
         "type": "git",
         "url": "ssh://gitea@code.m3ta.dev/m3tam3re/m3ta-home"
       },
@@ -731,11 +748,11 @@
         "openspec": "openspec"
       },
       "locked": {
-        "lastModified": 1783016129,
+        "lastModified": 1783269665,
-        "narHash": "sha256-Uxie6uBNLUFJ8Tts7e6AYk5cOER5/EnCqKNL77Tzgc4=",
+        "narHash": "sha256-bA4PooGV1x3vtmhs6NEprZbWErwaEJdgxoKfAWaH0do=",
         "ref": "refs/heads/master",
-        "rev": "fc5092d72dda7db13f81e5910a159de5108590d5",
+        "rev": "f279ae9c89d59f3e0deb0fd6307800f4cb993815",
-        "revCount": 350,
+        "revCount": 356,
         "type": "git",
         "url": "ssh://gitea@code.m3ta.dev/m3tam3re/nixpkgs"
       },
@@ -753,11 +770,11 @@
         "openspec": "openspec_2"
       },
       "locked": {
-        "lastModified": 1783269665,
+        "lastModified": 1783682547,
-        "narHash": "sha256-bA4PooGV1x3vtmhs6NEprZbWErwaEJdgxoKfAWaH0do=",
+        "narHash": "sha256-usY1tQB2srsjjFPoKA6BRwoRTzP48ChiM2qu3vju85w=",
         "ref": "refs/heads/master",
-        "rev": "f279ae9c89d59f3e0deb0fd6307800f4cb993815",
+        "rev": "01bede8826fb0b35a0e432d788ce0a8fad2453f1",
-        "revCount": 356,
+        "revCount": 358,
         "type": "git",
         "url": "https://code.m3ta.dev/m3tam3re/nixpkgs"
       },
@@ -857,11 +874,11 @@
         "treefmt-nix": "treefmt-nix_2"
       },
       "locked": {
-        "lastModified": 1781681115,
+        "lastModified": 1782890500,
-        "narHash": "sha256-z8bD7qctIVWSv42xIvdPaw8IyC5hVTsoz4Nr6wAt46o=",
+        "narHash": "sha256-UwamW3kYf/7vSsYq2VRb3VBq8A1hvOiLzXVnJeKVS38=",
         "owner": "nix-community",
         "repo": "nixos-anywhere",
-        "rev": "13d4a2c51a896417289acc0664ed41936c6897e2",
+        "rev": "4dfb813db065afb0aba1f61658ef77993d382db1",
         "type": "github"
       },
       "original": {
@@ -882,11 +899,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1781173814,
+        "lastModified": 1782383811,
-        "narHash": "sha256-lCoAwHfFXM0bhy7bY3QkN3XPW5HEyCwVOAgWyO3jl/w=",
+        "narHash": "sha256-UFSWbDEltShkthOfvlAyxDq4L7dLHcsuJzHbgu79lHM=",
         "owner": "nix-community",
         "repo": "nixos-images",
-        "rev": "bf315e58d33dcc5a04df17304166f64bc550983f",
+        "rev": "45c188c452d274e003c9acc6b43fab911fa6cfa5",
         "type": "github"
       },
       "original": {
@@ -897,11 +914,11 @@
     },
     "nixos-stable": {
       "locked": {
-        "lastModified": 1781216227,
+        "lastModified": 1782691344,
-        "narHash": "sha256-9mUW6gNwoN2SWc/l0fW4svPNOulXLl8ijqKyeSOGgJE=",
+        "narHash": "sha256-i5nw9BYYsMDAaOC4J+JmTof6b2GhlyH076awYRNrTV8=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "a0374025a863d007d98e3297f6aa46cc3141c2f0",
+        "rev": "1f01958ffb5b3545c96d9ef2f4e24c5e5e1eb846",
         "type": "github"
       },
       "original": {
@@ -1007,11 +1024,11 @@
     },
     "nixpkgs-unstable": {
       "locked": {
-        "lastModified": 1781577229,
+        "lastModified": 1783522502,
-        "narHash": "sha256-lrp67w8AulE9Ks53n27I45ADSzbOCn4H+CNW1Ck8B+8=",
+        "narHash": "sha256-iffAls3iaNTyJC2faYcUXSI+Gp02cDjYl+MygxKl2GI=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "567a49d1913ce81ac6e9582e3553dd90a955875f",
+        "rev": "0bb7ec54c8483066ec9d7720e780a5caa71f8612",
         "type": "github"
       },
       "original": {
@@ -1071,11 +1088,11 @@
     },
     "nixpkgs_5": {
       "locked": {
-        "lastModified": 1782723713,
+        "lastModified": 1783224372,
-        "narHash": "sha256-oPXCU/SSUokcGaJREHibG1CBX3+s/W7orDWQOZDsEeQ=",
+        "narHash": "sha256-8i/87eeoqiGE4yOTjwSA3Eh/ziJRQEmd/unYU+K27sk=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "b5aa0fbd538984f6e3d201be0005b4463d8b09f8",
+        "rev": "d407951447dcd00442e97087bf374aad70c04cea",
         "type": "github"
       },
       "original": {
@@ -1103,11 +1120,11 @@
     },
     "nixpkgs_7": {
       "locked": {
-        "lastModified": 1781216227,
+        "lastModified": 1783389287,
-        "narHash": "sha256-9mUW6gNwoN2SWc/l0fW4svPNOulXLl8ijqKyeSOGgJE=",
+        "narHash": "sha256-0xIy4dVLqq47rA+mRy0hXDfjhQd4E5PoIns/RmB7nR4=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "a0374025a863d007d98e3297f6aa46cc3141c2f0",
+        "rev": "0ad6f47ea4fe188f4bc8f0380f93ae8523337c6c",
         "type": "github"
       },
       "original": {
@@ -1177,11 +1194,11 @@
         "nixpkgs": "nixpkgs_5"
       },
       "locked": {
-        "lastModified": 1782796354,
+        "lastModified": 1783593872,
-        "narHash": "sha256-zHZEX+twYRO4n369Nrk21DBb519JZCGJLHBo/trhHO0=",
+        "narHash": "sha256-KvFRxsi81gBoK7JOTHD2LoR1y2sWobHkrHGPhM3v/NY=",
         "owner": "nix-community",
         "repo": "NUR",
-        "rev": "df4f4323fe161a8bab3aafa2d067ff9d7e223a67",
+        "rev": "73078fa6f0d34686ad26808f90cc74e81039ba9f",
         "type": "github"
       },
       "original": {
@@ -1198,11 +1215,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1782104183,
+        "lastModified": 1783688207,
-        "narHash": "sha256-BZD9AAFrbIJ+sIfUxI4wKs9B+J8e7gfOmewdcAgi/HY=",
+        "narHash": "sha256-003aVBGJLPSn/kjlkDAMUaHTz0hBBN7QK1qjSJpCsIQ=",
         "owner": "nix-community",
         "repo": "NUR",
-        "rev": "adda660080fa2677e1b3f368024956b15b066439",
+        "rev": "bea54b7713e4a22b8c1c9d4a12c706d31d235462",
         "type": "github"
       },
       "original": {
@@ -1369,29 +1386,6 @@
         "type": "github"
       }
     },
-    "quickshell": {
-      "inputs": {
-        "nixpkgs": [
-          "m3ta-home",
-          "dms",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1776854048,
-        "narHash": "sha256-lLbV66V3RMNp1l8/UelmR4YzoJ5ONtgvEtiUMJATH/o=",
-        "ref": "refs/heads/master",
-        "rev": "783c953987dc56ff0601abe6845ed96f1d00495a",
-        "revCount": 806,
-        "type": "git",
-        "url": "https://git.outfoxxed.me/quickshell/quickshell"
-      },
-      "original": {
-        "rev": "783c953987dc56ff0601abe6845ed96f1d00495a",
-        "type": "git",
-        "url": "https://git.outfoxxed.me/quickshell/quickshell"
-      }
-    },
     "root": {
       "inputs": {
         "agenix": "agenix",

hosts/AZ-CLD-1/services/containers/litellm.nix

@@ -1,20 +1,97 @@
-{config, ...}: let
+{
+  config,
+  pkgs,
+  ...
+}: let
   serviceName = "litellm";
   servicePort = config.m3ta.ports.get serviceName;
+  prmNetbirdIP = "100.91.49.26";
+  cldNetbirdIP = "100.91.203.184";
+
+  python = pkgs.python3.withPackages (ps: [ps.pyyaml]);
+  litellmConfigGenerator = pkgs.writeText "litellm-config-generator.py" ''
+    import sys
+    from pathlib import Path
+
+    import yaml
+
+    base_path = Path(sys.argv[1])
+    target_path = Path(sys.argv[2])
+
+    with base_path.open("r", encoding="utf-8") as fh:
+        config = yaml.safe_load(fh) or {}
+
+    settings = config.get("litellm_settings")
+    if not isinstance(settings, dict):
+        settings = {}
+        config["litellm_settings"] = settings
+
+    def ensure_list(value):
+        if value is None:
+            return []
+        if isinstance(value, list):
+            return value
+        return [value]
+
+    callbacks = ensure_list(settings.get("callbacks"))
+    if "prometheus" not in callbacks:
+        callbacks.append("prometheus")
+    settings["callbacks"] = callbacks
+
+    service_callbacks = ensure_list(settings.get("service_callback"))
+    if "prometheus_system" not in service_callbacks:
+        service_callbacks.append("prometheus_system")
+    settings["service_callback"] = service_callbacks
+
+    # LiteLLM >= 1.85 protects /metrics by default. Keep auth enabled;
+    # Prometheus scrapes with a bearer token from an agenix secret on AZ-PRM-1.
+    settings["require_auth_for_metrics_endpoint"] = True
+
+    target_path.parent.mkdir(parents=True, exist_ok=True)
+    with target_path.open("w", encoding="utf-8") as fh:
+        yaml.safe_dump(config, fh, sort_keys=False)
+  '';
 in {
+  systemd.services."podman-${serviceName}".preStart = ''
+    set -euo pipefail
+
+    base_config="/var/lib/${serviceName}/config.yaml"
+    target_config="/run/${serviceName}/config.yaml"
+    multiproc_dir="/var/lib/${serviceName}/prometheus-multiproc"
+
+    if [ ! -f "$base_config" ]; then
+      echo "Missing LiteLLM base config: $base_config" >&2
+      exit 1
+    fi
+
+    mkdir -p "$(dirname "$target_config")" "$multiproc_dir"
+    rm -f "$multiproc_dir"/*
+    chmod 0777 "$multiproc_dir"
+
+    ${python}/bin/python ${litellmConfigGenerator} "$base_config" "$target_config"
+    chmod 0644 "$target_config"
+  '';
+
   virtualisation.oci-containers.containers.${serviceName} = {
     #image = "ghcr.io/berriai/litellm:v1.78.5-stable";
-    image = "docker.litellm.ai/berriai/litellm:1.89.0";
+    image = "docker.litellm.ai/berriai/litellm:1.92.0";
-    ports = ["127.0.0.1:${toString servicePort}:4000"];
+    ports = [
+      "127.0.0.1:${toString servicePort}:4000"
+      "${cldNetbirdIP}:${toString servicePort}:4000"
+    ];
+    cmd = ["--config" "/app/config.yaml" "--port" "4000"];
     environmentFiles = [config.age.secrets.litellm-env.path];
     environment = {
       ANONYMIZED_TELEMETRY = "False";
       DO_NOT_TRACK = "True";
       SCARF_NO_ANALYTICS = "True";
       STORE_MODEL_IN_DB = "True";
-      LITELLM_LOG = "json";
+      PROMETHEUS_MULTIPROC_DIR = "/prometheus_multiproc";
     };
-    volumes = ["/var/lib/litellm/config.yaml:/app/config.yaml"];
+    volumes = [
+      "/run/${serviceName}/config.yaml:/app/config.yaml:ro"
+      "/var/lib/${serviceName}/prometheus-multiproc:/prometheus_multiproc"
+    ];
     extraOptions = ["--add-host=postgres:10.89.0.1" "--ip=10.89.0.30" "--network=web"];
   };
 
@@ -26,6 +103,23 @@ in {
       }
     ];
 
+    middlewares."${serviceName}-metrics-local-only".ipAllowList.sourceRange = [
+      "127.0.0.1/32"
+      "::1/128"
+      "${prmNetbirdIP}/32"
+    ];
+
+    routers."${serviceName}-metrics" = {
+      rule = "Host(`llm.az-gruppe.com`) && PathPrefix(`/metrics`)";
+      tls = {
+        certResolver = "ionos";
+      };
+      service = serviceName;
+      entrypoints = "websecure";
+      middlewares = ["${serviceName}-metrics-local-only"];
+      priority = 200;
+    };
+
     routers.${serviceName} = {
       rule = "Host(`llm.az-gruppe.com`)";
       tls = {
@@ -33,6 +127,11 @@ in {
       };
       service = serviceName;
       entrypoints = "websecure";
+      priority = 1;
     };
   };
+
+  networking.firewall.extraCommands = ''
+    iptables -A INPUT -p tcp -s ${prmNetbirdIP} --dport ${toString servicePort} -j ACCEPT
+  '';
 }

hosts/AZ-PRM-1/secrets.nix

@@ -1,5 +1,6 @@
 {lib, ...}: let
   ntfyGrafanaWebhookSecret = ../../secrets/ntfy-grafana-webhook.age;
+  litellmPrometheusBearerSecret = ../../secrets/litellm-prometheus-bearer.age;
 in {
   age = {
     secrets =
@@ -64,6 +65,14 @@ in {
           file = ntfyGrafanaWebhookSecret;
           mode = "0400";
         };
+      }
+      // lib.optionalAttrs (builtins.pathExists litellmPrometheusBearerSecret) {
+        litellm-prometheus-bearer = {
+          file = litellmPrometheusBearerSecret;
+          owner = "prometheus";
+          group = "prometheus";
+          mode = "0400";
+        };
       };
   };
 }

hosts/AZ-PRM-1/services/monitoring/alerting.nix

@@ -195,6 +195,47 @@ in {
               };
               annotations.description = "A Kestra flow execution ended with FAILED, WARNING, or KILLED.";
             })
+            (mkAlertRule {
+              uid = "az_litellm_down";
+              title = "LiteLLM metrics unreachable";
+              expr = ''up{job="litellm"}'';
+              for = "2m";
+              evaluatorType = "lt";
+              evaluatorParams = [1 0];
+              labels = {
+                service = "litellm";
+                severity = "critical";
+              };
+              annotations.description = "Prometheus cannot scrape LiteLLM /metrics on AZ-CLD-1 over Netbird.";
+            })
+            (mkAlertRule {
+              uid = "az_litellm_proxy_errors";
+              title = "LiteLLM proxy errors";
+              expr = ''sum(increase(litellm_proxy_failed_requests_metric_total{job="litellm"}[5m]))'';
+              for = "1m";
+              noDataState = "OK";
+              evaluatorType = "gt";
+              evaluatorParams = [0 0];
+              labels = {
+                service = "litellm";
+                severity = "warning";
+              };
+              annotations.description = "LiteLLM reported one or more failed proxy responses in the last 5 minutes.";
+            })
+            (mkAlertRule {
+              uid = "az_litellm_latency_high";
+              title = "LiteLLM latency high";
+              expr = ''histogram_quantile(0.95, sum(rate(litellm_request_total_latency_metric_bucket{job="litellm"}[5m])) by (le))'';
+              for = "10m";
+              noDataState = "OK";
+              evaluatorType = "gt";
+              evaluatorParams = [30 0];
+              labels = {
+                service = "litellm";
+                severity = "warning";
+              };
+              annotations.description = "LiteLLM p95 total request latency has been above 30 seconds for 10 minutes.";
+            })
             (mkAlertRule {
               uid = "az_http_probe_failed";
               title = "HTTP probe failed";

hosts/AZ-PRM-1/services/monitoring/cld-scrape.nix

@@ -1,7 +1,23 @@
-{config, ...}: let
+{
+  config,
+  lib,
+  ...
+}: let
   nodeExporterPort = config.m3ta.ports.get "node-exporter";
+  litellmPort = config.m3ta.ports.get "litellm";
   cldNetbirdIP = "100.91.203.184";
+  litellmPrometheusBearerSecretAvailable = builtins.hasAttr "litellm-prometheus-bearer" config.age.secrets;
+  litellmPrometheusAuthorization = lib.optionalAttrs litellmPrometheusBearerSecretAvailable {
+    authorization = {
+      type = "Bearer";
+      credentials_file = config.age.secrets."litellm-prometheus-bearer".path;
+    };
+  };
 in {
+  # Prometheus' config checker validates credentials_file paths at build time,
+  # but agenix secrets only exist at runtime under /run/agenix.
+  services.prometheus.checkConfig = lib.mkIf litellmPrometheusBearerSecretAvailable false;
+
   services.prometheus.scrapeConfigs = [
     {
       job_name = "node-cld";
@@ -14,11 +30,26 @@ in {
         }
       ];
     }
+    ({
+        job_name = "litellm";
+        metrics_path = "/metrics/";
+        scrape_interval = "15s";
+        static_configs = [
+          {
+            targets = ["${cldNetbirdIP}:${toString litellmPort}"];
+            labels = {
+              instance = "AZ-CLD-1";
+              service = "litellm";
+            };
+          }
+        ];
+      }
+      // litellmPrometheusAuthorization)
   ];
 
-  # Allow CLD to reach PRM prometheus scrapes (prometheus initiates connection TO CLD exporter)
+  # Allow CLD to reach PRM prometheus scrapes (prometheus initiates connection TO CLD exporters)
   networking.firewall.extraCommands = ''
-    # No PRM-side firewall change needed: PRM scrapes outbound to CLD:9100.
+    # No PRM-side firewall change needed: PRM scrapes outbound to CLD:9100/4000.
     # CLD-side must allow inbound from 100.91.49.26 (PRM netbird IP) — see CLD config.
   '';
 }

hosts/AZ-PRM-1/services/monitoring/dashboards/az-litellm-observability.json

@@ -0,0 +1,568 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "mappings": [
+            {
+              "options": {
+                "0": {
+                  "color": "red",
+                  "index": 1,
+                  "text": "down"
+                },
+                "1": {
+                  "color": "green",
+                  "index": 0,
+                  "text": "up"
+                }
+              },
+              "type": "value"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "green",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "colorMode": "background",
+        "graphMode": "none",
+        "justifyMode": "center",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "11.0.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "up{job=\"litellm\", instance=~\"$instance\"}",
+          "legendFormat": "{{instance}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Scrape status",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "decimals": 3,
+          "unit": "reqps"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 6,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "11.0.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(litellm_proxy_total_requests_metric_total{job=\"litellm\", instance=~\"$instance\"}[$__rate_interval]))",
+          "legendFormat": "requests/s",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Proxy request rate",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "decimals": 3,
+          "unit": "reqps"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 12,
+        "y": 0
+      },
+      "id": 3,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "11.0.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum(rate(litellm_proxy_failed_requests_metric_total{job=\"litellm\", instance=~\"$instance\"}[$__rate_interval])) or vector(0)",
+          "legendFormat": "errors/s",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Proxy error rate",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "decimals": 2,
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 18,
+        "y": 0
+      },
+      "id": 4,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "11.0.0",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "histogram_quantile(0.95, sum(rate(litellm_request_total_latency_metric_bucket{job=\"litellm\", instance=~\"$instance\"}[$__rate_interval])) by (le))",
+          "legendFormat": "p95",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "p95 total latency",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "lineInterpolation": "linear",
+            "lineWidth": 2,
+            "showPoints": "never",
+            "spanNulls": false
+          },
+          "unit": "reqps"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 4
+      },
+      "id": 5,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum by (requested_model) (rate(litellm_proxy_total_requests_metric_total{job=\"litellm\", instance=~\"$instance\"}[$__rate_interval]))",
+          "legendFormat": "{{requested_model}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Requests by requested model",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "lineInterpolation": "linear",
+            "lineWidth": 2,
+            "showPoints": "never",
+            "spanNulls": false
+          },
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 4
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "sum by (model) (rate(litellm_total_tokens_metric_total{job=\"litellm\", instance=~\"$instance\"}[$__rate_interval]))",
+          "legendFormat": "{{model}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Token rate by model",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "currencyUSD"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 12
+      },
+      "id": 7,
+      "options": {
+        "displayMode": "gradient",
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": false
+        },
+        "orientation": "horizontal",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": true
+        },
+        "showUnfilled": true,
+        "valueMode": "color"
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "topk(10, sum by (model) (increase(litellm_spend_metric_total{job=\"litellm\", instance=~\"$instance\"}[$__range])))",
+          "legendFormat": "{{model}}",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Spend by model in selected range",
+      "type": "bargauge"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "lineInterpolation": "linear",
+            "lineWidth": 2,
+            "showPoints": "never",
+            "spanNulls": false
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 12
+      },
+      "id": 8,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "prometheus"
+          },
+          "editorMode": "code",
+          "expr": "histogram_quantile(0.95, sum by (le, model) (rate(litellm_llm_api_latency_metric_bucket{job=\"litellm\", instance=~\"$instance\"}[$__rate_interval])))",
+          "legendFormat": "{{model}} p95",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "LLM API latency by model",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "loki"
+      },
+      "gridPos": {
+        "h": 10,
+        "w": 24,
+        "x": 0,
+        "y": 20
+      },
+      "id": 9,
+      "options": {
+        "dedupStrategy": "none",
+        "enableLogDetails": true,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showLabels": false,
+        "showTime": true,
+        "sortOrder": "Descending",
+        "wrapLogMessage": true
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "loki"
+          },
+          "editorMode": "code",
+          "expr": "{host=\"AZ-CLD-1\", unit=~\"podman-litellm.service|docker-litellm.service|litellm.service\"}",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "LiteLLM logs from Loki",
+      "type": "logs"
+    }
+  ],
+  "refresh": "30s",
+  "schemaVersion": 39,
+  "style": "dark",
+  "tags": [
+    "az",
+    "litellm",
+    "prometheus",
+    "loki",
+    "provisioned"
+  ],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": true,
+          "text": "All",
+          "value": "$__all"
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "prometheus"
+        },
+        "definition": "label_values(up{job=\"litellm\"}, instance)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "Instance",
+        "multi": true,
+        "name": "instance",
+        "options": [],
+        "query": {
+          "query": "label_values(up{job=\"litellm\"}, instance)",
+          "refId": "PrometheusVariableQueryEditor-VariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 1,
+        "type": "query"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "AZ LiteLLM Observability",
+  "uid": "az-litellm-observability",
+  "version": 1,
+  "weekStart": ""
+}

secrets.nix

@@ -26,6 +26,7 @@ in {
   "secrets/librechat-env-prod.age".publicKeys = systems ++ users;
   "secrets/librechat-env-dev.age".publicKeys = systems ++ users;
   "secrets/litellm-env.age".publicKeys = systems ++ users;
+  "secrets/litellm-prometheus-bearer.age".publicKeys = [AZ-PRM-1] ++ users;
   "secrets/metabase-env.age".publicKeys = systems ++ users;
   "secrets/n8n-env.age".publicKeys = systems ++ users;
   "secrets/n8n-env-prm.age".publicKeys = systems ++ users;

secrets/litellm-prometheus-bearer.age

@@ -0,0 +1,11 @@
+-----BEGIN AGE ENCRYPTED FILE-----
+YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHNzaC1lZDI1NTE5IGxHcVZaZyBJZkVq
+aWU5RTljeXR1ZTdWcmQrL21aMDB1aTQ0Q1hrVWJWVzFSUmorLzBvCmhRWUxjaWsx
+aHczT0h0Z2NhbGhuMjZqRHdMdGRGeU9ueDd1eHJwdk90Y0EKLT4gc3NoLWVkMjU1
+MTkgQ1NNeWhnIEpMbUlxZ2xLR3NQOXhlZzFlWGdrNkd4N3pKWWVJN1I1bjZ5clJZ
+WTR0amsKZW5peVlnZzZvYUxtdU5COGNqVFRoRVQrQ2ludU9XTjc2V2tQak0zL0xP
+dwotPiBiWTt8OyUlZi1ncmVhc2UgRjEyIGUqUykgNV1WVykgdyFTe0wiYwp5aGZw
+bjRPeitGTlhzNVFpbUNuSEw3T1V2YVJlT2IxUkJRCi0tLSA3amRwTWtNOVh1cm1R
+WElER3NYZVZGeXJNdFQwTno5cnFXQWVraUNDOFVVCjQsmAl0ywy880NBAG+Zv2Wz
+jAOi0Sm6BNRdDkkfUGq1i63qy4ZnFxVGlAwmsmIxseicJ2sqteUmlqN2eqLkC3jO
+-----END AGE ENCRYPTED FILE-----