AZ-Intec-GmbH/AZ-NIX-SERVERS
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: litellm -> grafana

Browse Source
This commit is contained in:
Sascha König
2026-07-13 10:32:34 +02:00
parent 8d57aa4bc0
commit d58173760d
8 changed files with 849 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hosts/AZ-PRM-1/services/monitoring/alerting.nix
+41
View File
@@ -195,6 +195,47 @@ in {
};
annotations.description = "A Kestra flow execution ended with FAILED, WARNING, or KILLED.";
})
(mkAlertRule {
uid = "az_litellm_down";
title = "LiteLLM metrics unreachable";
expr = ''up{job="litellm"}'';
for = "2m";
evaluatorType = "lt";
evaluatorParams = [1 0];
labels = {
service = "litellm";
severity = "critical";
};
annotations.description = "Prometheus cannot scrape LiteLLM /metrics on AZ-CLD-1 over Netbird.";
})
(mkAlertRule {
uid = "az_litellm_proxy_errors";
title = "LiteLLM proxy errors";
expr = ''sum(increase(litellm_proxy_failed_requests_metric_total{job="litellm"}[5m]))'';
for = "1m";
noDataState = "OK";
evaluatorType = "gt";
evaluatorParams = [0 0];
labels = {
service = "litellm";
severity = "warning";
};
annotations.description = "LiteLLM reported one or more failed proxy responses in the last 5 minutes.";
})
(mkAlertRule {
uid = "az_litellm_latency_high";
title = "LiteLLM latency high";
expr = ''histogram_quantile(0.95, sum(rate(litellm_request_total_latency_metric_bucket{job="litellm"}[5m])) by (le))'';
for = "10m";
noDataState = "OK";
evaluatorType = "gt";
evaluatorParams = [30 0];
labels = {
service = "litellm";
severity = "warning";
};
annotations.description = "LiteLLM p95 total request latency has been above 30 seconds for 10 minutes.";
})
(mkAlertRule {
uid = "az_http_probe_failed";
title = "HTTP probe failed";