monitor kestra with prometheus alerts

2026-07-09 20:38:19 +02:00
parent de873eba19
commit 4c69d2476a
4 changed files with 48 additions and 1 deletions
@@ -168,6 +168,33 @@ in {
              labels.severity = "warning";
              annotations.description = "One or more systemd units are failed on the host.";
            })
+            (mkAlertRule {
+              uid = "az_kestra_down";
+              title = "Kestra metrics unreachable";
+              expr = ''up{job="kestra"}'';
+              for = "2m";
+              evaluatorType = "lt";
+              evaluatorParams = [1 0];
+              labels = {
+                service = "kestra";
+                severity = "critical";
+              };
+              annotations.description = "Prometheus cannot scrape Kestra metrics.";
+            })
+            (mkAlertRule {
+              uid = "az_kestra_execution_failed";
+              title = "Kestra execution failed";
+              expr = ''sum by(namespace_id, flow_id, state) (increase(kestra_executor_execution_end_count_total{job="kestra",state=~"FAILED|WARNING|KILLED"}[5m]))'';
+              for = "1m";
+              noDataState = "OK";
+              evaluatorType = "gt";
+              evaluatorParams = [0 0];
+              labels = {
+                service = "kestra";
+                severity = "critical";
+              };
+              annotations.description = "A Kestra flow execution ended with FAILED, WARNING, or KILLED.";
+            })
            (mkAlertRule {
              uid = "az_http_probe_failed";
              title = "HTTP probe failed";