monitor kestra with prometheus alerts

hosts/AZ-PRM-1/services/monitoring/alerting.nix

@@ -168,6 +168,33 @@ in {
               labels.severity = "warning";
               annotations.description = "One or more systemd units are failed on the host.";
             })
+            (mkAlertRule {
+              uid = "az_kestra_down";
+              title = "Kestra metrics unreachable";
+              expr = ''up{job="kestra"}'';
+              for = "2m";
+              evaluatorType = "lt";
+              evaluatorParams = [1 0];
+              labels = {
+                service = "kestra";
+                severity = "critical";
+              };
+              annotations.description = "Prometheus cannot scrape Kestra metrics.";
+            })
+            (mkAlertRule {
+              uid = "az_kestra_execution_failed";
+              title = "Kestra execution failed";
+              expr = ''sum by(namespace_id, flow_id, state) (increase(kestra_executor_execution_end_count_total{job="kestra",state=~"FAILED|WARNING|KILLED"}[5m]))'';
+              for = "1m";
+              noDataState = "OK";
+              evaluatorType = "gt";
+              evaluatorParams = [0 0];
+              labels = {
+                service = "kestra";
+                severity = "critical";
+              };
+              annotations.description = "A Kestra flow execution ended with FAILED, WARNING, or KILLED.";
+            })
             (mkAlertRule {
               uid = "az_http_probe_failed";
               title = "HTTP probe failed";

hosts/AZ-PRM-1/services/monitoring/prometheus.nix

@@ -1,6 +1,7 @@
 {config, ...}: let
   prometheusPort = config.m3ta.ports.get "prometheus";
   nodeExporterPort = config.m3ta.ports.get "node-exporter";
+  kestraMetricsPort = config.m3ta.ports.get "kestra-metrics";
 in {
   services.prometheus = {
     enable = true;
@@ -20,6 +21,20 @@ in {
           }
         ];
       }
+      {
+        job_name = "kestra";
+        metrics_path = "/prometheus";
+        scrape_interval = "15s";
+        static_configs = [
+          {
+            targets = ["localhost:${toString kestraMetricsPort}"];
+            labels = {
+              instance = "AZ-PRM-1";
+              service = "kestra";
+            };
+          }
+        ];
+      }
       # Phase 2: add AZ-CLD-1 node target (scrape over netbird 100.x).
       # Phase 2: add blackbox_exporter, podman/cadvisor targets.
       # Phase 2: enable remote-write receiver (extraFlags) for Windows clients.