monitor kestra with prometheus alerts

hosts/AZ-PRM-1/services/containers/kestra.nix

@@ -1,6 +1,7 @@
 {config, ...}: let
   serviceName = "kestra";
   servicePort = config.m3ta.ports.get serviceName;
+  metricsPort = config.m3ta.ports.get "kestra-metrics";
 in {
   virtualisation.oci-containers.containers."${serviceName}" = {
     image = "docker.io/kestra/kestra:latest";
@@ -9,7 +10,10 @@ in {
       config.age.secrets.kestra-secrets.path
     ];
     cmd = ["server" "standalone" "--config" "/etc/config/application.yaml"];
-    ports = ["127.0.0.1:${toString servicePort}:8080"];
+    ports = [
+      "127.0.0.1:${toString servicePort}:8080"
+      "127.0.0.1:${toString metricsPort}:8081"
+    ];
     user = "root";
     volumes = [
       "/var/run/podman/podman.sock:/var/run/docker.sock"

hosts/AZ-PRM-1/services/monitoring/alerting.nix

@@ -168,6 +168,33 @@ in {
               labels.severity = "warning";
               annotations.description = "One or more systemd units are failed on the host.";
             })
+            (mkAlertRule {
+              uid = "az_kestra_down";
+              title = "Kestra metrics unreachable";
+              expr = ''up{job="kestra"}'';
+              for = "2m";
+              evaluatorType = "lt";
+              evaluatorParams = [1 0];
+              labels = {
+                service = "kestra";
+                severity = "critical";
+              };
+              annotations.description = "Prometheus cannot scrape Kestra metrics.";
+            })
+            (mkAlertRule {
+              uid = "az_kestra_execution_failed";
+              title = "Kestra execution failed";
+              expr = ''sum by(namespace_id, flow_id, state) (increase(kestra_executor_execution_end_count_total{job="kestra",state=~"FAILED|WARNING|KILLED"}[5m]))'';
+              for = "1m";
+              noDataState = "OK";
+              evaluatorType = "gt";
+              evaluatorParams = [0 0];
+              labels = {
+                service = "kestra";
+                severity = "critical";
+              };
+              annotations.description = "A Kestra flow execution ended with FAILED, WARNING, or KILLED.";
+            })
             (mkAlertRule {
               uid = "az_http_probe_failed";
               title = "HTTP probe failed";

hosts/AZ-PRM-1/services/monitoring/prometheus.nix

@@ -1,6 +1,7 @@
 {config, ...}: let
   prometheusPort = config.m3ta.ports.get "prometheus";
   nodeExporterPort = config.m3ta.ports.get "node-exporter";
+  kestraMetricsPort = config.m3ta.ports.get "kestra-metrics";
 in {
   services.prometheus = {
     enable = true;
@@ -20,6 +21,20 @@ in {
           }
         ];
       }
+      {
+        job_name = "kestra";
+        metrics_path = "/prometheus";
+        scrape_interval = "15s";
+        static_configs = [
+          {
+            targets = ["localhost:${toString kestraMetricsPort}"];
+            labels = {
+              instance = "AZ-PRM-1";
+              service = "kestra";
+            };
+          }
+        ];
+      }
       # Phase 2: add AZ-CLD-1 node target (scrape over netbird 100.x).
       # Phase 2: add blackbox_exporter, podman/cadvisor targets.
       # Phase 2: enable remote-write receiver (extraFlags) for Windows clients.

hosts/common/ports.nix

@@ -34,6 +34,7 @@
 
       n8n = 5678;
       kestra = 5080;
+      kestra-metrics = 5081;
       zugferd-service = 5060;
       gotenberg = 5070;