From 4c69d2476a16ede3eacdce4b1c619cec0ef0b8b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sascha=20K=C3=B6nig?= <sascha.koenig@azintec.com>
Date: Thu, 9 Jul 2026 20:38:19 +0200
Subject: [PATCH] monitor kestra with prometheus alerts

---
 hosts/AZ-PRM-1/services/containers/kestra.nix |  6 ++++-
 .../AZ-PRM-1/services/monitoring/alerting.nix | 27 +++++++++++++++++++
 .../services/monitoring/prometheus.nix        | 15 +++++++++++
 hosts/common/ports.nix                        |  1 +
 4 files changed, 48 insertions(+), 1 deletion(-)

diff --git a/hosts/AZ-PRM-1/services/containers/kestra.nix b/hosts/AZ-PRM-1/services/containers/kestra.nix
index 50e52fd..afd089c 100644
--- a/hosts/AZ-PRM-1/services/containers/kestra.nix
+++ b/hosts/AZ-PRM-1/services/containers/kestra.nix
@@ -1,6 +1,7 @@
 {config, ...}: let
   serviceName = "kestra";
   servicePort = config.m3ta.ports.get serviceName;
+  metricsPort = config.m3ta.ports.get "kestra-metrics";
 in {
   virtualisation.oci-containers.containers."${serviceName}" = {
     image = "docker.io/kestra/kestra:latest";
@@ -9,7 +10,10 @@ in {
       config.age.secrets.kestra-secrets.path
     ];
     cmd = ["server" "standalone" "--config" "/etc/config/application.yaml"];
-    ports = ["127.0.0.1:${toString servicePort}:8080"];
+    ports = [
+      "127.0.0.1:${toString servicePort}:8080"
+      "127.0.0.1:${toString metricsPort}:8081"
+    ];
     user = "root";
     volumes = [
       "/var/run/podman/podman.sock:/var/run/docker.sock"
diff --git a/hosts/AZ-PRM-1/services/monitoring/alerting.nix b/hosts/AZ-PRM-1/services/monitoring/alerting.nix
index c1ff669..d96dbca 100644
--- a/hosts/AZ-PRM-1/services/monitoring/alerting.nix
+++ b/hosts/AZ-PRM-1/services/monitoring/alerting.nix
@@ -168,6 +168,33 @@ in {
               labels.severity = "warning";
               annotations.description = "One or more systemd units are failed on the host.";
             })
+            (mkAlertRule {
+              uid = "az_kestra_down";
+              title = "Kestra metrics unreachable";
+              expr = ''up{job="kestra"}'';
+              for = "2m";
+              evaluatorType = "lt";
+              evaluatorParams = [1 0];
+              labels = {
+                service = "kestra";
+                severity = "critical";
+              };
+              annotations.description = "Prometheus cannot scrape Kestra metrics.";
+            })
+            (mkAlertRule {
+              uid = "az_kestra_execution_failed";
+              title = "Kestra execution failed";
+              expr = ''sum by(namespace_id, flow_id, state) (increase(kestra_executor_execution_end_count_total{job="kestra",state=~"FAILED|WARNING|KILLED"}[5m]))'';
+              for = "1m";
+              noDataState = "OK";
+              evaluatorType = "gt";
+              evaluatorParams = [0 0];
+              labels = {
+                service = "kestra";
+                severity = "critical";
+              };
+              annotations.description = "A Kestra flow execution ended with FAILED, WARNING, or KILLED.";
+            })
             (mkAlertRule {
               uid = "az_http_probe_failed";
               title = "HTTP probe failed";
diff --git a/hosts/AZ-PRM-1/services/monitoring/prometheus.nix b/hosts/AZ-PRM-1/services/monitoring/prometheus.nix
index 054a529..d15a3f8 100644
--- a/hosts/AZ-PRM-1/services/monitoring/prometheus.nix
+++ b/hosts/AZ-PRM-1/services/monitoring/prometheus.nix
@@ -1,6 +1,7 @@
 {config, ...}: let
   prometheusPort = config.m3ta.ports.get "prometheus";
   nodeExporterPort = config.m3ta.ports.get "node-exporter";
+  kestraMetricsPort = config.m3ta.ports.get "kestra-metrics";
 in {
   services.prometheus = {
     enable = true;
@@ -20,6 +21,20 @@ in {
           }
         ];
       }
+      {
+        job_name = "kestra";
+        metrics_path = "/prometheus";
+        scrape_interval = "15s";
+        static_configs = [
+          {
+            targets = ["localhost:${toString kestraMetricsPort}"];
+            labels = {
+              instance = "AZ-PRM-1";
+              service = "kestra";
+            };
+          }
+        ];
+      }
       # Phase 2: add AZ-CLD-1 node target (scrape over netbird 100.x).
       # Phase 2: add blackbox_exporter, podman/cadvisor targets.
       # Phase 2: enable remote-write receiver (extraFlags) for Windows clients.
diff --git a/hosts/common/ports.nix b/hosts/common/ports.nix
index 2b7ca51..372b5da 100644
--- a/hosts/common/ports.nix
+++ b/hosts/common/ports.nix
@@ -34,6 +34,7 @@
 
       n8n = 5678;
       kestra = 5080;
+      kestra-metrics = 5081;
       zugferd-service = 5060;
       gotenberg = 5070;