AZ-NIX-SERVERS/hosts/AZ-PRM-1/services/monitoring/alerting.nix

{
  config,
  lib,
  ...
}: let
  prometheusDatasourceUid = "prometheus";
  ntfySecretAvailable = builtins.hasAttr "ntfy-grafana-webhook" config.age.secrets;

  prometheusQuery = refId: expr: {
    inherit refId;
    datasourceUid = prometheusDatasourceUid;
    relativeTimeRange = {
      from = 600;
      to = 0;
    };
    model = {
      datasource = {
        type = "prometheus";
        uid = prometheusDatasourceUid;
      };
      editorMode = "code";
      inherit expr refId;
      hide = false;
      instant = true;
      intervalMs = 1000;
      legendFormat = "__auto";
      maxDataPoints = 43200;
      range = false;
    };
  };

  thresholdExpression = {
    refId,
    expressionRefId,
    evaluator,
  }: {
    inherit refId;
    datasourceUid = "__expr__";
    model = {
      conditions = [
        {
          inherit evaluator;
          operator.type = "and";
          query.params = [];
          reducer = {
            params = [];
            type = "avg";
          };
          type = "query";
        }
      ];
      datasource = {
        name = "Expression";
        type = "__expr__";
        uid = "__expr__";
      };
      expression = expressionRefId;
      hide = false;
      inherit refId;
      type = "threshold";
    };
  };

  mkAlertRule = {
    uid,
    title,
    expr,
    for ? "5m",
    noDataState ? "Alerting",
    execErrState ? "Error",
    evaluatorType ? "gt",
    evaluatorParams ? [0 0],
    labels ? {},
    annotations ? {},
  }: {
    inherit uid title for noDataState execErrState;
    condition = "B";
    data = [
      (prometheusQuery "A" expr)
      (thresholdExpression {
        refId = "B";
        expressionRefId = "A";
        evaluator = {
          type = evaluatorType;
          params = evaluatorParams;
        };
      })
    ];
    annotations =
      {
        summary = title;
      }
      // annotations;
    labels =
      {
        service = "monitoring";
      }
      // labels;
    isPaused = false;
  };
in {
  warnings = lib.optional (!ntfySecretAvailable) ''
    Grafana alert rules are provisioned, but ntfy notifications are disabled because secrets/ntfy-grafana-webhook.age is missing.
    Create it as an EnvironmentFile containing: GRAFANA_NTFY_WEBHOOK_URL=https://<ntfy-webhook-url>
  '';

  systemd.services.grafana.serviceConfig.EnvironmentFile = lib.mkIf ntfySecretAvailable [
    config.age.secrets."ntfy-grafana-webhook".path
  ];

  services.grafana.provision.alerting = {
    rules.settings = {
      apiVersion = 1;
      groups = [
        {
          orgId = 1;
          name = "az-infrastructure";
          folder = "Infrastructure";
          interval = "60s";
          rules = [
            (mkAlertRule {
              uid = "az_host_down";
              title = "Host down";
              expr = ''up{job=~"node|node-cld"}'';
              for = "2m";
              evaluatorType = "lt";
              evaluatorParams = [1 0];
              labels.severity = "critical";
              annotations.description = "Prometheus cannot scrape a node_exporter target.";
            })
            (mkAlertRule {
              uid = "az_cpu_high";
              title = "CPU usage high";
              expr = ''100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)'';
              for = "10m";
              evaluatorType = "gt";
              evaluatorParams = [90 0];
              labels.severity = "warning";
              annotations.description = "Average CPU usage has been above 90% for 10 minutes.";
            })
            (mkAlertRule {
              uid = "az_memory_high";
              title = "Memory usage high";
              expr = ''100 * (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes))'';
              for = "10m";
              evaluatorType = "gt";
              evaluatorParams = [90 0];
              labels.severity = "warning";
              annotations.description = "Memory usage has been above 90% for 10 minutes.";
            })
            (mkAlertRule {
              uid = "az_rootfs_high";
              title = "Root filesystem usage high";
              expr = ''100 * (1 - (node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs|ramfs"}))'';
              for = "15m";
              evaluatorType = "gt";
              evaluatorParams = [90 0];
              labels.severity = "warning";
              annotations.description = "Root filesystem usage has been above 90% for 15 minutes.";
            })
            (mkAlertRule {
              uid = "az_systemd_failed";
              title = "Systemd units failed";
              expr = ''sum by(instance) (node_systemd_units{state="failed"})'';
              for = "5m";
              evaluatorType = "gt";
              evaluatorParams = [0 0];
              labels.severity = "warning";
              annotations.description = "One or more systemd units are failed on the host.";
            })
            (mkAlertRule {
              uid = "az_kestra_down";
              title = "Kestra metrics unreachable";
              expr = ''up{job="kestra"}'';
              for = "2m";
              evaluatorType = "lt";
              evaluatorParams = [1 0];
              labels = {
                service = "kestra";
                severity = "critical";
              };
              annotations.description = "Prometheus cannot scrape Kestra metrics.";
            })
            (mkAlertRule {
              uid = "az_kestra_execution_failed";
              title = "Kestra execution failed";
              expr = ''sum by(namespace_id, flow_id, state) (increase(kestra_executor_execution_end_count_total{job="kestra",state=~"FAILED|WARNING|KILLED"}[5m]))'';
              for = "1m";
              noDataState = "OK";
              evaluatorType = "gt";
              evaluatorParams = [0 0];
              labels = {
                service = "kestra";
                severity = "critical";
              };
              annotations.description = "A Kestra flow execution ended with FAILED, WARNING, or KILLED.";
            })
            (mkAlertRule {
              uid = "az_http_probe_failed";
              title = "HTTP probe failed";
              expr = ''probe_success{job="blackbox_http"}'';
              for = "2m";
              evaluatorType = "lt";
              evaluatorParams = [1 0];
              labels.severity = "critical";
              annotations.description = "A blackbox HTTP probe is failing.";
            })
            (mkAlertRule {
              uid = "az_icmp_probe_failed";
              title = "ICMP probe failed";
              expr = ''probe_success{job="blackbox_icmp"}'';
              for = "2m";
              evaluatorType = "lt";
              evaluatorParams = [1 0];
              labels.severity = "warning";
              annotations.description = "A blackbox ICMP probe is failing.";
            })
            (mkAlertRule {
              uid = "az_tls_cert_expiring";
              title = "TLS certificate expires soon";
              expr = ''(probe_ssl_earliest_cert_expiry{job="blackbox_http"} - time()) / 86400'';
              for = "1h";
              noDataState = "OK";
              evaluatorType = "lt";
              evaluatorParams = [14 0];
              labels.severity = "warning";
              annotations.description = "A probed TLS certificate expires in less than 14 days.";
            })
          ];
        }
      ];
    };

    contactPoints.settings = lib.mkIf ntfySecretAvailable {
      apiVersion = 1;
      contactPoints = [
        {
          orgId = 1;
          name = "ntfy";
          receivers = [
            {
              uid = "ntfy-webhook";
              type = "webhook";
              disableResolveMessage = false;
              settings = {
                url = "$GRAFANA_NTFY_WEBHOOK_URL";
                httpMethod = "POST";
                title = ''
                  {{ if eq .Status "resolved" }}RESOLVED{{ else }}FIRING{{ end }}: {{ index .CommonLabels "alertname" }}{{ with index .CommonLabels "target" }} {{ . }}{{ else }}{{ with index .CommonLabels "flow_id" }} {{ . }}{{ else }}{{ with index .CommonLabels "instance" }} {{ . }}{{ end }}{{ end }}{{ end }}
                '';
                message = ''
                  {{ range .Alerts -}}
                  {{ $labels := .Labels -}}
                  {{ if eq .Status "resolved" }}✅ Resolved{{ else }}🚨 Firing{{ end }}: {{ index $labels "alertname" }}
                  {{ with index $labels "target" }}Target: {{ . }}
                  {{ end -}}
                  {{ with index $labels "flow_id" }}Flow: {{ with index $labels "namespace_id" }}{{ . }}/{{ end }}{{ . }}
                  {{ end -}}
                  {{ with index $labels "instance" }}Instance: {{ . }}
                  {{ end -}}
                  {{ with index $labels "severity" }}Severity: {{ . }}
                  {{ end -}}
                  {{ with index .Annotations "summary" }}Summary: {{ . }}
                  {{ else }}{{ with index $.CommonAnnotations "summary" }}Summary: {{ . }}
                  {{ end }}{{ end -}}
                  {{ with index .Annotations "description" }}Details: {{ . }}
                  {{ else }}{{ with index $.CommonAnnotations "description" }}Details: {{ . }}
                  {{ end }}{{ end -}}
                  {{ with .GeneratorURL }}Source: {{ . }}
                  {{ end }}
                  {{ end -}}
                '';
                headers = {
                  Template = "grafana";
                  Markdown = "yes";
                };
              };
            }
          ];
        }
      ];
    };

    policies.settings = lib.mkIf ntfySecretAvailable {
      apiVersion = 1;
      policies = [
        {
          orgId = 1;
          receiver = "ntfy";
          group_by = ["alertname" "instance" "target" "severity"];
          group_wait = "30s";
          group_interval = "5m";
          repeat_interval = "4h";
        }
      ];
    };
  };
}