319 lines
10 KiB
Nix
319 lines
10 KiB
Nix
{
|
|
config,
|
|
lib,
|
|
...
|
|
}: let
|
|
prometheusDatasourceUid = "prometheus";
|
|
ntfySecretAvailable = builtins.hasAttr "ntfy-grafana-webhook" config.age.secrets;
|
|
|
|
prometheusQuery = refId: expr: {
|
|
inherit refId;
|
|
datasourceUid = prometheusDatasourceUid;
|
|
relativeTimeRange = {
|
|
from = 600;
|
|
to = 0;
|
|
};
|
|
model = {
|
|
datasource = {
|
|
type = "prometheus";
|
|
uid = prometheusDatasourceUid;
|
|
};
|
|
editorMode = "code";
|
|
inherit expr refId;
|
|
hide = false;
|
|
instant = true;
|
|
intervalMs = 1000;
|
|
legendFormat = "__auto";
|
|
maxDataPoints = 43200;
|
|
range = false;
|
|
};
|
|
};
|
|
|
|
thresholdExpression = {
|
|
refId,
|
|
expressionRefId,
|
|
evaluator,
|
|
}: {
|
|
inherit refId;
|
|
datasourceUid = "__expr__";
|
|
model = {
|
|
conditions = [
|
|
{
|
|
inherit evaluator;
|
|
operator.type = "and";
|
|
query.params = [];
|
|
reducer = {
|
|
params = [];
|
|
type = "avg";
|
|
};
|
|
type = "query";
|
|
}
|
|
];
|
|
datasource = {
|
|
name = "Expression";
|
|
type = "__expr__";
|
|
uid = "__expr__";
|
|
};
|
|
expression = expressionRefId;
|
|
hide = false;
|
|
inherit refId;
|
|
type = "threshold";
|
|
};
|
|
};
|
|
|
|
mkAlertRule = {
|
|
uid,
|
|
title,
|
|
expr,
|
|
for ? "5m",
|
|
noDataState ? "Alerting",
|
|
execErrState ? "Error",
|
|
evaluatorType ? "gt",
|
|
evaluatorParams ? [0 0],
|
|
labels ? {},
|
|
annotations ? {},
|
|
}: {
|
|
inherit uid title for noDataState execErrState;
|
|
condition = "B";
|
|
data = [
|
|
(prometheusQuery "A" expr)
|
|
(thresholdExpression {
|
|
refId = "B";
|
|
expressionRefId = "A";
|
|
evaluator = {
|
|
type = evaluatorType;
|
|
params = evaluatorParams;
|
|
};
|
|
})
|
|
];
|
|
annotations =
|
|
{
|
|
summary = title;
|
|
}
|
|
// annotations;
|
|
labels =
|
|
{
|
|
service = "monitoring";
|
|
}
|
|
// labels;
|
|
isPaused = false;
|
|
};
|
|
in {
|
|
warnings = lib.optional (!ntfySecretAvailable) ''
|
|
Grafana alert rules are provisioned, but ntfy notifications are disabled because secrets/ntfy-grafana-webhook.age is missing.
|
|
Create it as an EnvironmentFile containing: GRAFANA_NTFY_WEBHOOK_URL=https://<ntfy-webhook-url>
|
|
'';
|
|
|
|
systemd.services.grafana.serviceConfig.EnvironmentFile = lib.mkIf ntfySecretAvailable [
|
|
config.age.secrets."ntfy-grafana-webhook".path
|
|
];
|
|
|
|
services.grafana.provision.alerting = {
|
|
rules.settings = {
|
|
apiVersion = 1;
|
|
groups = [
|
|
{
|
|
orgId = 1;
|
|
name = "az-infrastructure";
|
|
folder = "Infrastructure";
|
|
interval = "60s";
|
|
rules = [
|
|
(mkAlertRule {
|
|
uid = "az_host_down";
|
|
title = "Host down";
|
|
expr = ''up{job=~"node|node-cld"}'';
|
|
for = "2m";
|
|
evaluatorType = "lt";
|
|
evaluatorParams = [1 0];
|
|
labels.severity = "critical";
|
|
annotations.description = "Prometheus cannot scrape a node_exporter target.";
|
|
})
|
|
(mkAlertRule {
|
|
uid = "az_cpu_high";
|
|
title = "CPU usage high";
|
|
expr = ''100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)'';
|
|
for = "10m";
|
|
evaluatorType = "gt";
|
|
evaluatorParams = [90 0];
|
|
labels.severity = "warning";
|
|
annotations.description = "Average CPU usage has been above 90% for 10 minutes.";
|
|
})
|
|
(mkAlertRule {
|
|
uid = "az_memory_high";
|
|
title = "Memory usage high";
|
|
expr = ''100 * (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes))'';
|
|
for = "10m";
|
|
evaluatorType = "gt";
|
|
evaluatorParams = [90 0];
|
|
labels.severity = "warning";
|
|
annotations.description = "Memory usage has been above 90% for 10 minutes.";
|
|
})
|
|
(mkAlertRule {
|
|
uid = "az_rootfs_high";
|
|
title = "Root filesystem usage high";
|
|
expr = ''100 * (1 - (node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs|ramfs"}))'';
|
|
for = "15m";
|
|
evaluatorType = "gt";
|
|
evaluatorParams = [90 0];
|
|
labels.severity = "warning";
|
|
annotations.description = "Root filesystem usage has been above 90% for 15 minutes.";
|
|
})
|
|
(mkAlertRule {
|
|
uid = "az_systemd_failed";
|
|
title = "Systemd units failed";
|
|
expr = ''sum by(instance) (node_systemd_units{state="failed"})'';
|
|
for = "5m";
|
|
evaluatorType = "gt";
|
|
evaluatorParams = [0 0];
|
|
labels.severity = "warning";
|
|
annotations.description = "One or more systemd units are failed on the host.";
|
|
})
|
|
(mkAlertRule {
|
|
uid = "az_kestra_down";
|
|
title = "Kestra metrics unreachable";
|
|
expr = ''up{job="kestra"}'';
|
|
for = "2m";
|
|
evaluatorType = "lt";
|
|
evaluatorParams = [1 0];
|
|
labels = {
|
|
service = "kestra";
|
|
severity = "critical";
|
|
};
|
|
annotations.description = "Prometheus cannot scrape Kestra metrics.";
|
|
})
|
|
(mkAlertRule {
|
|
uid = "az_kestra_execution_failed";
|
|
title = "Kestra execution failed";
|
|
expr = ''sum by(namespace_id, flow_id, state) (increase(kestra_executor_execution_end_count_total{job="kestra",state=~"FAILED|WARNING|KILLED"}[5m]))'';
|
|
for = "1m";
|
|
noDataState = "OK";
|
|
evaluatorType = "gt";
|
|
evaluatorParams = [0 0];
|
|
labels = {
|
|
service = "kestra";
|
|
severity = "critical";
|
|
};
|
|
annotations.description = "A Kestra flow execution ended with FAILED, WARNING, or KILLED.";
|
|
})
|
|
(mkAlertRule {
|
|
uid = "az_litellm_down";
|
|
title = "LiteLLM metrics unreachable";
|
|
expr = ''up{job="litellm"}'';
|
|
for = "2m";
|
|
evaluatorType = "lt";
|
|
evaluatorParams = [1 0];
|
|
labels = {
|
|
service = "litellm";
|
|
severity = "critical";
|
|
};
|
|
annotations.description = "Prometheus cannot scrape LiteLLM /metrics on AZ-CLD-1 over Netbird.";
|
|
})
|
|
(mkAlertRule {
|
|
uid = "az_litellm_proxy_errors";
|
|
title = "LiteLLM proxy errors";
|
|
expr = ''sum(increase(litellm_proxy_failed_requests_metric_total{job="litellm"}[5m]))'';
|
|
for = "1m";
|
|
noDataState = "OK";
|
|
evaluatorType = "gt";
|
|
evaluatorParams = [0 0];
|
|
labels = {
|
|
service = "litellm";
|
|
severity = "warning";
|
|
};
|
|
annotations.description = "LiteLLM reported one or more failed proxy responses in the last 5 minutes.";
|
|
})
|
|
(mkAlertRule {
|
|
uid = "az_litellm_latency_high";
|
|
title = "LiteLLM latency high";
|
|
expr = ''histogram_quantile(0.95, sum(rate(litellm_request_total_latency_metric_bucket{job="litellm"}[5m])) by (le))'';
|
|
for = "10m";
|
|
noDataState = "OK";
|
|
evaluatorType = "gt";
|
|
evaluatorParams = [30 0];
|
|
labels = {
|
|
service = "litellm";
|
|
severity = "warning";
|
|
};
|
|
annotations.description = "LiteLLM p95 total request latency has been above 30 seconds for 10 minutes.";
|
|
})
|
|
(mkAlertRule {
|
|
uid = "az_http_probe_failed";
|
|
title = "HTTP probe failed";
|
|
expr = ''probe_success{job="blackbox_http"}'';
|
|
for = "2m";
|
|
evaluatorType = "lt";
|
|
evaluatorParams = [1 0];
|
|
labels.severity = "critical";
|
|
annotations.description = "A blackbox HTTP probe is failing.";
|
|
})
|
|
(mkAlertRule {
|
|
uid = "az_icmp_probe_failed";
|
|
title = "ICMP probe failed";
|
|
expr = ''probe_success{job="blackbox_icmp"}'';
|
|
for = "2m";
|
|
evaluatorType = "lt";
|
|
evaluatorParams = [1 0];
|
|
labels.severity = "warning";
|
|
annotations.description = "A blackbox ICMP probe is failing.";
|
|
})
|
|
(mkAlertRule {
|
|
uid = "az_tls_cert_expiring";
|
|
title = "TLS certificate expires soon";
|
|
expr = ''(probe_ssl_earliest_cert_expiry{job="blackbox_http"} - time()) / 86400'';
|
|
for = "1h";
|
|
noDataState = "OK";
|
|
evaluatorType = "lt";
|
|
evaluatorParams = [14 0];
|
|
labels.severity = "warning";
|
|
annotations.description = "A probed TLS certificate expires in less than 14 days.";
|
|
})
|
|
];
|
|
}
|
|
];
|
|
};
|
|
|
|
contactPoints.settings = lib.mkIf ntfySecretAvailable {
|
|
apiVersion = 1;
|
|
contactPoints = [
|
|
{
|
|
orgId = 1;
|
|
name = "ntfy";
|
|
receivers = [
|
|
{
|
|
uid = "ntfy-webhook";
|
|
type = "webhook";
|
|
disableResolveMessage = false;
|
|
settings = {
|
|
url = "$GRAFANA_NTFY_WEBHOOK_URL";
|
|
httpMethod = "POST";
|
|
# Let Grafana render its default title/message. The default message
|
|
# includes all alert annotations via .Annotations.SortedPairs;
|
|
# ntfy's grafana template then displays only title/message instead
|
|
# of the full webhook JSON body.
|
|
headers = {
|
|
Template = "grafana";
|
|
Markdown = "yes";
|
|
};
|
|
};
|
|
}
|
|
];
|
|
}
|
|
];
|
|
};
|
|
|
|
policies.settings = lib.mkIf ntfySecretAvailable {
|
|
apiVersion = 1;
|
|
policies = [
|
|
{
|
|
orgId = 1;
|
|
receiver = "ntfy";
|
|
group_by = ["alertname" "instance" "target" "severity"];
|
|
group_wait = "30s";
|
|
group_interval = "5m";
|
|
repeat_interval = "4h";
|
|
}
|
|
];
|
|
};
|
|
};
|
|
}
|