AZ-Intec-GmbH/AZ-NIX-SERVERS
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: prometheus + grafana

Browse Source
This commit is contained in:
Sascha König
2026-07-09 13:25:15 +02:00
parent 800a78848d
commit 61fe3f4338
27 changed files with 2282 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hosts/AZ-PRM-1/services/monitoring/alerting.nix
+242
View File
@@ -0,0 +1,242 @@
{
config,
lib,
...
}: let
prometheusDatasourceUid = "prometheus";
ntfySecretAvailable = builtins.hasAttr "ntfy-grafana-webhook" config.age.secrets;
prometheusQuery = refId: expr: {
inherit refId;
datasourceUid = prometheusDatasourceUid;
relativeTimeRange = {
from = 600;
to = 0;
};
model = {
datasource = {
type = "prometheus";
uid = prometheusDatasourceUid;
};
editorMode = "code";
inherit expr refId;
hide = false;
instant = true;
intervalMs = 1000;
legendFormat = "__auto";
maxDataPoints = 43200;
range = false;
};
};
thresholdExpression = {
refId,
expressionRefId,
evaluator,
}: {
inherit refId;
datasourceUid = "__expr__";
model = {
conditions = [
{
inherit evaluator;
operator.type = "and";
query.params = [];
reducer = {
params = [];
type = "avg";
};
type = "query";
}
];
datasource = {
name = "Expression";
type = "__expr__";
uid = "__expr__";
};
expression = expressionRefId;
hide = false;
inherit refId;
type = "threshold";
};
};
mkAlertRule = {
uid,
title,
expr,
for ? "5m",
noDataState ? "Alerting",
execErrState ? "Error",
evaluatorType ? "gt",
evaluatorParams ? [0 0],
labels ? {},
annotations ? {},
}: {
inherit uid title for noDataState execErrState;
condition = "B";
data = [
(prometheusQuery "A" expr)
(thresholdExpression {
refId = "B";
expressionRefId = "A";
evaluator = {
type = evaluatorType;
params = evaluatorParams;
};
})
];
annotations =
{
summary = title;
}
// annotations;
labels =
{
service = "monitoring";
}
// labels;
isPaused = false;
};
in {
warnings = lib.optional (!ntfySecretAvailable) ''
Grafana alert rules are provisioned, but ntfy notifications are disabled because secrets/ntfy-grafana-webhook.age is missing.
Create it as an EnvironmentFile containing: GRAFANA_NTFY_WEBHOOK_URL=https://<ntfy-webhook-url>
'';
systemd.services.grafana.serviceConfig.EnvironmentFile = lib.mkIf ntfySecretAvailable [
config.age.secrets."ntfy-grafana-webhook".path
];
services.grafana.provision.alerting = {
rules.settings = {
apiVersion = 1;
groups = [
{
orgId = 1;
name = "az-infrastructure";
folder = "Infrastructure";
interval = "60s";
rules = [
(mkAlertRule {
uid = "az_host_down";
title = "Host down";
expr = ''up{job=~"node|node-cld"}'';
for = "2m";
evaluatorType = "lt";
evaluatorParams = [1 0];
labels.severity = "critical";
annotations.description = "Prometheus cannot scrape a node_exporter target.";
})
(mkAlertRule {
uid = "az_cpu_high";
title = "CPU usage high";
expr = ''100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)'';
for = "10m";
evaluatorType = "gt";
evaluatorParams = [90 0];
labels.severity = "warning";
annotations.description = "Average CPU usage has been above 90% for 10 minutes.";
})
(mkAlertRule {
uid = "az_memory_high";
title = "Memory usage high";
expr = ''100 * (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes))'';
for = "10m";
evaluatorType = "gt";
evaluatorParams = [90 0];
labels.severity = "warning";
annotations.description = "Memory usage has been above 90% for 10 minutes.";
})
(mkAlertRule {
uid = "az_rootfs_high";
title = "Root filesystem usage high";
expr = ''100 * (1 - (node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs|ramfs"}))'';
for = "15m";
evaluatorType = "gt";
evaluatorParams = [90 0];
labels.severity = "warning";
annotations.description = "Root filesystem usage has been above 90% for 15 minutes.";
})
(mkAlertRule {
uid = "az_systemd_failed";
title = "Systemd units failed";
expr = ''sum by(instance) (node_systemd_units{state="failed"})'';
for = "5m";
evaluatorType = "gt";
evaluatorParams = [0 0];
labels.severity = "warning";
annotations.description = "One or more systemd units are failed on the host.";
})
(mkAlertRule {
uid = "az_http_probe_failed";
title = "HTTP probe failed";
expr = ''probe_success{job="blackbox_http"}'';
for = "2m";
evaluatorType = "lt";
evaluatorParams = [1 0];
labels.severity = "critical";
annotations.description = "A blackbox HTTP probe is failing.";
})
(mkAlertRule {
uid = "az_icmp_probe_failed";
title = "ICMP probe failed";
expr = ''probe_success{job="blackbox_icmp"}'';
for = "2m";
evaluatorType = "lt";
evaluatorParams = [1 0];
labels.severity = "warning";
annotations.description = "A blackbox ICMP probe is failing.";
})
(mkAlertRule {
uid = "az_tls_cert_expiring";
title = "TLS certificate expires soon";
expr = ''(probe_ssl_earliest_cert_expiry{job="blackbox_http"} - time()) / 86400'';
for = "1h";
noDataState = "OK";
evaluatorType = "lt";
evaluatorParams = [14 0];
labels.severity = "warning";
annotations.description = "A probed TLS certificate expires in less than 14 days.";
})
];
}
];
};
contactPoints.settings = lib.mkIf ntfySecretAvailable {
apiVersion = 1;
contactPoints = [
{
orgId = 1;
name = "ntfy";
receivers = [
{
uid = "ntfy-webhook";
type = "webhook";
disableResolveMessage = false;
settings = {
url = "$GRAFANA_NTFY_WEBHOOK_URL";
httpMethod = "POST";
};
}
];
}
];
};
policies.settings = lib.mkIf ntfySecretAvailable {
apiVersion = 1;
policies = [
{
orgId = 1;
receiver = "ntfy";
group_by = ["alertname" "instance" "target" "severity"];
group_wait = "30s";
group_interval = "5m";
repeat_interval = "4h";
}
];
};
};
}