AZ-Intec-GmbH/AZ-NIX-SERVERS
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: prometheus + grafana

Browse Source
This commit is contained in:
Sascha König
2026-07-09 13:25:15 +02:00
parent 800a78848d
commit 61fe3f4338
27 changed files with 2282 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hosts/AZ-CLD-1/services/default.nix
+1
View File
@@ -1,6 +1,7 @@
{
imports = [
./containers
./monitoring
./gitea.nix
# ./gotenberg.nix
hosts/AZ-CLD-1/services/monitoring/default.nix
+6
View File
@@ -0,0 +1,6 @@
# Phase 2 — activate by adding `./monitoring` to hosts/AZ-CLD-1/services/default.nix imports.
{...}: {
imports = [
./node-exporter.nix
];
}
hosts/AZ-CLD-1/services/monitoring/node-exporter.nix
+26
View File
@@ -0,0 +1,26 @@
# Phase 2 — not active until imported.
# Activate by:
# 1. Create hosts/AZ-CLD-1/services/monitoring/default.nix with `imports = [ ./node-exporter.nix ];`
# 2. Add `./monitoring` to hosts/AZ-CLD-1/services/default.nix imports.
#
# binds to netbird interface so PRM prometheus can scrape it over VPN.
{config, ...}: let
nodeExporterPort = config.m3ta.ports.get "node-exporter";
prmNetbirdIP = "100.91.49.26";
in {
services.prometheus.exporters.node = {
enable = true;
port = nodeExporterPort;
listenAddress = "100.91.203.184"; # CLD netbird IP — overwrite if changed
enabledCollectors = [
"systemd"
"diskstats"
"filesystem"
];
openFirewall = false;
};
networking.firewall.extraCommands = ''
iptables -A INPUT -p tcp -s ${prmNetbirdIP} --dport ${toString nodeExporterPort} -j ACCEPT
'';
}
hosts/AZ-CLD-1/services/n8n.nix
+8 -1
View File
@@ -9,8 +9,15 @@ in {
N8N_RUNNERS_ENABLED = true;
N8N_NATIVE_PYTHON_RUNNER = true;
N8N_RUNNERS_AUTH_TOKEN_FILE = config.age.secrets.n8n-runner-auth-token.path;
NODE_FUNCTION_ALLOW_EXTERNAL = "*";
N8N_RUNNERS_STDLIB_ALLOW = "*";
};
taskRunners = {
enable = true;
environment = {
N8N_RUNNERS_STDLIB_ALLOW = "*";
};
};
taskRunners.enable = true;
};
systemd.services.${serviceName}.serviceConfig = {
hosts/AZ-CLD-1/services/postgres.nix
+3 -3
View File
@@ -98,7 +98,7 @@ in {
local all jannik_mueller scram-sha-256
local az_test az_test scram-sha-256
local metabase,az_kpi_raw metabase scram-sha-256
local n8n n8n scram-sha-256
local all n8n scram-sha-256
local outline outline scram-sha-256
local vaultwarden vaultwarden scram-sha-256
local zammad zammad scram-sha-256
@@ -122,8 +122,8 @@ in {
host metabase,az_kpi_raw metabase 127.0.0.1/32 scram-sha-256
host metabase,az_kpi_raw metabase ::1/128 scram-sha-256
host n8n n8n 127.0.0.1/32 scram-sha-256
host n8n n8n ::1/128 scram-sha-256
host all n8n 127.0.0.1/32 scram-sha-256
host all n8n ::1/128 scram-sha-256
host vaultwarden vaultwarden 127.0.0.1/32 scram-sha-256
host vaultwarden vaultwarden ::1/128 scram-sha-256
hosts/AZ-PRM-1/secrets.nix
+60 -39
View File
@@ -1,43 +1,64 @@
{
{lib, ...}: let
ntfyGrafanaWebhookSecret = ../../secrets/ntfy-grafana-webhook.age;
in {
age = {
secrets = {
azion-env = {
file = ../../secrets/azion-env.age;
secrets =
{
azion-env = {
file = ../../secrets/azion-env.age;
};
grafana-admin-pw = {
file = ../../secrets/grafana-admin-pw.age;
owner = "grafana";
};
grafana-db-password = {
file = ../../secrets/grafana-db-password.age;
owner = "grafana";
};
grafana-secret-key = {
file = ../../secrets/grafana-secret-key.age;
owner = "grafana";
};
traefik-env = {
file = ../../secrets/traefik-env.age;
};
kestra-config = {
file = ../../secrets/kestra-config.age;
mode = "644";
};
kestra-env = {file = ../../secrets/kestra-env.age;};
kestra-secrets = {file = ../../secrets/kestra-secrets.age;};
n8n-env = {
file = ../../secrets/n8n-env-prm.age;
};
n8n-runner-auth-token = {
file = ../../secrets/n8n-runner-auth-token-prm.age;
};
pgadmin-pw = {
file = ../../secrets/pgadmin-pw.age;
owner = "pgadmin";
};
pg-cert = {
file = ../../secrets/server.crt.age;
owner = "postgres";
group = "postgres";
mode = "0644";
};
pg-key = {
file = ../../secrets/server.key.age;
owner = "postgres";
group = "postgres";
mode = "0600";
};
smb-autoablage = {
file = ../../secrets/smb-autoablage.age;
};
}
// lib.optionalAttrs (builtins.pathExists ntfyGrafanaWebhookSecret) {
ntfy-grafana-webhook = {
file = ntfyGrafanaWebhookSecret;
mode = "0400";
};
};
traefik-env = {
file = ../../secrets/traefik-env.age;
};
kestra-config = {
file = ../../secrets/kestra-config.age;
mode = "644";
};
kestra-env = {file = ../../secrets/kestra-env.age;};
kestra-secrets = {file = ../../secrets/kestra-secrets.age;};
n8n-env = {
file = ../../secrets/n8n-env-prm.age;
};
n8n-runner-auth-token = {
file = ../../secrets/n8n-runner-auth-token-prm.age;
};
pgadmin-pw = {
file = ../../secrets/pgadmin-pw.age;
owner = "pgadmin";
};
pg-cert = {
file = ../../secrets/server.crt.age;
owner = "postgres";
group = "postgres";
mode = "0644";
};
pg-key = {
file = ../../secrets/server.key.age;
owner = "postgres";
group = "postgres";
mode = "0600";
};
smb-autoablage = {
file = ../../secrets/smb-autoablage.age;
};
};
};
}
hosts/AZ-PRM-1/services/azess.nix
+26
View File
@@ -0,0 +1,26 @@
{config, ...}: let
serviceName = "azess";
servicePort = config.m3ta.ports.get serviceName;
in {
services.azess = {
enable = true;
host = "127.0.0.1";
port = servicePort;
workers = 4;
};
services.traefik.dynamicConfigOptions.http = {
services.${serviceName}.loadBalancer.servers = [
{url = "http://localhost:${toString servicePort}/";}
];
routers.${serviceName} = {
rule = "Host(`azess.l.az-gruppe.com`)";
tls = {
certResolver = "ionos";
};
service = serviceName;
entrypoints = "websecure";
};
};
}
hosts/AZ-PRM-1/services/default.nix
+2
View File
@@ -2,8 +2,10 @@
imports = [
./containers
./backup-ingest.nix
./azess.nix
./azion-scheduler.nix
./bpi.nix
./monitoring
./n8n.nix
./netbird.nix
./pgadmin.nix
hosts/AZ-PRM-1/services/monitoring/alerting.nix
+242
View File
@@ -0,0 +1,242 @@
{
config,
lib,
...
}: let
prometheusDatasourceUid = "prometheus";
ntfySecretAvailable = builtins.hasAttr "ntfy-grafana-webhook" config.age.secrets;
prometheusQuery = refId: expr: {
inherit refId;
datasourceUid = prometheusDatasourceUid;
relativeTimeRange = {
from = 600;
to = 0;
};
model = {
datasource = {
type = "prometheus";
uid = prometheusDatasourceUid;
};
editorMode = "code";
inherit expr refId;
hide = false;
instant = true;
intervalMs = 1000;
legendFormat = "__auto";
maxDataPoints = 43200;
range = false;
};
};
thresholdExpression = {
refId,
expressionRefId,
evaluator,
}: {
inherit refId;
datasourceUid = "__expr__";
model = {
conditions = [
{
inherit evaluator;
operator.type = "and";
query.params = [];
reducer = {
params = [];
type = "avg";
};
type = "query";
}
];
datasource = {
name = "Expression";
type = "__expr__";
uid = "__expr__";
};
expression = expressionRefId;
hide = false;
inherit refId;
type = "threshold";
};
};
mkAlertRule = {
uid,
title,
expr,
for ? "5m",
noDataState ? "Alerting",
execErrState ? "Error",
evaluatorType ? "gt",
evaluatorParams ? [0 0],
labels ? {},
annotations ? {},
}: {
inherit uid title for noDataState execErrState;
condition = "B";
data = [
(prometheusQuery "A" expr)
(thresholdExpression {
refId = "B";
expressionRefId = "A";
evaluator = {
type = evaluatorType;
params = evaluatorParams;
};
})
];
annotations =
{
summary = title;
}
// annotations;
labels =
{
service = "monitoring";
}
// labels;
isPaused = false;
};
in {
warnings = lib.optional (!ntfySecretAvailable) ''
Grafana alert rules are provisioned, but ntfy notifications are disabled because secrets/ntfy-grafana-webhook.age is missing.
Create it as an EnvironmentFile containing: GRAFANA_NTFY_WEBHOOK_URL=https://<ntfy-webhook-url>
'';
systemd.services.grafana.serviceConfig.EnvironmentFile = lib.mkIf ntfySecretAvailable [
config.age.secrets."ntfy-grafana-webhook".path
];
services.grafana.provision.alerting = {
rules.settings = {
apiVersion = 1;
groups = [
{
orgId = 1;
name = "az-infrastructure";
folder = "Infrastructure";
interval = "60s";
rules = [
(mkAlertRule {
uid = "az_host_down";
title = "Host down";
expr = ''up{job=~"node|node-cld"}'';
for = "2m";
evaluatorType = "lt";
evaluatorParams = [1 0];
labels.severity = "critical";
annotations.description = "Prometheus cannot scrape a node_exporter target.";
})
(mkAlertRule {
uid = "az_cpu_high";
title = "CPU usage high";
expr = ''100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)'';
for = "10m";
evaluatorType = "gt";
evaluatorParams = [90 0];
labels.severity = "warning";
annotations.description = "Average CPU usage has been above 90% for 10 minutes.";
})
(mkAlertRule {
uid = "az_memory_high";
title = "Memory usage high";
expr = ''100 * (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes))'';
for = "10m";
evaluatorType = "gt";
evaluatorParams = [90 0];
labels.severity = "warning";
annotations.description = "Memory usage has been above 90% for 10 minutes.";
})
(mkAlertRule {
uid = "az_rootfs_high";
title = "Root filesystem usage high";
expr = ''100 * (1 - (node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs|ramfs"}))'';
for = "15m";
evaluatorType = "gt";
evaluatorParams = [90 0];
labels.severity = "warning";
annotations.description = "Root filesystem usage has been above 90% for 15 minutes.";
})
(mkAlertRule {
uid = "az_systemd_failed";
title = "Systemd units failed";
expr = ''sum by(instance) (node_systemd_units{state="failed"})'';
for = "5m";
evaluatorType = "gt";
evaluatorParams = [0 0];
labels.severity = "warning";
annotations.description = "One or more systemd units are failed on the host.";
})
(mkAlertRule {
uid = "az_http_probe_failed";
title = "HTTP probe failed";
expr = ''probe_success{job="blackbox_http"}'';
for = "2m";
evaluatorType = "lt";
evaluatorParams = [1 0];
labels.severity = "critical";
annotations.description = "A blackbox HTTP probe is failing.";
})
(mkAlertRule {
uid = "az_icmp_probe_failed";
title = "ICMP probe failed";
expr = ''probe_success{job="blackbox_icmp"}'';
for = "2m";
evaluatorType = "lt";
evaluatorParams = [1 0];
labels.severity = "warning";
annotations.description = "A blackbox ICMP probe is failing.";
})
(mkAlertRule {
uid = "az_tls_cert_expiring";
title = "TLS certificate expires soon";
expr = ''(probe_ssl_earliest_cert_expiry{job="blackbox_http"} - time()) / 86400'';
for = "1h";
noDataState = "OK";
evaluatorType = "lt";
evaluatorParams = [14 0];
labels.severity = "warning";
annotations.description = "A probed TLS certificate expires in less than 14 days.";
})
];
}
];
};
contactPoints.settings = lib.mkIf ntfySecretAvailable {
apiVersion = 1;
contactPoints = [
{
orgId = 1;
name = "ntfy";
receivers = [
{
uid = "ntfy-webhook";
type = "webhook";
disableResolveMessage = false;
settings = {
url = "$GRAFANA_NTFY_WEBHOOK_URL";
httpMethod = "POST";
};
}
];
}
];
};
policies.settings = lib.mkIf ntfySecretAvailable {
apiVersion = 1;
policies = [
{
orgId = 1;
receiver = "ntfy";
group_by = ["alertname" "instance" "target" "severity"];
group_wait = "30s";
group_interval = "5m";
repeat_interval = "4h";
}
];
};
};
}
hosts/AZ-PRM-1/services/monitoring/blackbox.nix
+99
View File
@@ -0,0 +1,99 @@
# Phase 2 — not active until imported in services/default.nix.
# Activate by adding `./blackbox.nix` to hosts/AZ-PRM-1/services/monitoring/default.nix imports.
{config, ...}: let
blackboxPort = config.m3ta.ports.get "blackbox-exporter";
prometheusPort = config.m3ta.ports.get "prometheus";
in {
services.prometheus.exporters.blackbox = {
enable = true;
port = blackboxPort;
listenAddress = "127.0.0.1";
openFirewall = false;
configFile = (builtins.toFile "blackbox.yml" ''
modules:
http_2xx:
prober: http
timeout: 5s
http_post_2xx:
prober: http
timeout: 5s
http:
method: POST
icmp_ping:
prober: icmp
timeout: 5s
tcp_connect:
prober: tcp
timeout: 5s
'');
};
services.prometheus.scrapeConfigs = [
{
job_name = "blackbox_http";
metrics_path = "/probe";
params.module = ["http_2xx"];
static_configs = [
{
targets = [
"https://g.l.az-gruppe.com"
"https://wf.l.az-gruppe.com"
"https://k.l.az-gruppe.com"
"https://git.az-gruppe.com"
"https://ping.az-gruppe.com"
"https://llm.az-gruppe.com"
"https://r.az-gruppe.com"
];
labels = {
instance = "AZ-PRM-1-blackbox";
};
}
];
relabel_configs = [
{
source_labels = ["__address__"];
target_label = "__param_target";
}
{
source_labels = ["__param_target"];
target_label = "target";
}
{
target_label = "__address__";
replacement = "localhost:${toString blackboxPort}";
}
];
}
{
job_name = "blackbox_icmp";
metrics_path = "/probe";
params.module = ["icmp_ping"];
static_configs = [
{
targets = [
"100.91.203.184" # AZ-CLD-1 netbird
"192.168.152.97" # SMB/DMS share
"192.168.152.98" # SkriptHelper
"192.168.152.102" # legacy PRTG (until decommissioned)
"1.1.1.1" # upstream DNS reachability
"8.8.8.8" # upstream DNS reachability
];
}
];
relabel_configs = [
{
source_labels = ["__address__"];
target_label = "__param_target";
}
{
source_labels = ["__param_target"];
target_label = "target";
}
{
target_label = "__address__";
replacement = "localhost:${toString blackboxPort}";
}
];
}
];
}
hosts/AZ-PRM-1/services/monitoring/cld-scrape.nix
+24
View File
@@ -0,0 +1,24 @@
{config, ...}: let
nodeExporterPort = config.m3ta.ports.get "node-exporter";
cldNetbirdIP = "100.91.203.184";
in {
services.prometheus.scrapeConfigs = [
{
job_name = "node-cld";
static_configs = [
{
targets = ["${cldNetbirdIP}:${toString nodeExporterPort}"];
labels = {
instance = "AZ-CLD-1";
};
}
];
}
];
# Allow CLD to reach PRM prometheus scrapes (prometheus initiates connection TO CLD exporter)
networking.firewall.extraCommands = ''
# No PRM-side firewall change needed: PRM scrapes outbound to CLD:9100.
# CLD-side must allow inbound from 100.91.49.26 (PRM netbird IP) — see CLD config.
'';
}
hosts/AZ-PRM-1/services/monitoring/dashboards/az-blackbox-probe-demo.json
+722
View File
@@ -0,0 +1,722 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"links": [],
"liveNow": false,
"schemaVersion": 39,
"style": "dark",
"tags": [
"az",
"demo",
"provisioned"
],
"templating": {
"list": []
},
"time": {
"from": "now-24h",
"to": "now"
},
"timepicker": {
"refresh_intervals": [
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h"
]
},
"timezone": "browser",
"version": 1,
"weekStart": "",
"uid": "az-blackbox-demo",
"title": "AZ Blackbox Probe Demo",
"refresh": "30s",
"description": "Demo dashboard for HTTP and ICMP probes from Prometheus blackbox_exporter.",
"panels": [
{
"id": 1,
"title": "HTTP probe success",
"type": "stat",
"datasource": null,
"gridPos": {
"h": 5,
"w": 8,
"x": 0,
"y": 0
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {},
"mappings": [],
"unit": "short",
"decimals": 0,
"min": 0,
"max": 1,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 1
}
]
}
},
"overrides": []
},
"options": {
"colorMode": "background",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"targets": [
{
"expr": "probe_success{job=\"blackbox_http\"}",
"legendFormat": "{{target}}",
"refId": "A"
}
],
"description": "1 means the last HTTP probe matched the http_2xx module."
},
{
"id": 2,
"title": "HTTP 24h availability",
"type": "stat",
"datasource": null,
"gridPos": {
"h": 5,
"w": 8,
"x": 8,
"y": 0
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {},
"mappings": [],
"unit": "percent",
"decimals": 2,
"min": 0,
"max": 100,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "yellow",
"value": 95
},
{
"color": "green",
"value": 99
}
]
}
},
"overrides": []
},
"options": {
"colorMode": "background",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"targets": [
{
"expr": "avg_over_time(probe_success{job=\"blackbox_http\"}[24h]) * 100",
"legendFormat": "{{target}}",
"refId": "A"
}
]
},
{
"id": 3,
"title": "TLS cert days left",
"type": "stat",
"datasource": null,
"gridPos": {
"h": 5,
"w": 8,
"x": 16,
"y": 0
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {},
"mappings": [],
"unit": "d",
"decimals": 0,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "yellow",
"value": 14
},
{
"color": "green",
"value": 30
}
]
}
},
"overrides": []
},
"options": {
"colorMode": "background",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"targets": [
{
"expr": "(probe_ssl_earliest_cert_expiry{job=\"blackbox_http\"} - time()) / 86400",
"legendFormat": "{{target}}",
"refId": "A"
}
]
},
{
"id": 4,
"title": "HTTP latency",
"type": "timeseries",
"datasource": null,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 5
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {},
"mappings": [],
"unit": "s",
"decimals": 3,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 1
},
{
"color": "red",
"value": 3
}
]
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "probe_duration_seconds{job=\"blackbox_http\"}",
"legendFormat": "{{target}}",
"refId": "A"
}
]
},
{
"id": 5,
"title": "HTTP status code",
"type": "timeseries",
"datasource": null,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 5
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {},
"mappings": [],
"unit": "short",
"decimals": 0,
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "probe_http_status_code{job=\"blackbox_http\"}",
"legendFormat": "{{target}}",
"refId": "A"
}
]
},
{
"id": 6,
"title": "ICMP probe success",
"type": "stat",
"datasource": null,
"gridPos": {
"h": 5,
"w": 8,
"x": 0,
"y": 13
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {},
"mappings": [],
"unit": "short",
"decimals": 0,
"min": 0,
"max": 1,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 1
}
]
}
},
"overrides": []
},
"options": {
"colorMode": "background",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"targets": [
{
"expr": "probe_success{job=\"blackbox_icmp\"}",
"legendFormat": "{{target}}",
"refId": "A"
}
],
"description": "1 means the last ICMP probe succeeded."
},
{
"id": 7,
"title": "ICMP 24h availability",
"type": "stat",
"datasource": null,
"gridPos": {
"h": 5,
"w": 8,
"x": 8,
"y": 13
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {},
"mappings": [],
"unit": "percent",
"decimals": 2,
"min": 0,
"max": 100,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "yellow",
"value": 95
},
{
"color": "green",
"value": 99
}
]
}
},
"overrides": []
},
"options": {
"colorMode": "background",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"targets": [
{
"expr": "avg_over_time(probe_success{job=\"blackbox_icmp\"}[24h]) * 100",
"legendFormat": "{{target}}",
"refId": "A"
}
]
},
{
"id": 8,
"title": "ICMP last latency",
"type": "stat",
"datasource": null,
"gridPos": {
"h": 5,
"w": 8,
"x": 16,
"y": 13
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {},
"mappings": [],
"unit": "s",
"decimals": 3,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 0.1
},
{
"color": "red",
"value": 0.3
}
]
}
},
"overrides": []
},
"options": {
"colorMode": "background",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"targets": [
{
"expr": "probe_duration_seconds{job=\"blackbox_icmp\"}",
"legendFormat": "{{target}}",
"refId": "A"
}
]
},
{
"id": 9,
"title": "ICMP latency",
"type": "timeseries",
"datasource": null,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 18
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {},
"mappings": [],
"unit": "s",
"decimals": 3,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 0.1
},
{
"color": "red",
"value": 0.3
}
]
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "probe_duration_seconds{job=\"blackbox_icmp\"}",
"legendFormat": "{{target}}",
"refId": "A"
}
]
},
{
"id": 10,
"title": "Probe success timeline",
"type": "timeseries",
"datasource": null,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 18
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {},
"mappings": [],
"unit": "short",
"decimals": 0,
"min": 0,
"max": 1,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 1
}
]
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "probe_success{job=\"blackbox_http\"}",
"legendFormat": "HTTP {{target}}",
"refId": "A"
},
{
"expr": "probe_success{job=\"blackbox_icmp\"}",
"legendFormat": "ICMP {{target}}",
"refId": "B"
}
]
},
{
"id": 11,
"title": "DNS + connect + TLS phase timing",
"type": "timeseries",
"datasource": null,
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 26
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {},
"mappings": [],
"unit": "s",
"decimals": 3,
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "probe_http_duration_seconds{job=\"blackbox_http\", phase=\"resolve\"}",
"legendFormat": "{{target}} resolve",
"refId": "A"
},
{
"expr": "probe_http_duration_seconds{job=\"blackbox_http\", phase=\"connect\"}",
"legendFormat": "{{target}} connect",
"refId": "B"
},
{
"expr": "probe_http_duration_seconds{job=\"blackbox_http\", phase=\"tls\"}",
"legendFormat": "{{target}} tls",
"refId": "C"
},
{
"expr": "probe_http_duration_seconds{job=\"blackbox_http\", phase=\"processing\"}",
"legendFormat": "{{target}} processing",
"refId": "D"
}
]
}
]
}
hosts/AZ-PRM-1/services/monitoring/dashboards/az-node-fleet-demo.json
+731
View File
@@ -0,0 +1,731 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"links": [],
"liveNow": false,
"schemaVersion": 39,
"style": "dark",
"tags": [
"az",
"demo",
"provisioned"
],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {
"refresh_intervals": [
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h"
]
},
"timezone": "browser",
"version": 1,
"weekStart": "",
"uid": "az-node-fleet-demo",
"title": "AZ Node Fleet Demo",
"refresh": "30s",
"description": "Demo dashboard for node_exporter metrics on AZ-PRM-1 and AZ-CLD-1. Uses the default Prometheus datasource provisioned by NixOS.",
"panels": [
{
"id": 1,
"title": "Scrape up",
"type": "stat",
"datasource": null,
"gridPos": {
"h": 4,
"w": 6,
"x": 0,
"y": 0
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {},
"mappings": [],
"unit": "short",
"decimals": 0,
"min": 0,
"max": 1,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 1
}
]
}
},
"overrides": []
},
"options": {
"colorMode": "background",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"targets": [
{
"expr": "up{job=~\"node|node-cld\"}",
"legendFormat": "{{instance}}",
"refId": "A"
}
],
"description": "1 means Prometheus can scrape the host."
},
{
"id": 2,
"title": "Uptime",
"type": "stat",
"datasource": null,
"gridPos": {
"h": 4,
"w": 6,
"x": 6,
"y": 0
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {},
"mappings": [],
"unit": "d",
"decimals": 1,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"colorMode": "background",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"targets": [
{
"expr": "(time() - node_boot_time_seconds) / 86400",
"legendFormat": "{{instance}}",
"refId": "A"
}
]
},
{
"id": 3,
"title": "CPU busy",
"type": "gauge",
"datasource": null,
"gridPos": {
"h": 4,
"w": 6,
"x": 12,
"y": 0
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {},
"mappings": [],
"unit": "percent",
"decimals": 1,
"min": 0,
"max": 100,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 70
},
{
"color": "red",
"value": 90
}
]
}
},
"overrides": []
},
"options": {
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showThresholdLabels": false,
"showThresholdMarkers": true
},
"targets": [
{
"expr": "100 - (avg by(instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval])) * 100)",
"legendFormat": "{{instance}}",
"refId": "A"
}
]
},
{
"id": 4,
"title": "Memory used",
"type": "gauge",
"datasource": null,
"gridPos": {
"h": 4,
"w": 6,
"x": 18,
"y": 0
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {},
"mappings": [],
"unit": "percent",
"decimals": 1,
"min": 0,
"max": 100,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 75
},
{
"color": "red",
"value": 90
}
]
}
},
"overrides": []
},
"options": {
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showThresholdLabels": false,
"showThresholdMarkers": true
},
"targets": [
{
"expr": "100 * (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes))",
"legendFormat": "{{instance}}",
"refId": "A"
}
]
},
{
"id": 5,
"title": "CPU busy by host",
"type": "timeseries",
"datasource": null,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 4
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {},
"mappings": [],
"unit": "percent",
"decimals": 1,
"min": 0,
"max": 100,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 70
},
{
"color": "red",
"value": 90
}
]
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "100 - (avg by(instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval])) * 100)",
"legendFormat": "{{instance}}",
"refId": "A"
}
]
},
{
"id": 6,
"title": "Load average (5m)",
"type": "timeseries",
"datasource": null,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 4
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {},
"mappings": [],
"unit": "short",
"decimals": 2,
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "node_load5",
"legendFormat": "{{instance}}",
"refId": "A"
}
]
},
{
"id": 7,
"title": "Memory used",
"type": "timeseries",
"datasource": null,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 12
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {},
"mappings": [],
"unit": "percent",
"decimals": 1,
"min": 0,
"max": 100,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 75
},
{
"color": "red",
"value": 90
}
]
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "100 * (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes))",
"legendFormat": "{{instance}}",
"refId": "A"
}
]
},
{
"id": 8,
"title": "Root filesystem used",
"type": "timeseries",
"datasource": null,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 12
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {},
"mappings": [],
"unit": "percent",
"decimals": 1,
"min": 0,
"max": 100,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 80
},
{
"color": "red",
"value": 92
}
]
}
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "100 * (1 - (node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay|squashfs|ramfs\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay|squashfs|ramfs\"}))",
"legendFormat": "{{instance}}",
"refId": "A"
}
]
},
{
"id": 9,
"title": "Network throughput",
"type": "timeseries",
"datasource": null,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 20
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {},
"mappings": [],
"unit": "Bps",
"decimals": 1,
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "sum by(instance) (rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|br-.*\"}[$__rate_interval]))",
"legendFormat": "{{instance}} RX",
"refId": "A"
},
{
"expr": "sum by(instance) (rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|br-.*\"}[$__rate_interval]))",
"legendFormat": "{{instance}} TX",
"refId": "B"
}
]
},
{
"id": 10,
"title": "Disk IO",
"type": "timeseries",
"datasource": null,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 20
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {},
"mappings": [],
"unit": "Bps",
"decimals": 1,
"min": 0
},
"overrides": []
},
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "sum by(instance) (rate(node_disk_read_bytes_total[$__rate_interval]))",
"legendFormat": "{{instance}} read",
"refId": "A"
},
{
"expr": "sum by(instance) (rate(node_disk_written_bytes_total[$__rate_interval]))",
"legendFormat": "{{instance}} write",
"refId": "B"
}
]
},
{
"id": 11,
"title": "Failed systemd units",
"type": "stat",
"datasource": null,
"gridPos": {
"h": 4,
"w": 12,
"x": 0,
"y": 28
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {},
"mappings": [],
"unit": "short",
"decimals": 0,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 1
}
]
}
},
"overrides": []
},
"options": {
"colorMode": "background",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"targets": [
{
"expr": "sum by(instance) (node_systemd_units{state=\"failed\"})",
"legendFormat": "{{instance}}",
"refId": "A"
}
],
"description": "Requires node_exporter systemd collector."
},
{
"id": 12,
"title": "Filesystem free bytes",
"type": "stat",
"datasource": null,
"gridPos": {
"h": 4,
"w": 12,
"x": 12,
"y": 28
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {},
"mappings": [],
"unit": "bytes",
"decimals": 1,
"min": 0
},
"overrides": []
},
"options": {
"colorMode": "background",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"targets": [
{
"expr": "node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay|squashfs|ramfs\"}",
"legendFormat": "{{instance}}",
"refId": "A"
}
]
}
]
}
hosts/AZ-PRM-1/services/monitoring/default.nix
+10
View File
@@ -0,0 +1,10 @@
{...}: {
imports = [
./alerting.nix
./blackbox.nix
./cld-scrape.nix
./exporters.nix
./grafana.nix
./prometheus.nix
];
}
hosts/AZ-PRM-1/services/monitoring/exporters.nix
+15
View File
@@ -0,0 +1,15 @@
{config, ...}: let
nodeExporterPort = config.m3ta.ports.get "node-exporter";
in {
services.prometheus.exporters.node = {
enable = true;
port = nodeExporterPort;
listenAddress = "127.0.0.1";
enabledCollectors = [
"systemd"
"diskstats"
"filesystem"
];
openFirewall = false; # localhost only; prometheus scrapes via 127.0.0.1
};
}
hosts/AZ-PRM-1/services/monitoring/grafana.nix
+80
View File
@@ -0,0 +1,80 @@
{config, ...}: let
serviceName = "grafana";
servicePort = config.m3ta.ports.get serviceName;
prometheusPort = config.m3ta.ports.get "prometheus";
in {
services.grafana = {
enable = true;
settings = {
server = {
http_addr = "127.0.0.1";
http_port = servicePort;
domain = "g.l.az-gruppe.com";
root_url = "https://g.l.az-gruppe.com";
serve_from_sub_path = false;
};
database = {
type = "postgres";
host = "127.0.0.1";
name = "grafana";
user = "grafana";
password = "$__file{${config.age.secrets.grafana-db-password.path}}";
ssl_mode = "disable";
};
security = {
admin_user = "admin";
# Grafana file-provider: $__file{<path>} reads the raw secret from the file.
# The agenix secrets must contain ONLY the raw value (no KEY= prefix).
admin_password = "$__file{${config.age.secrets.grafana-admin-pw.path}}";
secret_key = "$__file{${config.age.secrets.grafana-secret-key.path}}";
disable_gravatar = true;
};
};
provision = {
datasources.settings.datasources = [
{
name = "Prometheus";
uid = "prometheus";
type = "prometheus";
url = "http://localhost:${toString prometheusPort}";
isDefault = true;
access = "proxy";
jsonData.timeInterval = "15s";
}
# Phase 3: add Loki datasource here once services.loki is enabled.
];
dashboards.settings.providers = [
{
name = "default";
options.path = "/etc/grafana-dashboards";
}
];
};
};
# Dashboards directory (Phase 1: empty placeholder, JSON files added later)
environment.etc."grafana-dashboards".source = ./. + "/dashboards";
systemd.services.grafana = {
after = ["postgresql.service"];
wants = ["postgresql.service"];
};
# Traefik configuration specific to grafana
services.traefik.dynamicConfigOptions.http = {
services.${serviceName}.loadBalancer.servers = [
{
url = "http://localhost:${toString servicePort}/";
}
];
routers.${serviceName} = {
rule = "Host(`g.l.az-gruppe.com`)";
tls = {
certResolver = "ionos";
};
service = serviceName;
entrypoints = "websecure";
};
};
}
hosts/AZ-PRM-1/services/monitoring/prometheus.nix
+28
View File
@@ -0,0 +1,28 @@
{config, ...}: let
prometheusPort = config.m3ta.ports.get "prometheus";
nodeExporterPort = config.m3ta.ports.get "node-exporter";
in {
services.prometheus = {
enable = true;
port = prometheusPort;
listenAddress = "127.0.0.1";
retentionTime = "30d";
scrapeConfigs = [
{
job_name = "node";
static_configs = [
{
targets = ["localhost:${toString nodeExporterPort}"];
labels = {
instance = "AZ-PRM-1";
};
}
];
}
# Phase 2: add AZ-CLD-1 node target (scrape over netbird 100.x).
# Phase 2: add blackbox_exporter, podman/cadvisor targets.
# Phase 2: enable remote-write receiver (extraFlags) for Windows clients.
];
};
}
hosts/AZ-PRM-1/services/postgres.nix
+8
View File
@@ -31,6 +31,10 @@
CREATE USER vaultwarden WITH ENCRYPTED PASSWORD 'n8n';
CREATE DATABASE vaultwarden;
ALTER DATABASE vaultwarden OWNER to vaultwarden;
CREATE USER grafana WITH ENCRYPTED PASSWORD 'grafana';
CREATE DATABASE grafana;
ALTER DATABASE grafana OWNER to grafana;
'';
authentication = pkgs.lib.mkOverride 10 ''
# Local connections (Unix socket)
@@ -48,6 +52,10 @@
host baserow baserow 10.89.0.0/24 scram-sha-256
host kestra kestra 10.89.0.0/24 scram-sha-256
# Grafana (local socket / localhost only)
host grafana grafana 127.0.0.1/32 scram-sha-256
host grafana grafana ::1/128 scram-sha-256
# Deny all other connections
host all all 0.0.0.0/0 reject
host all all ::/0 reject
hosts/common/ports.nix
+7
View File
@@ -24,6 +24,7 @@
baserow = 3050;
frappe-lms = 3052;
snipe-it = 3053;
azess = 3054;
librechat = 3040;
librechat-dev = 3141;
@@ -41,6 +42,12 @@
postgres = 5432;
pgadmin = 5050;
mysql = 3306;
# Observability stack (AZ-PRM-1)
grafana = 3060;
prometheus = 9090;
node-exporter = 9100;
blackbox-exporter = 9115;
};
hostOverrides = {