feat: prometheus + grafana
This commit is contained in:
@@ -0,0 +1,26 @@
|
||||
{config, ...}: let
|
||||
serviceName = "azess";
|
||||
servicePort = config.m3ta.ports.get serviceName;
|
||||
in {
|
||||
services.azess = {
|
||||
enable = true;
|
||||
host = "127.0.0.1";
|
||||
port = servicePort;
|
||||
workers = 4;
|
||||
};
|
||||
|
||||
services.traefik.dynamicConfigOptions.http = {
|
||||
services.${serviceName}.loadBalancer.servers = [
|
||||
{url = "http://localhost:${toString servicePort}/";}
|
||||
];
|
||||
|
||||
routers.${serviceName} = {
|
||||
rule = "Host(`azess.l.az-gruppe.com`)";
|
||||
tls = {
|
||||
certResolver = "ionos";
|
||||
};
|
||||
service = serviceName;
|
||||
entrypoints = "websecure";
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -2,8 +2,10 @@
|
||||
imports = [
|
||||
./containers
|
||||
./backup-ingest.nix
|
||||
./azess.nix
|
||||
./azion-scheduler.nix
|
||||
./bpi.nix
|
||||
./monitoring
|
||||
./n8n.nix
|
||||
./netbird.nix
|
||||
./pgadmin.nix
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
...
|
||||
}: let
|
||||
prometheusDatasourceUid = "prometheus";
|
||||
ntfySecretAvailable = builtins.hasAttr "ntfy-grafana-webhook" config.age.secrets;
|
||||
|
||||
prometheusQuery = refId: expr: {
|
||||
inherit refId;
|
||||
datasourceUid = prometheusDatasourceUid;
|
||||
relativeTimeRange = {
|
||||
from = 600;
|
||||
to = 0;
|
||||
};
|
||||
model = {
|
||||
datasource = {
|
||||
type = "prometheus";
|
||||
uid = prometheusDatasourceUid;
|
||||
};
|
||||
editorMode = "code";
|
||||
inherit expr refId;
|
||||
hide = false;
|
||||
instant = true;
|
||||
intervalMs = 1000;
|
||||
legendFormat = "__auto";
|
||||
maxDataPoints = 43200;
|
||||
range = false;
|
||||
};
|
||||
};
|
||||
|
||||
thresholdExpression = {
|
||||
refId,
|
||||
expressionRefId,
|
||||
evaluator,
|
||||
}: {
|
||||
inherit refId;
|
||||
datasourceUid = "__expr__";
|
||||
model = {
|
||||
conditions = [
|
||||
{
|
||||
inherit evaluator;
|
||||
operator.type = "and";
|
||||
query.params = [];
|
||||
reducer = {
|
||||
params = [];
|
||||
type = "avg";
|
||||
};
|
||||
type = "query";
|
||||
}
|
||||
];
|
||||
datasource = {
|
||||
name = "Expression";
|
||||
type = "__expr__";
|
||||
uid = "__expr__";
|
||||
};
|
||||
expression = expressionRefId;
|
||||
hide = false;
|
||||
inherit refId;
|
||||
type = "threshold";
|
||||
};
|
||||
};
|
||||
|
||||
mkAlertRule = {
|
||||
uid,
|
||||
title,
|
||||
expr,
|
||||
for ? "5m",
|
||||
noDataState ? "Alerting",
|
||||
execErrState ? "Error",
|
||||
evaluatorType ? "gt",
|
||||
evaluatorParams ? [0 0],
|
||||
labels ? {},
|
||||
annotations ? {},
|
||||
}: {
|
||||
inherit uid title for noDataState execErrState;
|
||||
condition = "B";
|
||||
data = [
|
||||
(prometheusQuery "A" expr)
|
||||
(thresholdExpression {
|
||||
refId = "B";
|
||||
expressionRefId = "A";
|
||||
evaluator = {
|
||||
type = evaluatorType;
|
||||
params = evaluatorParams;
|
||||
};
|
||||
})
|
||||
];
|
||||
annotations =
|
||||
{
|
||||
summary = title;
|
||||
}
|
||||
// annotations;
|
||||
labels =
|
||||
{
|
||||
service = "monitoring";
|
||||
}
|
||||
// labels;
|
||||
isPaused = false;
|
||||
};
|
||||
in {
|
||||
warnings = lib.optional (!ntfySecretAvailable) ''
|
||||
Grafana alert rules are provisioned, but ntfy notifications are disabled because secrets/ntfy-grafana-webhook.age is missing.
|
||||
Create it as an EnvironmentFile containing: GRAFANA_NTFY_WEBHOOK_URL=https://<ntfy-webhook-url>
|
||||
'';
|
||||
|
||||
systemd.services.grafana.serviceConfig.EnvironmentFile = lib.mkIf ntfySecretAvailable [
|
||||
config.age.secrets."ntfy-grafana-webhook".path
|
||||
];
|
||||
|
||||
services.grafana.provision.alerting = {
|
||||
rules.settings = {
|
||||
apiVersion = 1;
|
||||
groups = [
|
||||
{
|
||||
orgId = 1;
|
||||
name = "az-infrastructure";
|
||||
folder = "Infrastructure";
|
||||
interval = "60s";
|
||||
rules = [
|
||||
(mkAlertRule {
|
||||
uid = "az_host_down";
|
||||
title = "Host down";
|
||||
expr = ''up{job=~"node|node-cld"}'';
|
||||
for = "2m";
|
||||
evaluatorType = "lt";
|
||||
evaluatorParams = [1 0];
|
||||
labels.severity = "critical";
|
||||
annotations.description = "Prometheus cannot scrape a node_exporter target.";
|
||||
})
|
||||
(mkAlertRule {
|
||||
uid = "az_cpu_high";
|
||||
title = "CPU usage high";
|
||||
expr = ''100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)'';
|
||||
for = "10m";
|
||||
evaluatorType = "gt";
|
||||
evaluatorParams = [90 0];
|
||||
labels.severity = "warning";
|
||||
annotations.description = "Average CPU usage has been above 90% for 10 minutes.";
|
||||
})
|
||||
(mkAlertRule {
|
||||
uid = "az_memory_high";
|
||||
title = "Memory usage high";
|
||||
expr = ''100 * (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes))'';
|
||||
for = "10m";
|
||||
evaluatorType = "gt";
|
||||
evaluatorParams = [90 0];
|
||||
labels.severity = "warning";
|
||||
annotations.description = "Memory usage has been above 90% for 10 minutes.";
|
||||
})
|
||||
(mkAlertRule {
|
||||
uid = "az_rootfs_high";
|
||||
title = "Root filesystem usage high";
|
||||
expr = ''100 * (1 - (node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs|ramfs"}))'';
|
||||
for = "15m";
|
||||
evaluatorType = "gt";
|
||||
evaluatorParams = [90 0];
|
||||
labels.severity = "warning";
|
||||
annotations.description = "Root filesystem usage has been above 90% for 15 minutes.";
|
||||
})
|
||||
(mkAlertRule {
|
||||
uid = "az_systemd_failed";
|
||||
title = "Systemd units failed";
|
||||
expr = ''sum by(instance) (node_systemd_units{state="failed"})'';
|
||||
for = "5m";
|
||||
evaluatorType = "gt";
|
||||
evaluatorParams = [0 0];
|
||||
labels.severity = "warning";
|
||||
annotations.description = "One or more systemd units are failed on the host.";
|
||||
})
|
||||
(mkAlertRule {
|
||||
uid = "az_http_probe_failed";
|
||||
title = "HTTP probe failed";
|
||||
expr = ''probe_success{job="blackbox_http"}'';
|
||||
for = "2m";
|
||||
evaluatorType = "lt";
|
||||
evaluatorParams = [1 0];
|
||||
labels.severity = "critical";
|
||||
annotations.description = "A blackbox HTTP probe is failing.";
|
||||
})
|
||||
(mkAlertRule {
|
||||
uid = "az_icmp_probe_failed";
|
||||
title = "ICMP probe failed";
|
||||
expr = ''probe_success{job="blackbox_icmp"}'';
|
||||
for = "2m";
|
||||
evaluatorType = "lt";
|
||||
evaluatorParams = [1 0];
|
||||
labels.severity = "warning";
|
||||
annotations.description = "A blackbox ICMP probe is failing.";
|
||||
})
|
||||
(mkAlertRule {
|
||||
uid = "az_tls_cert_expiring";
|
||||
title = "TLS certificate expires soon";
|
||||
expr = ''(probe_ssl_earliest_cert_expiry{job="blackbox_http"} - time()) / 86400'';
|
||||
for = "1h";
|
||||
noDataState = "OK";
|
||||
evaluatorType = "lt";
|
||||
evaluatorParams = [14 0];
|
||||
labels.severity = "warning";
|
||||
annotations.description = "A probed TLS certificate expires in less than 14 days.";
|
||||
})
|
||||
];
|
||||
}
|
||||
];
|
||||
};
|
||||
|
||||
contactPoints.settings = lib.mkIf ntfySecretAvailable {
|
||||
apiVersion = 1;
|
||||
contactPoints = [
|
||||
{
|
||||
orgId = 1;
|
||||
name = "ntfy";
|
||||
receivers = [
|
||||
{
|
||||
uid = "ntfy-webhook";
|
||||
type = "webhook";
|
||||
disableResolveMessage = false;
|
||||
settings = {
|
||||
url = "$GRAFANA_NTFY_WEBHOOK_URL";
|
||||
httpMethod = "POST";
|
||||
};
|
||||
}
|
||||
];
|
||||
}
|
||||
];
|
||||
};
|
||||
|
||||
policies.settings = lib.mkIf ntfySecretAvailable {
|
||||
apiVersion = 1;
|
||||
policies = [
|
||||
{
|
||||
orgId = 1;
|
||||
receiver = "ntfy";
|
||||
group_by = ["alertname" "instance" "target" "severity"];
|
||||
group_wait = "30s";
|
||||
group_interval = "5m";
|
||||
repeat_interval = "4h";
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
# Phase 2 — not active until imported in services/default.nix.
|
||||
# Activate by adding `./blackbox.nix` to hosts/AZ-PRM-1/services/monitoring/default.nix imports.
|
||||
{config, ...}: let
|
||||
blackboxPort = config.m3ta.ports.get "blackbox-exporter";
|
||||
prometheusPort = config.m3ta.ports.get "prometheus";
|
||||
in {
|
||||
services.prometheus.exporters.blackbox = {
|
||||
enable = true;
|
||||
port = blackboxPort;
|
||||
listenAddress = "127.0.0.1";
|
||||
openFirewall = false;
|
||||
configFile = (builtins.toFile "blackbox.yml" ''
|
||||
modules:
|
||||
http_2xx:
|
||||
prober: http
|
||||
timeout: 5s
|
||||
http_post_2xx:
|
||||
prober: http
|
||||
timeout: 5s
|
||||
http:
|
||||
method: POST
|
||||
icmp_ping:
|
||||
prober: icmp
|
||||
timeout: 5s
|
||||
tcp_connect:
|
||||
prober: tcp
|
||||
timeout: 5s
|
||||
'');
|
||||
};
|
||||
|
||||
services.prometheus.scrapeConfigs = [
|
||||
{
|
||||
job_name = "blackbox_http";
|
||||
metrics_path = "/probe";
|
||||
params.module = ["http_2xx"];
|
||||
static_configs = [
|
||||
{
|
||||
targets = [
|
||||
"https://g.l.az-gruppe.com"
|
||||
"https://wf.l.az-gruppe.com"
|
||||
"https://k.l.az-gruppe.com"
|
||||
"https://git.az-gruppe.com"
|
||||
"https://ping.az-gruppe.com"
|
||||
"https://llm.az-gruppe.com"
|
||||
"https://r.az-gruppe.com"
|
||||
];
|
||||
labels = {
|
||||
instance = "AZ-PRM-1-blackbox";
|
||||
};
|
||||
}
|
||||
];
|
||||
relabel_configs = [
|
||||
{
|
||||
source_labels = ["__address__"];
|
||||
target_label = "__param_target";
|
||||
}
|
||||
{
|
||||
source_labels = ["__param_target"];
|
||||
target_label = "target";
|
||||
}
|
||||
{
|
||||
target_label = "__address__";
|
||||
replacement = "localhost:${toString blackboxPort}";
|
||||
}
|
||||
];
|
||||
}
|
||||
{
|
||||
job_name = "blackbox_icmp";
|
||||
metrics_path = "/probe";
|
||||
params.module = ["icmp_ping"];
|
||||
static_configs = [
|
||||
{
|
||||
targets = [
|
||||
"100.91.203.184" # AZ-CLD-1 netbird
|
||||
"192.168.152.97" # SMB/DMS share
|
||||
"192.168.152.98" # SkriptHelper
|
||||
"192.168.152.102" # legacy PRTG (until decommissioned)
|
||||
"1.1.1.1" # upstream DNS reachability
|
||||
"8.8.8.8" # upstream DNS reachability
|
||||
];
|
||||
}
|
||||
];
|
||||
relabel_configs = [
|
||||
{
|
||||
source_labels = ["__address__"];
|
||||
target_label = "__param_target";
|
||||
}
|
||||
{
|
||||
source_labels = ["__param_target"];
|
||||
target_label = "target";
|
||||
}
|
||||
{
|
||||
target_label = "__address__";
|
||||
replacement = "localhost:${toString blackboxPort}";
|
||||
}
|
||||
];
|
||||
}
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
{config, ...}: let
|
||||
nodeExporterPort = config.m3ta.ports.get "node-exporter";
|
||||
cldNetbirdIP = "100.91.203.184";
|
||||
in {
|
||||
services.prometheus.scrapeConfigs = [
|
||||
{
|
||||
job_name = "node-cld";
|
||||
static_configs = [
|
||||
{
|
||||
targets = ["${cldNetbirdIP}:${toString nodeExporterPort}"];
|
||||
labels = {
|
||||
instance = "AZ-CLD-1";
|
||||
};
|
||||
}
|
||||
];
|
||||
}
|
||||
];
|
||||
|
||||
# Allow CLD to reach PRM prometheus scrapes (prometheus initiates connection TO CLD exporter)
|
||||
networking.firewall.extraCommands = ''
|
||||
# No PRM-side firewall change needed: PRM scrapes outbound to CLD:9100.
|
||||
# CLD-side must allow inbound from 100.91.49.26 (PRM netbird IP) — see CLD config.
|
||||
'';
|
||||
}
|
||||
@@ -0,0 +1,722 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"target": {
|
||||
"limit": 100,
|
||||
"matchAny": false,
|
||||
"tags": [],
|
||||
"type": "dashboard"
|
||||
},
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
"links": [],
|
||||
"liveNow": false,
|
||||
"schemaVersion": 39,
|
||||
"style": "dark",
|
||||
"tags": [
|
||||
"az",
|
||||
"demo",
|
||||
"provisioned"
|
||||
],
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"time": {
|
||||
"from": "now-24h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {
|
||||
"refresh_intervals": [
|
||||
"10s",
|
||||
"30s",
|
||||
"1m",
|
||||
"5m",
|
||||
"15m",
|
||||
"30m",
|
||||
"1h"
|
||||
]
|
||||
},
|
||||
"timezone": "browser",
|
||||
"version": 1,
|
||||
"weekStart": "",
|
||||
"uid": "az-blackbox-demo",
|
||||
"title": "AZ Blackbox Probe Demo",
|
||||
"refresh": "30s",
|
||||
"description": "Demo dashboard for HTTP and ICMP probes from Prometheus blackbox_exporter.",
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "HTTP probe success",
|
||||
"type": "stat",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 5,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "short",
|
||||
"decimals": 0,
|
||||
"min": 0,
|
||||
"max": 1,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "red",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "green",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "probe_success{job=\"blackbox_http\"}",
|
||||
"legendFormat": "{{target}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"description": "1 means the last HTTP probe matched the http_2xx module."
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "HTTP 24h availability",
|
||||
"type": "stat",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 5,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 0
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "percent",
|
||||
"decimals": 2,
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "red",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 95
|
||||
},
|
||||
{
|
||||
"color": "green",
|
||||
"value": 99
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "avg_over_time(probe_success{job=\"blackbox_http\"}[24h]) * 100",
|
||||
"legendFormat": "{{target}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "TLS cert days left",
|
||||
"type": "stat",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 5,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 0
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "d",
|
||||
"decimals": 0,
|
||||
"min": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "red",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 14
|
||||
},
|
||||
{
|
||||
"color": "green",
|
||||
"value": 30
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(probe_ssl_earliest_cert_expiry{job=\"blackbox_http\"} - time()) / 86400",
|
||||
"legendFormat": "{{target}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "HTTP latency",
|
||||
"type": "timeseries",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 5
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "s",
|
||||
"decimals": 3,
|
||||
"min": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 1
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 3
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "probe_duration_seconds{job=\"blackbox_http\"}",
|
||||
"legendFormat": "{{target}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "HTTP status code",
|
||||
"type": "timeseries",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 5
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "short",
|
||||
"decimals": 0,
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "probe_http_status_code{job=\"blackbox_http\"}",
|
||||
"legendFormat": "{{target}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "ICMP probe success",
|
||||
"type": "stat",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 5,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 13
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "short",
|
||||
"decimals": 0,
|
||||
"min": 0,
|
||||
"max": 1,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "red",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "green",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "probe_success{job=\"blackbox_icmp\"}",
|
||||
"legendFormat": "{{target}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"description": "1 means the last ICMP probe succeeded."
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "ICMP 24h availability",
|
||||
"type": "stat",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 5,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 13
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "percent",
|
||||
"decimals": 2,
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "red",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 95
|
||||
},
|
||||
{
|
||||
"color": "green",
|
||||
"value": 99
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "avg_over_time(probe_success{job=\"blackbox_icmp\"}[24h]) * 100",
|
||||
"legendFormat": "{{target}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"title": "ICMP last latency",
|
||||
"type": "stat",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 5,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 13
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "s",
|
||||
"decimals": 3,
|
||||
"min": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 0.1
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 0.3
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "probe_duration_seconds{job=\"blackbox_icmp\"}",
|
||||
"legendFormat": "{{target}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"title": "ICMP latency",
|
||||
"type": "timeseries",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 18
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "s",
|
||||
"decimals": 3,
|
||||
"min": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 0.1
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 0.3
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "probe_duration_seconds{job=\"blackbox_icmp\"}",
|
||||
"legendFormat": "{{target}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"title": "Probe success timeline",
|
||||
"type": "timeseries",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 18
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "short",
|
||||
"decimals": 0,
|
||||
"min": 0,
|
||||
"max": 1,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "red",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "green",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "probe_success{job=\"blackbox_http\"}",
|
||||
"legendFormat": "HTTP {{target}}",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "probe_success{job=\"blackbox_icmp\"}",
|
||||
"legendFormat": "ICMP {{target}}",
|
||||
"refId": "B"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"title": "DNS + connect + TLS phase timing",
|
||||
"type": "timeseries",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 26
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "s",
|
||||
"decimals": 3,
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "probe_http_duration_seconds{job=\"blackbox_http\", phase=\"resolve\"}",
|
||||
"legendFormat": "{{target}} resolve",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "probe_http_duration_seconds{job=\"blackbox_http\", phase=\"connect\"}",
|
||||
"legendFormat": "{{target}} connect",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "probe_http_duration_seconds{job=\"blackbox_http\", phase=\"tls\"}",
|
||||
"legendFormat": "{{target}} tls",
|
||||
"refId": "C"
|
||||
},
|
||||
{
|
||||
"expr": "probe_http_duration_seconds{job=\"blackbox_http\", phase=\"processing\"}",
|
||||
"legendFormat": "{{target}} processing",
|
||||
"refId": "D"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,731 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"target": {
|
||||
"limit": 100,
|
||||
"matchAny": false,
|
||||
"tags": [],
|
||||
"type": "dashboard"
|
||||
},
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
"links": [],
|
||||
"liveNow": false,
|
||||
"schemaVersion": 39,
|
||||
"style": "dark",
|
||||
"tags": [
|
||||
"az",
|
||||
"demo",
|
||||
"provisioned"
|
||||
],
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"time": {
|
||||
"from": "now-6h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {
|
||||
"refresh_intervals": [
|
||||
"10s",
|
||||
"30s",
|
||||
"1m",
|
||||
"5m",
|
||||
"15m",
|
||||
"30m",
|
||||
"1h"
|
||||
]
|
||||
},
|
||||
"timezone": "browser",
|
||||
"version": 1,
|
||||
"weekStart": "",
|
||||
"uid": "az-node-fleet-demo",
|
||||
"title": "AZ Node Fleet Demo",
|
||||
"refresh": "30s",
|
||||
"description": "Demo dashboard for node_exporter metrics on AZ-PRM-1 and AZ-CLD-1. Uses the default Prometheus datasource provisioned by NixOS.",
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "Scrape up",
|
||||
"type": "stat",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "short",
|
||||
"decimals": 0,
|
||||
"min": 0,
|
||||
"max": 1,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "red",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "green",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "up{job=~\"node|node-cld\"}",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"description": "1 means Prometheus can scrape the host."
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "Uptime",
|
||||
"type": "stat",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 6,
|
||||
"y": 0
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "d",
|
||||
"decimals": 1,
|
||||
"min": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(time() - node_boot_time_seconds) / 86400",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "CPU busy",
|
||||
"type": "gauge",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 12,
|
||||
"y": 0
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "percent",
|
||||
"decimals": 1,
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 70
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 90
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showThresholdLabels": false,
|
||||
"showThresholdMarkers": true
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "100 - (avg by(instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval])) * 100)",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "Memory used",
|
||||
"type": "gauge",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 18,
|
||||
"y": 0
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "percent",
|
||||
"decimals": 1,
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 75
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 90
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showThresholdLabels": false,
|
||||
"showThresholdMarkers": true
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "100 * (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes))",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "CPU busy by host",
|
||||
"type": "timeseries",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 4
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "percent",
|
||||
"decimals": 1,
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 70
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 90
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "100 - (avg by(instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval])) * 100)",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "Load average (5m)",
|
||||
"type": "timeseries",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 4
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "short",
|
||||
"decimals": 2,
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "node_load5",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "Memory used",
|
||||
"type": "timeseries",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 12
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "percent",
|
||||
"decimals": 1,
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 75
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 90
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "100 * (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes))",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"title": "Root filesystem used",
|
||||
"type": "timeseries",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 12
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "percent",
|
||||
"decimals": 1,
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 80
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 92
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "100 * (1 - (node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay|squashfs|ramfs\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay|squashfs|ramfs\"}))",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"title": "Network throughput",
|
||||
"type": "timeseries",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 20
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "Bps",
|
||||
"decimals": 1,
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by(instance) (rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|br-.*\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{instance}} RX",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "sum by(instance) (rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|br-.*\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{instance}} TX",
|
||||
"refId": "B"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"title": "Disk IO",
|
||||
"type": "timeseries",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 20
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "Bps",
|
||||
"decimals": 1,
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by(instance) (rate(node_disk_read_bytes_total[$__rate_interval]))",
|
||||
"legendFormat": "{{instance}} read",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "sum by(instance) (rate(node_disk_written_bytes_total[$__rate_interval]))",
|
||||
"legendFormat": "{{instance}} write",
|
||||
"refId": "B"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"title": "Failed systemd units",
|
||||
"type": "stat",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 28
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "short",
|
||||
"decimals": 0,
|
||||
"min": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by(instance) (node_systemd_units{state=\"failed\"})",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"description": "Requires node_exporter systemd collector."
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"title": "Filesystem free bytes",
|
||||
"type": "stat",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 28
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "bytes",
|
||||
"decimals": 1,
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay|squashfs|ramfs\"}",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
{...}: {
|
||||
imports = [
|
||||
./alerting.nix
|
||||
./blackbox.nix
|
||||
./cld-scrape.nix
|
||||
./exporters.nix
|
||||
./grafana.nix
|
||||
./prometheus.nix
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{config, ...}: let
|
||||
nodeExporterPort = config.m3ta.ports.get "node-exporter";
|
||||
in {
|
||||
services.prometheus.exporters.node = {
|
||||
enable = true;
|
||||
port = nodeExporterPort;
|
||||
listenAddress = "127.0.0.1";
|
||||
enabledCollectors = [
|
||||
"systemd"
|
||||
"diskstats"
|
||||
"filesystem"
|
||||
];
|
||||
openFirewall = false; # localhost only; prometheus scrapes via 127.0.0.1
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
{config, ...}: let
|
||||
serviceName = "grafana";
|
||||
servicePort = config.m3ta.ports.get serviceName;
|
||||
prometheusPort = config.m3ta.ports.get "prometheus";
|
||||
in {
|
||||
services.grafana = {
|
||||
enable = true;
|
||||
settings = {
|
||||
server = {
|
||||
http_addr = "127.0.0.1";
|
||||
http_port = servicePort;
|
||||
domain = "g.l.az-gruppe.com";
|
||||
root_url = "https://g.l.az-gruppe.com";
|
||||
serve_from_sub_path = false;
|
||||
};
|
||||
database = {
|
||||
type = "postgres";
|
||||
host = "127.0.0.1";
|
||||
name = "grafana";
|
||||
user = "grafana";
|
||||
password = "$__file{${config.age.secrets.grafana-db-password.path}}";
|
||||
ssl_mode = "disable";
|
||||
};
|
||||
security = {
|
||||
admin_user = "admin";
|
||||
# Grafana file-provider: $__file{<path>} reads the raw secret from the file.
|
||||
# The agenix secrets must contain ONLY the raw value (no KEY= prefix).
|
||||
admin_password = "$__file{${config.age.secrets.grafana-admin-pw.path}}";
|
||||
secret_key = "$__file{${config.age.secrets.grafana-secret-key.path}}";
|
||||
disable_gravatar = true;
|
||||
};
|
||||
};
|
||||
provision = {
|
||||
datasources.settings.datasources = [
|
||||
{
|
||||
name = "Prometheus";
|
||||
uid = "prometheus";
|
||||
type = "prometheus";
|
||||
url = "http://localhost:${toString prometheusPort}";
|
||||
isDefault = true;
|
||||
access = "proxy";
|
||||
jsonData.timeInterval = "15s";
|
||||
}
|
||||
# Phase 3: add Loki datasource here once services.loki is enabled.
|
||||
];
|
||||
dashboards.settings.providers = [
|
||||
{
|
||||
name = "default";
|
||||
options.path = "/etc/grafana-dashboards";
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
# Dashboards directory (Phase 1: empty placeholder, JSON files added later)
|
||||
environment.etc."grafana-dashboards".source = ./. + "/dashboards";
|
||||
|
||||
systemd.services.grafana = {
|
||||
after = ["postgresql.service"];
|
||||
wants = ["postgresql.service"];
|
||||
};
|
||||
|
||||
# Traefik configuration specific to grafana
|
||||
services.traefik.dynamicConfigOptions.http = {
|
||||
services.${serviceName}.loadBalancer.servers = [
|
||||
{
|
||||
url = "http://localhost:${toString servicePort}/";
|
||||
}
|
||||
];
|
||||
|
||||
routers.${serviceName} = {
|
||||
rule = "Host(`g.l.az-gruppe.com`)";
|
||||
tls = {
|
||||
certResolver = "ionos";
|
||||
};
|
||||
service = serviceName;
|
||||
entrypoints = "websecure";
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
{config, ...}: let
|
||||
prometheusPort = config.m3ta.ports.get "prometheus";
|
||||
nodeExporterPort = config.m3ta.ports.get "node-exporter";
|
||||
in {
|
||||
services.prometheus = {
|
||||
enable = true;
|
||||
port = prometheusPort;
|
||||
listenAddress = "127.0.0.1";
|
||||
retentionTime = "30d";
|
||||
|
||||
scrapeConfigs = [
|
||||
{
|
||||
job_name = "node";
|
||||
static_configs = [
|
||||
{
|
||||
targets = ["localhost:${toString nodeExporterPort}"];
|
||||
labels = {
|
||||
instance = "AZ-PRM-1";
|
||||
};
|
||||
}
|
||||
];
|
||||
}
|
||||
# Phase 2: add AZ-CLD-1 node target (scrape over netbird 100.x).
|
||||
# Phase 2: add blackbox_exporter, podman/cadvisor targets.
|
||||
# Phase 2: enable remote-write receiver (extraFlags) for Windows clients.
|
||||
];
|
||||
};
|
||||
}
|
||||
@@ -31,6 +31,10 @@
|
||||
CREATE USER vaultwarden WITH ENCRYPTED PASSWORD 'n8n';
|
||||
CREATE DATABASE vaultwarden;
|
||||
ALTER DATABASE vaultwarden OWNER to vaultwarden;
|
||||
|
||||
CREATE USER grafana WITH ENCRYPTED PASSWORD 'grafana';
|
||||
CREATE DATABASE grafana;
|
||||
ALTER DATABASE grafana OWNER to grafana;
|
||||
'';
|
||||
authentication = pkgs.lib.mkOverride 10 ''
|
||||
# Local connections (Unix socket)
|
||||
@@ -48,6 +52,10 @@
|
||||
host baserow baserow 10.89.0.0/24 scram-sha-256
|
||||
host kestra kestra 10.89.0.0/24 scram-sha-256
|
||||
|
||||
# Grafana (local socket / localhost only)
|
||||
host grafana grafana 127.0.0.1/32 scram-sha-256
|
||||
host grafana grafana ::1/128 scram-sha-256
|
||||
|
||||
# Deny all other connections
|
||||
host all all 0.0.0.0/0 reject
|
||||
host all all ::/0 reject
|
||||
|
||||
Reference in New Issue
Block a user