AZ-Intec-GmbH/AZ-NIX-SERVERS
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: grafana loki

Browse Source
This commit is contained in:
Sascha König
2026-07-10 13:10:58 +02:00
parent b800bfe08b
commit 8d57aa4bc0
16 changed files with 1095 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hosts/AZ-PRM-1/services/monitoring/alloy.nix
+92
View File
@@ -0,0 +1,92 @@
{
config,
lib,
...
}: let
lokiPort = config.m3ta.ports.get "loki";
alloyPort = config.m3ta.ports.get "alloy";
in {
services.alloy = {
enable = true;
configPath = "/etc/alloy";
extraFlags = [
"--server.http.listen-addr=127.0.0.1:${toString alloyPort}"
"--disable-reporting"
];
};
environment.etc."alloy/config.alloy".text = ''
loki.relabel "journal" {
forward_to = []
rule {
source_labels = ["__journal__systemd_unit"]
target_label = "unit"
}
rule {
source_labels = ["__journal_priority_keyword"]
target_label = "level"
}
rule {
source_labels = ["__journal_syslog_identifier"]
target_label = "syslog_identifier"
}
}
loki.source.journal "system" {
forward_to = [loki.process.journal.receiver]
relabel_rules = loki.relabel.journal.rules
labels = {
host = "AZ-PRM-1",
environment = "prod",
}
}
loki.process "journal" {
forward_to = [loki.write.local.receiver]
stage.json {
expressions = {
app = "app",
service = "service",
level = "level",
trace_id = "trace_id",
session_id = "session_id",
n8n_execution_id = "execution_id",
workflow_id = "workflow_id",
execution_id = "execution_id",
message = "message",
}
drop_malformed = false
}
stage.labels {
values = {
app = "",
service = "",
level = "",
}
}
stage.structured_metadata {
values = {
trace_id = "",
session_id = "",
n8n_execution_id = "",
workflow_id = "",
execution_id = "",
}
}
}
loki.write "local" {
endpoint {
url = "http://127.0.0.1:${toString lokiPort}/loki/api/v1/push"
}
}
'';
systemd.services.alloy.serviceConfig.SupplementaryGroups = lib.mkAfter ["adm"];
}
hosts/AZ-PRM-1/services/monitoring/blackbox.nix
+2 -2
View File
@@ -9,7 +9,7 @@ in {
port = blackboxPort;
listenAddress = "127.0.0.1";
openFirewall = false;
configFile = (builtins.toFile "blackbox.yml" ''
configFile = builtins.toFile "blackbox.yml" ''
modules:
http_2xx:
prober: http
@@ -25,7 +25,7 @@ in {
tcp_connect:
prober: tcp
timeout: 5s
'');
'';
};
services.prometheus.scrapeConfigs = [
hosts/AZ-PRM-1/services/monitoring/dashboards/az-loki-usage-demo.json
+764
View File
@@ -0,0 +1,764 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"links": [],
"liveNow": false,
"schemaVersion": 39,
"style": "dark",
"tags": [
"az",
"demo",
"loki",
"logs",
"provisioned"
],
"templating": {
"list": [
{
"name": "host",
"label": "Host",
"type": "query",
"datasource": {
"type": "loki",
"uid": "loki"
},
"definition": "label_values(host)",
"query": "label_values(host)",
"refresh": 1,
"sort": 1,
"includeAll": true,
"multi": true,
"allValue": ".+",
"current": {
"selected": true,
"text": "All",
"value": "$__all"
},
"hide": 0,
"skipUrlSync": false
},
{
"name": "unit",
"label": "Systemd Unit",
"type": "query",
"datasource": {
"type": "loki",
"uid": "loki"
},
"definition": "label_values({host=~\"$host\"}, unit)",
"query": "label_values({host=~\"$host\"}, unit)",
"refresh": 1,
"sort": 1,
"includeAll": true,
"multi": true,
"allValue": ".*",
"current": {
"selected": true,
"text": "All",
"value": "$__all"
},
"hide": 0,
"skipUrlSync": false
},
{
"name": "app",
"label": "App",
"type": "query",
"datasource": {
"type": "loki",
"uid": "loki"
},
"definition": "label_values({host=~\"$host\", unit=~\"$unit\"}, app)",
"query": "label_values({host=~\"$host\", unit=~\"$unit\"}, app)",
"refresh": 1,
"sort": 1,
"includeAll": true,
"multi": true,
"allValue": ".*",
"current": {
"selected": true,
"text": "All",
"value": "$__all"
},
"hide": 0,
"skipUrlSync": false
},
{
"name": "level",
"label": "Level",
"type": "query",
"datasource": {
"type": "loki",
"uid": "loki"
},
"definition": "label_values(level)",
"query": "label_values(level)",
"refresh": 1,
"sort": 1,
"includeAll": true,
"multi": true,
"allValue": ".+",
"current": {
"selected": true,
"text": "All",
"value": "$__all"
},
"hide": 0,
"skipUrlSync": false
}
]
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {
"refresh_intervals": [
"10s",
"30s",
"1m",
"5m",
"15m",
"30m",
"1h"
]
},
"timezone": "browser",
"version": 1,
"weekStart": "",
"uid": "az-loki-usage-demo",
"title": "AZ Loki Usage Demo",
"refresh": "30s",
"description": "Provisioned demo dashboard for Loki ingestion health, log volume, noisy units/apps, and recent warnings/errors from Alloy journald streams.",
"panels": [
{
"id": 1,
"title": "Log ingest rate",
"type": "stat",
"datasource": {
"type": "loki",
"uid": "loki"
},
"gridPos": {
"h": 4,
"w": 6,
"x": 0,
"y": 0
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {},
"mappings": [],
"unit": "logs/s",
"decimals": 2,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 100
},
{
"color": "red",
"value": 500
}
]
}
},
"overrides": []
},
"options": {
"colorMode": "background",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "sum(rate({host=~\"$host\", unit=~\"$unit\", app=~\"$app\"}[5m]))",
"legendFormat": "logs/sec",
"queryType": "range",
"refId": "A"
}
],
"description": "Total Loki log ingestion rate for the selected hosts."
},
{
"id": 2,
"title": "Warn/Error rate",
"type": "stat",
"datasource": {
"type": "loki",
"uid": "loki"
},
"gridPos": {
"h": 4,
"w": 6,
"x": 6,
"y": 0
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {},
"mappings": [],
"unit": "logs/s",
"decimals": 2,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 0.1
},
{
"color": "red",
"value": 1
}
]
}
},
"overrides": []
},
"options": {
"colorMode": "background",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "sum(rate({host=~\"$host\", unit=~\"$unit\", app=~\"$app\", level=~\"warning|err|error|crit|alert|emerg\"}[5m]))",
"legendFormat": "warn+error/sec",
"queryType": "range",
"refId": "A"
}
],
"description": "Rate of warning and higher priority logs."
},
{
"id": 3,
"title": "Logs last hour",
"type": "stat",
"datasource": {
"type": "loki",
"uid": "loki"
},
"gridPos": {
"h": 4,
"w": 6,
"x": 12,
"y": 0
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {},
"mappings": [],
"unit": "short",
"decimals": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"colorMode": "background",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "sum(count_over_time({host=~\"$host\", unit=~\"$unit\", app=~\"$app\"}[1h]))",
"legendFormat": "logs",
"queryType": "range",
"refId": "A"
}
],
"description": "Total log lines ingested in the last hour."
},
{
"id": 4,
"title": "Noisy units",
"type": "stat",
"datasource": {
"type": "loki",
"uid": "loki"
},
"gridPos": {
"h": 4,
"w": 6,
"x": 18,
"y": 0
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {},
"mappings": [],
"unit": "short",
"decimals": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"colorMode": "background",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto"
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "count(topk(10, sum by (unit) (count_over_time({host=~\"$host\", unit=~\"$unit\", app=~\"$app\"}[1h]))))",
"legendFormat": "units",
"queryType": "range",
"refId": "A"
}
],
"description": "How many systemd units are present in the top-10 noisy-unit set."
},
{
"id": 5,
"title": "Log rate by host",
"type": "timeseries",
"datasource": {
"type": "loki",
"uid": "loki"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 4
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {},
"mappings": [],
"unit": "logs/s",
"decimals": 2,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "sum by (host) (rate({host=~\"$host\", unit=~\"$unit\", app=~\"$app\"}[5m]))",
"legendFormat": "{{host}}",
"queryType": "range",
"refId": "A"
}
],
"description": "Compares PRM and CLD log volume over time."
},
{
"id": 6,
"title": "Warnings/errors by host and level",
"type": "timeseries",
"datasource": {
"type": "loki",
"uid": "loki"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 4
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {},
"mappings": [],
"unit": "logs/s",
"decimals": 2,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "desc"
}
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "sum by (host, level) (rate({host=~\"$host\", unit=~\"$unit\", app=~\"$app\", level=~\"warning|err|error|crit|alert|emerg\"}[5m]))",
"legendFormat": "{{host}} {{level}}",
"queryType": "range",
"refId": "A"
}
],
"description": "Highlights noisy warning/error streams before they become incidents."
},
{
"id": 7,
"title": "Top systemd units by log volume (1h)",
"type": "table",
"datasource": {
"type": "loki",
"uid": "loki"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 12
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {},
"mappings": [],
"unit": "short",
"decimals": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"cellHeight": "sm",
"footer": {
"countRows": false,
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "topk(10, sum by (unit) (count_over_time({host=~\"$host\", unit=~\"$unit\", app=~\"$app\"}[1h])))",
"legendFormat": "{{unit}}",
"queryType": "range",
"refId": "A"
}
],
"description": "Top systemd units by raw log-line count in the last hour."
},
{
"id": 8,
"title": "Top warning/error units (1h)",
"type": "table",
"datasource": {
"type": "loki",
"uid": "loki"
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 12
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {},
"mappings": [],
"unit": "short",
"decimals": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
}
]
}
},
"overrides": []
},
"options": {
"cellHeight": "sm",
"footer": {
"countRows": false,
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "topk(10, sum by (unit, level) (count_over_time({host=~\"$host\", unit=~\"$unit\", app=~\"$app\", level=~\"warning|err|error|crit|alert|emerg\"}[1h])))",
"legendFormat": "{{unit}} {{level}}",
"queryType": "range",
"refId": "A"
}
],
"description": "Top warning/error-producing units in the last hour."
},
{
"id": 9,
"title": "Recent warnings and errors",
"type": "logs",
"datasource": {
"type": "loki",
"uid": "loki"
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 20
},
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "{host=~\"$host\", unit=~\"$unit\", app=~\"$app\", level=~\"warning|err|error|crit|alert|emerg\"}",
"queryType": "range",
"refId": "A",
"maxLines": 500
}
],
"description": "Latest warning and higher priority logs. Use host/level variables to narrow the stream."
},
{
"id": 10,
"title": "Live filtered log stream",
"type": "logs",
"datasource": {
"type": "loki",
"uid": "loki"
},
"gridPos": {
"h": 10,
"w": 24,
"x": 0,
"y": 28
},
"fieldConfig": {
"defaults": {
"custom": {}
},
"overrides": []
},
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": false,
"showCommonLabels": false,
"showLabels": false,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"datasource": {
"type": "loki",
"uid": "loki"
},
"expr": "{host=~\"$host\", unit=~\"$unit\", app=~\"$app\", level=~\"$level\"}",
"queryType": "range",
"refId": "A",
"maxLines": 1000
}
],
"description": "General Loki stream for selected host and level values."
}
]
}
hosts/AZ-PRM-1/services/monitoring/default.nix
+2
View File
@@ -4,6 +4,8 @@
./blackbox.nix
./cld-scrape.nix
./exporters.nix
./loki.nix
./alloy.nix
./grafana.nix
./prometheus.nix
];
hosts/AZ-PRM-1/services/monitoring/grafana.nix
+11 -1
View File
@@ -2,6 +2,7 @@
serviceName = "grafana";
servicePort = config.m3ta.ports.get serviceName;
prometheusPort = config.m3ta.ports.get "prometheus";
lokiPort = config.m3ta.ports.get "loki";
in {
services.grafana = {
enable = true;
@@ -41,7 +42,16 @@ in {
access = "proxy";
jsonData.timeInterval = "15s";
}
# Phase 3: add Loki datasource here once services.loki is enabled.
{
name = "Loki";
uid = "loki";
type = "loki";
url = "http://localhost:${toString lokiPort}";
access = "proxy";
jsonData = {
maxLines = 1000;
};
}
];
dashboards.settings.providers = [
{
hosts/AZ-PRM-1/services/monitoring/loki.nix
+91
View File
@@ -0,0 +1,91 @@
{config, ...}: let
lokiPort = config.m3ta.ports.get "loki";
prmNetbirdIP = "100.91.49.26";
cldNetbirdIP = "100.91.203.184";
in {
services.loki = {
enable = true;
dataDir = "/var/lib/loki";
configuration = {
auth_enabled = false;
server = {
http_listen_address = "127.0.0.1";
http_listen_port = lokiPort;
};
common = {
path_prefix = config.services.loki.dataDir;
replication_factor = 1;
ring = {
instance_addr = "127.0.0.1";
kvstore.store = "inmemory";
};
};
schema_config.configs = [
{
from = "2024-04-01";
store = "tsdb";
object_store = "filesystem";
schema = "v13";
index = {
prefix = "index_";
period = "24h";
};
}
];
storage_config = {
filesystem.directory = "${config.services.loki.dataDir}/chunks";
tsdb_shipper = {
active_index_directory = "${config.services.loki.dataDir}/tsdb-index";
cache_location = "${config.services.loki.dataDir}/tsdb-cache";
};
};
limits_config = {
retention_period = "336h";
max_query_lookback = "336h";
allow_structured_metadata = true;
volume_enabled = true;
reject_old_samples = true;
reject_old_samples_max_age = "168h";
};
compactor = {
working_directory = "${config.services.loki.dataDir}/compactor";
compaction_interval = "10m";
retention_enabled = true;
retention_delete_delay = "2h";
retention_delete_worker_count = 50;
delete_request_store = "filesystem";
};
analytics.reporting_enabled = false;
};
};
services.traefik.staticConfigOptions.entryPoints.loki = {
address = "${prmNetbirdIP}:${toString lokiPort}";
};
services.traefik.dynamicConfigOptions.http = {
middlewares.loki-basic-auth.basicAuth.usersFile = config.age.secrets.monitoring-loki-htpasswd.path;
services.loki.loadBalancer.servers = [
{url = "http://127.0.0.1:${toString lokiPort}/";}
];
routers.loki-push = {
rule = "PathPrefix(`/loki/api/v1/push`)";
entrypoints = ["loki"];
service = "loki";
middlewares = ["loki-basic-auth"];
};
};
networking.firewall.extraCommands = ''
iptables -A INPUT -p tcp -s ${cldNetbirdIP} --dport ${toString lokiPort} -j ACCEPT
'';
}