From 61fe3f4338625e7ebe664b2a776a12f7f643b365 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sascha=20K=C3=B6nig?= Date: Thu, 9 Jul 2026 13:25:15 +0200 Subject: [PATCH] feat: prometheus + grafana --- flake.lock | 119 +-- flake.nix | 6 + hosts/AZ-CLD-1/services/default.nix | 1 + .../AZ-CLD-1/services/monitoring/default.nix | 6 + .../services/monitoring/node-exporter.nix | 26 + hosts/AZ-CLD-1/services/n8n.nix | 9 +- hosts/AZ-CLD-1/services/postgres.nix | 6 +- hosts/AZ-PRM-1/secrets.nix | 99 ++- hosts/AZ-PRM-1/services/azess.nix | 26 + hosts/AZ-PRM-1/services/default.nix | 2 + .../AZ-PRM-1/services/monitoring/alerting.nix | 242 ++++++ .../AZ-PRM-1/services/monitoring/blackbox.nix | 99 +++ .../services/monitoring/cld-scrape.nix | 24 + .../dashboards/az-blackbox-probe-demo.json | 722 +++++++++++++++++ .../dashboards/az-node-fleet-demo.json | 731 ++++++++++++++++++ .../AZ-PRM-1/services/monitoring/default.nix | 10 + .../services/monitoring/exporters.nix | 15 + .../AZ-PRM-1/services/monitoring/grafana.nix | 80 ++ .../services/monitoring/prometheus.nix | 28 + hosts/AZ-PRM-1/services/postgres.nix | 8 + hosts/common/ports.nix | 7 + overlays/mods/n8n.nix | 60 +- secrets.nix | 4 + secrets/grafana-admin-pw.age | 16 + secrets/grafana-db-password.age | 16 + secrets/grafana-secret-key.age | 18 + secrets/ntfy-grafana-webhook.age | 17 + 27 files changed, 2282 insertions(+), 115 deletions(-) create mode 100644 hosts/AZ-CLD-1/services/monitoring/default.nix create mode 100644 hosts/AZ-CLD-1/services/monitoring/node-exporter.nix create mode 100644 hosts/AZ-PRM-1/services/azess.nix create mode 100644 hosts/AZ-PRM-1/services/monitoring/alerting.nix create mode 100644 hosts/AZ-PRM-1/services/monitoring/blackbox.nix create mode 100644 hosts/AZ-PRM-1/services/monitoring/cld-scrape.nix create mode 100644 hosts/AZ-PRM-1/services/monitoring/dashboards/az-blackbox-probe-demo.json create mode 100644 hosts/AZ-PRM-1/services/monitoring/dashboards/az-node-fleet-demo.json create mode 100644 hosts/AZ-PRM-1/services/monitoring/default.nix create mode 100644 hosts/AZ-PRM-1/services/monitoring/exporters.nix create mode 100644 hosts/AZ-PRM-1/services/monitoring/grafana.nix create mode 100644 hosts/AZ-PRM-1/services/monitoring/prometheus.nix create mode 100644 secrets/grafana-admin-pw.age create mode 100644 secrets/grafana-db-password.age create mode 100644 secrets/grafana-secret-key.age create mode 100644 secrets/ntfy-grafana-webhook.age diff --git a/flake.lock b/flake.lock index 04ef65a..5c57508 100644 --- a/flake.lock +++ b/flake.lock @@ -84,11 +84,11 @@ "nixpkgs": "nixpkgs_4" }, "locked": { - "lastModified": 1780133320, - "narHash": "sha256-8AiN9tV9PBb5xblJiPlhumBbKj61qLjzqXXFtkj3vvY=", + "lastModified": 1782789853, + "narHash": "sha256-LEmctqYRQRq0wB1MoS+IVr/0/uu/0nKahqNeJBGTjJ8=", "ref": "refs/heads/master", - "rev": "920c00313ae242bd93275c30131b9ab1e52ee2fb", - "revCount": 88, + "rev": "cd36a91440b971582fcf2d4eedf9a46fb755f6e1", + "revCount": 92, "type": "git", "url": "ssh://gitea@code.m3ta.dev/m3tam3re/AGENTS" }, @@ -145,6 +145,26 @@ "url": "https://code.m3ta.dev/m3tam3re/AGENTS" } }, + "azess": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1783575830, + "narHash": "sha256-JQY3gB7d7YYgEBigcg7FDmuJ825v8HUK14GU0h0oBY4=", + "ref": "refs/heads/main", + "rev": "3a745690a57fd9c8588682808a0d64328c8b81e3", + "revCount": 3, + "type": "git", + "url": "https://git.az-gruppe.com/AZ-Intec-GmbH/AZess" + }, + "original": { + "type": "git", + "url": "https://git.az-gruppe.com/AZ-Intec-GmbH/AZess" + } + }, "azion-scheduler": { "inputs": { "nixpkgs": [ @@ -287,11 +307,11 @@ ] }, "locked": { - "lastModified": 1778446047, - "narHash": "sha256-oQvcadh2BCkrog+SGrG6YffKJrveYpjj3TdQJWaKhaM=", + "lastModified": 1782772816, + "narHash": "sha256-s9BuFv0mRuZx9C1MF8qPHRdcAK14ONi0A5m6E2wqOoM=", "owner": "nix-community", "repo": "bun2nix", - "rev": "f2bc12af1a6369648aac41041ceeaa0b866599c6", + "rev": "5a39d717029e94163ac223aee8d5c9946cafed1c", "type": "github" }, "original": { @@ -418,11 +438,11 @@ ] }, "locked": { - "lastModified": 1781317940, - "narHash": "sha256-uMVOhV6pVPgm2hn1WGEbIcJRWjnsyWKy8PHCUn0++iI=", + "lastModified": 1782780764, + "narHash": "sha256-Q2wPFsuDo1y6neZ3ttxxOOItupg/4+mPkyAZbC0+wfw=", "owner": "AvengeMedia", "repo": "dms-plugin-registry", - "rev": "4ab59f3da3df33bf106045b856db8de875cc42c6", + "rev": "9fa716427ab5905845b2a10a01d67bee0b1e4c4b", "type": "github" }, "original": { @@ -439,11 +459,11 @@ ] }, "locked": { - "lastModified": 1778716662, - "narHash": "sha256-m1Yf0wZ8j1OHjTc2UwHwyQRSnNeSgLJOd7q5Y45hzi4=", + "lastModified": 1782949081, + "narHash": "sha256-vp6Y/Grm98ESt6ceOkWiHWyZRDV3J1RID4w+6NWK9yA=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "f7c1a2d347e4c52d5fb8d10cb4d94b5884e546fb", + "rev": "17c9d6cdfc60c64f4ee8d306f9bc0b4ccb51481e", "type": "github" }, "original": { @@ -548,11 +568,11 @@ "uv2nix": "uv2nix_2" }, "locked": { - "lastModified": 1781346807, - "narHash": "sha256-ytT4ojx0qFW4b/oYeW+MkmaA3b/BZ9pqkPmpAg8j1gg=", + "lastModified": 1782794050, + "narHash": "sha256-+qKEwIhTkml3DNSmIvGFJyssW9ZXTp78KGclFdcDvX8=", "owner": "NousResearch", "repo": "hermes-agent", - "rev": "2a5dc0ef3df433a36abed9ee544ea067d807c438", + "rev": "972b1620906a1b80772c2f67492ad50b3c83f048", "type": "github" }, "original": { @@ -633,11 +653,11 @@ ] }, "locked": { - "lastModified": 1781305496, - "narHash": "sha256-g8Vv4Qfc7n+lgov97REu3X6BeJtvYY0hlSUZR1GrGQQ=", + "lastModified": 1782749631, + "narHash": "sha256-slFTUgDy0KTPA4LBAmC/9SngDq8GCPdX+ZR0yQHHN1E=", "owner": "nix-community", "repo": "home-manager", - "rev": "c87a39aa979acc4848016d2220c6238390d84779", + "rev": "5d72a29fc36ac21adae6ae35568fe5ee6700850f", "type": "github" }, "original": { @@ -656,11 +676,11 @@ "treefmt-nix": "treefmt-nix" }, "locked": { - "lastModified": 1782091854, - "narHash": "sha256-mnwUTV0WzFm3XH3fR/JKjh7e1WLxQw84ImPMm6CDURQ=", + "lastModified": 1783567605, + "narHash": "sha256-LyF84yqWppXAAEOAkL325Lt+DrVmkbbeXGzDgX9zMzI=", "owner": "numtide", "repo": "llm-agents.nix", - "rev": "7c8390eaa41343b3c0c107a426520fb1efcff5b1", + "rev": "a4c847460f0e773d02d0655ce28dc6b532dd65d6", "type": "github" }, "original": { @@ -686,11 +706,11 @@ "nur": "nur" }, "locked": { - "lastModified": 1781945902, - "narHash": "sha256-CxhK2GmZVttHJ1ltN+MgOvPPP8Faok6xoz42s6s7YEE=", + "lastModified": 1783536074, + "narHash": "sha256-oftWmLYtGzP81WdaCOPN0KxPp5rWdsnYcXfjh2h24AM=", "ref": "refs/heads/master", - "rev": "9cd60383f885e877892e60a3f6034ffd1734c64c", - "revCount": 74, + "rev": "32fdaddf7ca5018154bb97813c3492101ed9aa98", + "revCount": 88, "type": "git", "url": "ssh://gitea@code.m3ta.dev/m3tam3re/m3ta-home" }, @@ -711,11 +731,11 @@ "openspec": "openspec" }, "locked": { - "lastModified": 1781500279, - "narHash": "sha256-810qVHwu6jVhu01FWj9dXuruK84Gw2smAlbu6FPBfeY=", + "lastModified": 1783016129, + "narHash": "sha256-Uxie6uBNLUFJ8Tts7e6AYk5cOER5/EnCqKNL77Tzgc4=", "ref": "refs/heads/master", - "rev": "050c273c65b36dc03c34a7547d80e88afff5ac48", - "revCount": 327, + "rev": "fc5092d72dda7db13f81e5910a159de5108590d5", + "revCount": 350, "type": "git", "url": "ssh://gitea@code.m3ta.dev/m3tam3re/nixpkgs" }, @@ -733,11 +753,11 @@ "openspec": "openspec_2" }, "locked": { - "lastModified": 1782800249, - "narHash": "sha256-jslSMVQf3sn9wC+DkXUW245He/64JxVX4j4y7y2HS+U=", + "lastModified": 1783269665, + "narHash": "sha256-bA4PooGV1x3vtmhs6NEprZbWErwaEJdgxoKfAWaH0do=", "ref": "refs/heads/master", - "rev": "e695a8c6364c118ea60534a7be8eb2097e804e24", - "revCount": 333, + "rev": "f279ae9c89d59f3e0deb0fd6307800f4cb993815", + "revCount": 356, "type": "git", "url": "https://code.m3ta.dev/m3tam3re/nixpkgs" }, @@ -939,11 +959,11 @@ }, "nixpkgs-master": { "locked": { - "lastModified": 1781153468, - "narHash": "sha256-ZBRmjFtJn/XmHBV230OSabKQqxOoOJunJmBtSt1sLs0=", + "lastModified": 1782540244, + "narHash": "sha256-3skOZJEfAUG7LSB/Ok2AQUiqeagYLSq+8wKjtS1hOyc=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "cd265fd6b43f2ec1257c2e400f648895d2ad7ccd", + "rev": "a65184fc373636356fcaf460dc09655a919c66ea", "type": "github" }, "original": { @@ -1003,11 +1023,11 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1781607440, - "narHash": "sha256-rxO+uc/KFbSJp+pgyXRuAX6QlG9hJdnt0BXpEQRXY+U=", + "lastModified": 1783279667, + "narHash": "sha256-/NAkDSsve+GNM0Bt6tleJdCGfsTlK89nPjkVOzZMo0s=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "3e41b24abd260e8f71dbe2f5737d24122f972158", + "rev": "f205b5574fd0cb7da5b702a2da51507b7f4fdd1b", "type": "github" }, "original": { @@ -1051,11 +1071,11 @@ }, "nixpkgs_5": { "locked": { - "lastModified": 1781074563, - "narHash": "sha256-md8WlXOlfnIeHeOScMTTHFyf2d6iaTwPl2apR5EQ3P4=", + "lastModified": 1782723713, + "narHash": "sha256-oPXCU/SSUokcGaJREHibG1CBX3+s/W7orDWQOZDsEeQ=", "owner": "nixos", "repo": "nixpkgs", - "rev": "9ae611a455b90cf061d8f332b977e387bda8e1ca", + "rev": "b5aa0fbd538984f6e3d201be0005b4463d8b09f8", "type": "github" }, "original": { @@ -1157,11 +1177,11 @@ "nixpkgs": "nixpkgs_5" }, "locked": { - "lastModified": 1781346642, - "narHash": "sha256-o92OOSMAB08HQgG7pW2BZVIO53Pkv4oAjLk4Iol3Iko=", + "lastModified": 1782796354, + "narHash": "sha256-zHZEX+twYRO4n369Nrk21DBb519JZCGJLHBo/trhHO0=", "owner": "nix-community", "repo": "NUR", - "rev": "6e4e8d731fbb3831296607d5f88de727cf7bf6de", + "rev": "df4f4323fe161a8bab3aafa2d067ff9d7e223a67", "type": "github" }, "original": { @@ -1223,11 +1243,11 @@ ] }, "locked": { - "lastModified": 1780479100, - "narHash": "sha256-VZZ/ukjciXqiebwei2JizyOnxx0T3IeoowFWElKec4o=", + "lastModified": 1782291243, + "narHash": "sha256-0pVn5pDqlKpqSgzf4eG9TFCjkjzOtnmGTpbILso7GwI=", "owner": "Fission-AI", "repo": "OpenSpec", - "rev": "1b06fddd59d8e592d5b5794a1970b22867e85b1f", + "rev": "737518b36fe4b6fdb09c83eeaf8d873a428c92e6", "type": "github" }, "original": { @@ -1376,6 +1396,7 @@ "inputs": { "agenix": "agenix", "agents": "agents", + "azess": "azess", "azion-scheduler": "azion-scheduler", "disko": "disko", "home-manager": "home-manager_2", diff --git a/flake.nix b/flake.nix index 4f2410c..191d8ec 100644 --- a/flake.nix +++ b/flake.nix @@ -61,6 +61,11 @@ url = "git+https://git.az-gruppe.com/AZ-Intec-GmbH/AZion"; inputs.nixpkgs.follows = "nixpkgs"; }; + + azess = { + url = "git+https://git.az-gruppe.com/AZ-Intec-GmbH/AZess"; + inputs.nixpkgs.follows = "nixpkgs"; + }; }; outputs = { @@ -162,6 +167,7 @@ inputs.disko.nixosModules.disko inputs.azion-scheduler.nixosModules.default inputs.zugferd-service.nixosModules.default + inputs.azess.nixosModules.default ]; }; }; diff --git a/hosts/AZ-CLD-1/services/default.nix b/hosts/AZ-CLD-1/services/default.nix index 6cbc7bc..f175187 100644 --- a/hosts/AZ-CLD-1/services/default.nix +++ b/hosts/AZ-CLD-1/services/default.nix @@ -1,6 +1,7 @@ { imports = [ ./containers + ./monitoring ./gitea.nix # ./gotenberg.nix diff --git a/hosts/AZ-CLD-1/services/monitoring/default.nix b/hosts/AZ-CLD-1/services/monitoring/default.nix new file mode 100644 index 0000000..b438dda --- /dev/null +++ b/hosts/AZ-CLD-1/services/monitoring/default.nix @@ -0,0 +1,6 @@ +# Phase 2 — activate by adding `./monitoring` to hosts/AZ-CLD-1/services/default.nix imports. +{...}: { + imports = [ + ./node-exporter.nix + ]; +} diff --git a/hosts/AZ-CLD-1/services/monitoring/node-exporter.nix b/hosts/AZ-CLD-1/services/monitoring/node-exporter.nix new file mode 100644 index 0000000..accdf32 --- /dev/null +++ b/hosts/AZ-CLD-1/services/monitoring/node-exporter.nix @@ -0,0 +1,26 @@ +# Phase 2 — not active until imported. +# Activate by: +# 1. Create hosts/AZ-CLD-1/services/monitoring/default.nix with `imports = [ ./node-exporter.nix ];` +# 2. Add `./monitoring` to hosts/AZ-CLD-1/services/default.nix imports. +# +# binds to netbird interface so PRM prometheus can scrape it over VPN. +{config, ...}: let + nodeExporterPort = config.m3ta.ports.get "node-exporter"; + prmNetbirdIP = "100.91.49.26"; +in { + services.prometheus.exporters.node = { + enable = true; + port = nodeExporterPort; + listenAddress = "100.91.203.184"; # CLD netbird IP — overwrite if changed + enabledCollectors = [ + "systemd" + "diskstats" + "filesystem" + ]; + openFirewall = false; + }; + + networking.firewall.extraCommands = '' + iptables -A INPUT -p tcp -s ${prmNetbirdIP} --dport ${toString nodeExporterPort} -j ACCEPT + ''; +} diff --git a/hosts/AZ-CLD-1/services/n8n.nix b/hosts/AZ-CLD-1/services/n8n.nix index eca70e6..6d2346a 100644 --- a/hosts/AZ-CLD-1/services/n8n.nix +++ b/hosts/AZ-CLD-1/services/n8n.nix @@ -9,8 +9,15 @@ in { N8N_RUNNERS_ENABLED = true; N8N_NATIVE_PYTHON_RUNNER = true; N8N_RUNNERS_AUTH_TOKEN_FILE = config.age.secrets.n8n-runner-auth-token.path; + NODE_FUNCTION_ALLOW_EXTERNAL = "*"; + N8N_RUNNERS_STDLIB_ALLOW = "*"; + }; + taskRunners = { + enable = true; + environment = { + N8N_RUNNERS_STDLIB_ALLOW = "*"; + }; }; - taskRunners.enable = true; }; systemd.services.${serviceName}.serviceConfig = { diff --git a/hosts/AZ-CLD-1/services/postgres.nix b/hosts/AZ-CLD-1/services/postgres.nix index 096adbc..e29bb33 100644 --- a/hosts/AZ-CLD-1/services/postgres.nix +++ b/hosts/AZ-CLD-1/services/postgres.nix @@ -98,7 +98,7 @@ in { local all jannik_mueller scram-sha-256 local az_test az_test scram-sha-256 local metabase,az_kpi_raw metabase scram-sha-256 - local n8n n8n scram-sha-256 + local all n8n scram-sha-256 local outline outline scram-sha-256 local vaultwarden vaultwarden scram-sha-256 local zammad zammad scram-sha-256 @@ -122,8 +122,8 @@ in { host metabase,az_kpi_raw metabase 127.0.0.1/32 scram-sha-256 host metabase,az_kpi_raw metabase ::1/128 scram-sha-256 - host n8n n8n 127.0.0.1/32 scram-sha-256 - host n8n n8n ::1/128 scram-sha-256 + host all n8n 127.0.0.1/32 scram-sha-256 + host all n8n ::1/128 scram-sha-256 host vaultwarden vaultwarden 127.0.0.1/32 scram-sha-256 host vaultwarden vaultwarden ::1/128 scram-sha-256 diff --git a/hosts/AZ-PRM-1/secrets.nix b/hosts/AZ-PRM-1/secrets.nix index b0429e2..bfc46d6 100644 --- a/hosts/AZ-PRM-1/secrets.nix +++ b/hosts/AZ-PRM-1/secrets.nix @@ -1,43 +1,64 @@ -{ +{lib, ...}: let + ntfyGrafanaWebhookSecret = ../../secrets/ntfy-grafana-webhook.age; +in { age = { - secrets = { - azion-env = { - file = ../../secrets/azion-env.age; + secrets = + { + azion-env = { + file = ../../secrets/azion-env.age; + }; + grafana-admin-pw = { + file = ../../secrets/grafana-admin-pw.age; + owner = "grafana"; + }; + grafana-db-password = { + file = ../../secrets/grafana-db-password.age; + owner = "grafana"; + }; + grafana-secret-key = { + file = ../../secrets/grafana-secret-key.age; + owner = "grafana"; + }; + traefik-env = { + file = ../../secrets/traefik-env.age; + }; + kestra-config = { + file = ../../secrets/kestra-config.age; + mode = "644"; + }; + kestra-env = {file = ../../secrets/kestra-env.age;}; + kestra-secrets = {file = ../../secrets/kestra-secrets.age;}; + n8n-env = { + file = ../../secrets/n8n-env-prm.age; + }; + n8n-runner-auth-token = { + file = ../../secrets/n8n-runner-auth-token-prm.age; + }; + pgadmin-pw = { + file = ../../secrets/pgadmin-pw.age; + owner = "pgadmin"; + }; + pg-cert = { + file = ../../secrets/server.crt.age; + owner = "postgres"; + group = "postgres"; + mode = "0644"; + }; + pg-key = { + file = ../../secrets/server.key.age; + owner = "postgres"; + group = "postgres"; + mode = "0600"; + }; + smb-autoablage = { + file = ../../secrets/smb-autoablage.age; + }; + } + // lib.optionalAttrs (builtins.pathExists ntfyGrafanaWebhookSecret) { + ntfy-grafana-webhook = { + file = ntfyGrafanaWebhookSecret; + mode = "0400"; + }; }; - traefik-env = { - file = ../../secrets/traefik-env.age; - }; - kestra-config = { - file = ../../secrets/kestra-config.age; - mode = "644"; - }; - kestra-env = {file = ../../secrets/kestra-env.age;}; - kestra-secrets = {file = ../../secrets/kestra-secrets.age;}; - n8n-env = { - file = ../../secrets/n8n-env-prm.age; - }; - n8n-runner-auth-token = { - file = ../../secrets/n8n-runner-auth-token-prm.age; - }; - pgadmin-pw = { - file = ../../secrets/pgadmin-pw.age; - owner = "pgadmin"; - }; - pg-cert = { - file = ../../secrets/server.crt.age; - owner = "postgres"; - group = "postgres"; - mode = "0644"; - }; - pg-key = { - file = ../../secrets/server.key.age; - owner = "postgres"; - group = "postgres"; - mode = "0600"; - }; - smb-autoablage = { - file = ../../secrets/smb-autoablage.age; - }; - }; }; } diff --git a/hosts/AZ-PRM-1/services/azess.nix b/hosts/AZ-PRM-1/services/azess.nix new file mode 100644 index 0000000..e6138ab --- /dev/null +++ b/hosts/AZ-PRM-1/services/azess.nix @@ -0,0 +1,26 @@ +{config, ...}: let + serviceName = "azess"; + servicePort = config.m3ta.ports.get serviceName; +in { + services.azess = { + enable = true; + host = "127.0.0.1"; + port = servicePort; + workers = 4; + }; + + services.traefik.dynamicConfigOptions.http = { + services.${serviceName}.loadBalancer.servers = [ + {url = "http://localhost:${toString servicePort}/";} + ]; + + routers.${serviceName} = { + rule = "Host(`azess.l.az-gruppe.com`)"; + tls = { + certResolver = "ionos"; + }; + service = serviceName; + entrypoints = "websecure"; + }; + }; +} diff --git a/hosts/AZ-PRM-1/services/default.nix b/hosts/AZ-PRM-1/services/default.nix index 14695b2..229986d 100644 --- a/hosts/AZ-PRM-1/services/default.nix +++ b/hosts/AZ-PRM-1/services/default.nix @@ -2,8 +2,10 @@ imports = [ ./containers ./backup-ingest.nix + ./azess.nix ./azion-scheduler.nix ./bpi.nix + ./monitoring ./n8n.nix ./netbird.nix ./pgadmin.nix diff --git a/hosts/AZ-PRM-1/services/monitoring/alerting.nix b/hosts/AZ-PRM-1/services/monitoring/alerting.nix new file mode 100644 index 0000000..c1ff669 --- /dev/null +++ b/hosts/AZ-PRM-1/services/monitoring/alerting.nix @@ -0,0 +1,242 @@ +{ + config, + lib, + ... +}: let + prometheusDatasourceUid = "prometheus"; + ntfySecretAvailable = builtins.hasAttr "ntfy-grafana-webhook" config.age.secrets; + + prometheusQuery = refId: expr: { + inherit refId; + datasourceUid = prometheusDatasourceUid; + relativeTimeRange = { + from = 600; + to = 0; + }; + model = { + datasource = { + type = "prometheus"; + uid = prometheusDatasourceUid; + }; + editorMode = "code"; + inherit expr refId; + hide = false; + instant = true; + intervalMs = 1000; + legendFormat = "__auto"; + maxDataPoints = 43200; + range = false; + }; + }; + + thresholdExpression = { + refId, + expressionRefId, + evaluator, + }: { + inherit refId; + datasourceUid = "__expr__"; + model = { + conditions = [ + { + inherit evaluator; + operator.type = "and"; + query.params = []; + reducer = { + params = []; + type = "avg"; + }; + type = "query"; + } + ]; + datasource = { + name = "Expression"; + type = "__expr__"; + uid = "__expr__"; + }; + expression = expressionRefId; + hide = false; + inherit refId; + type = "threshold"; + }; + }; + + mkAlertRule = { + uid, + title, + expr, + for ? "5m", + noDataState ? "Alerting", + execErrState ? "Error", + evaluatorType ? "gt", + evaluatorParams ? [0 0], + labels ? {}, + annotations ? {}, + }: { + inherit uid title for noDataState execErrState; + condition = "B"; + data = [ + (prometheusQuery "A" expr) + (thresholdExpression { + refId = "B"; + expressionRefId = "A"; + evaluator = { + type = evaluatorType; + params = evaluatorParams; + }; + }) + ]; + annotations = + { + summary = title; + } + // annotations; + labels = + { + service = "monitoring"; + } + // labels; + isPaused = false; + }; +in { + warnings = lib.optional (!ntfySecretAvailable) '' + Grafana alert rules are provisioned, but ntfy notifications are disabled because secrets/ntfy-grafana-webhook.age is missing. + Create it as an EnvironmentFile containing: GRAFANA_NTFY_WEBHOOK_URL=https:// + ''; + + systemd.services.grafana.serviceConfig.EnvironmentFile = lib.mkIf ntfySecretAvailable [ + config.age.secrets."ntfy-grafana-webhook".path + ]; + + services.grafana.provision.alerting = { + rules.settings = { + apiVersion = 1; + groups = [ + { + orgId = 1; + name = "az-infrastructure"; + folder = "Infrastructure"; + interval = "60s"; + rules = [ + (mkAlertRule { + uid = "az_host_down"; + title = "Host down"; + expr = ''up{job=~"node|node-cld"}''; + for = "2m"; + evaluatorType = "lt"; + evaluatorParams = [1 0]; + labels.severity = "critical"; + annotations.description = "Prometheus cannot scrape a node_exporter target."; + }) + (mkAlertRule { + uid = "az_cpu_high"; + title = "CPU usage high"; + expr = ''100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)''; + for = "10m"; + evaluatorType = "gt"; + evaluatorParams = [90 0]; + labels.severity = "warning"; + annotations.description = "Average CPU usage has been above 90% for 10 minutes."; + }) + (mkAlertRule { + uid = "az_memory_high"; + title = "Memory usage high"; + expr = ''100 * (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes))''; + for = "10m"; + evaluatorType = "gt"; + evaluatorParams = [90 0]; + labels.severity = "warning"; + annotations.description = "Memory usage has been above 90% for 10 minutes."; + }) + (mkAlertRule { + uid = "az_rootfs_high"; + title = "Root filesystem usage high"; + expr = ''100 * (1 - (node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs|ramfs"}))''; + for = "15m"; + evaluatorType = "gt"; + evaluatorParams = [90 0]; + labels.severity = "warning"; + annotations.description = "Root filesystem usage has been above 90% for 15 minutes."; + }) + (mkAlertRule { + uid = "az_systemd_failed"; + title = "Systemd units failed"; + expr = ''sum by(instance) (node_systemd_units{state="failed"})''; + for = "5m"; + evaluatorType = "gt"; + evaluatorParams = [0 0]; + labels.severity = "warning"; + annotations.description = "One or more systemd units are failed on the host."; + }) + (mkAlertRule { + uid = "az_http_probe_failed"; + title = "HTTP probe failed"; + expr = ''probe_success{job="blackbox_http"}''; + for = "2m"; + evaluatorType = "lt"; + evaluatorParams = [1 0]; + labels.severity = "critical"; + annotations.description = "A blackbox HTTP probe is failing."; + }) + (mkAlertRule { + uid = "az_icmp_probe_failed"; + title = "ICMP probe failed"; + expr = ''probe_success{job="blackbox_icmp"}''; + for = "2m"; + evaluatorType = "lt"; + evaluatorParams = [1 0]; + labels.severity = "warning"; + annotations.description = "A blackbox ICMP probe is failing."; + }) + (mkAlertRule { + uid = "az_tls_cert_expiring"; + title = "TLS certificate expires soon"; + expr = ''(probe_ssl_earliest_cert_expiry{job="blackbox_http"} - time()) / 86400''; + for = "1h"; + noDataState = "OK"; + evaluatorType = "lt"; + evaluatorParams = [14 0]; + labels.severity = "warning"; + annotations.description = "A probed TLS certificate expires in less than 14 days."; + }) + ]; + } + ]; + }; + + contactPoints.settings = lib.mkIf ntfySecretAvailable { + apiVersion = 1; + contactPoints = [ + { + orgId = 1; + name = "ntfy"; + receivers = [ + { + uid = "ntfy-webhook"; + type = "webhook"; + disableResolveMessage = false; + settings = { + url = "$GRAFANA_NTFY_WEBHOOK_URL"; + httpMethod = "POST"; + }; + } + ]; + } + ]; + }; + + policies.settings = lib.mkIf ntfySecretAvailable { + apiVersion = 1; + policies = [ + { + orgId = 1; + receiver = "ntfy"; + group_by = ["alertname" "instance" "target" "severity"]; + group_wait = "30s"; + group_interval = "5m"; + repeat_interval = "4h"; + } + ]; + }; + }; +} diff --git a/hosts/AZ-PRM-1/services/monitoring/blackbox.nix b/hosts/AZ-PRM-1/services/monitoring/blackbox.nix new file mode 100644 index 0000000..255e91b --- /dev/null +++ b/hosts/AZ-PRM-1/services/monitoring/blackbox.nix @@ -0,0 +1,99 @@ +# Phase 2 — not active until imported in services/default.nix. +# Activate by adding `./blackbox.nix` to hosts/AZ-PRM-1/services/monitoring/default.nix imports. +{config, ...}: let + blackboxPort = config.m3ta.ports.get "blackbox-exporter"; + prometheusPort = config.m3ta.ports.get "prometheus"; +in { + services.prometheus.exporters.blackbox = { + enable = true; + port = blackboxPort; + listenAddress = "127.0.0.1"; + openFirewall = false; + configFile = (builtins.toFile "blackbox.yml" '' + modules: + http_2xx: + prober: http + timeout: 5s + http_post_2xx: + prober: http + timeout: 5s + http: + method: POST + icmp_ping: + prober: icmp + timeout: 5s + tcp_connect: + prober: tcp + timeout: 5s + ''); + }; + + services.prometheus.scrapeConfigs = [ + { + job_name = "blackbox_http"; + metrics_path = "/probe"; + params.module = ["http_2xx"]; + static_configs = [ + { + targets = [ + "https://g.l.az-gruppe.com" + "https://wf.l.az-gruppe.com" + "https://k.l.az-gruppe.com" + "https://git.az-gruppe.com" + "https://ping.az-gruppe.com" + "https://llm.az-gruppe.com" + "https://r.az-gruppe.com" + ]; + labels = { + instance = "AZ-PRM-1-blackbox"; + }; + } + ]; + relabel_configs = [ + { + source_labels = ["__address__"]; + target_label = "__param_target"; + } + { + source_labels = ["__param_target"]; + target_label = "target"; + } + { + target_label = "__address__"; + replacement = "localhost:${toString blackboxPort}"; + } + ]; + } + { + job_name = "blackbox_icmp"; + metrics_path = "/probe"; + params.module = ["icmp_ping"]; + static_configs = [ + { + targets = [ + "100.91.203.184" # AZ-CLD-1 netbird + "192.168.152.97" # SMB/DMS share + "192.168.152.98" # SkriptHelper + "192.168.152.102" # legacy PRTG (until decommissioned) + "1.1.1.1" # upstream DNS reachability + "8.8.8.8" # upstream DNS reachability + ]; + } + ]; + relabel_configs = [ + { + source_labels = ["__address__"]; + target_label = "__param_target"; + } + { + source_labels = ["__param_target"]; + target_label = "target"; + } + { + target_label = "__address__"; + replacement = "localhost:${toString blackboxPort}"; + } + ]; + } + ]; +} diff --git a/hosts/AZ-PRM-1/services/monitoring/cld-scrape.nix b/hosts/AZ-PRM-1/services/monitoring/cld-scrape.nix new file mode 100644 index 0000000..1af7da0 --- /dev/null +++ b/hosts/AZ-PRM-1/services/monitoring/cld-scrape.nix @@ -0,0 +1,24 @@ +{config, ...}: let + nodeExporterPort = config.m3ta.ports.get "node-exporter"; + cldNetbirdIP = "100.91.203.184"; +in { + services.prometheus.scrapeConfigs = [ + { + job_name = "node-cld"; + static_configs = [ + { + targets = ["${cldNetbirdIP}:${toString nodeExporterPort}"]; + labels = { + instance = "AZ-CLD-1"; + }; + } + ]; + } + ]; + + # Allow CLD to reach PRM prometheus scrapes (prometheus initiates connection TO CLD exporter) + networking.firewall.extraCommands = '' + # No PRM-side firewall change needed: PRM scrapes outbound to CLD:9100. + # CLD-side must allow inbound from 100.91.49.26 (PRM netbird IP) — see CLD config. + ''; +} diff --git a/hosts/AZ-PRM-1/services/monitoring/dashboards/az-blackbox-probe-demo.json b/hosts/AZ-PRM-1/services/monitoring/dashboards/az-blackbox-probe-demo.json new file mode 100644 index 0000000..260f99a --- /dev/null +++ b/hosts/AZ-PRM-1/services/monitoring/dashboards/az-blackbox-probe-demo.json @@ -0,0 +1,722 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "liveNow": false, + "schemaVersion": 39, + "style": "dark", + "tags": [ + "az", + "demo", + "provisioned" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h" + ] + }, + "timezone": "browser", + "version": 1, + "weekStart": "", + "uid": "az-blackbox-demo", + "title": "AZ Blackbox Probe Demo", + "refresh": "30s", + "description": "Demo dashboard for HTTP and ICMP probes from Prometheus blackbox_exporter.", + "panels": [ + { + "id": 1, + "title": "HTTP probe success", + "type": "stat", + "datasource": null, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "unit": "short", + "decimals": 0, + "min": 0, + "max": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "probe_success{job=\"blackbox_http\"}", + "legendFormat": "{{target}}", + "refId": "A" + } + ], + "description": "1 means the last HTTP probe matched the http_2xx module." + }, + { + "id": 2, + "title": "HTTP 24h availability", + "type": "stat", + "datasource": null, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "unit": "percent", + "decimals": 2, + "min": 0, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 95 + }, + { + "color": "green", + "value": 99 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "avg_over_time(probe_success{job=\"blackbox_http\"}[24h]) * 100", + "legendFormat": "{{target}}", + "refId": "A" + } + ] + }, + { + "id": 3, + "title": "TLS cert days left", + "type": "stat", + "datasource": null, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "unit": "d", + "decimals": 0, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 14 + }, + { + "color": "green", + "value": 30 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "(probe_ssl_earliest_cert_expiry{job=\"blackbox_http\"} - time()) / 86400", + "legendFormat": "{{target}}", + "refId": "A" + } + ] + }, + { + "id": 4, + "title": "HTTP latency", + "type": "timeseries", + "datasource": null, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "unit": "s", + "decimals": 3, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 3 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "expr": "probe_duration_seconds{job=\"blackbox_http\"}", + "legendFormat": "{{target}}", + "refId": "A" + } + ] + }, + { + "id": 5, + "title": "HTTP status code", + "type": "timeseries", + "datasource": null, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "unit": "short", + "decimals": 0, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "expr": "probe_http_status_code{job=\"blackbox_http\"}", + "legendFormat": "{{target}}", + "refId": "A" + } + ] + }, + { + "id": 6, + "title": "ICMP probe success", + "type": "stat", + "datasource": null, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 13 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "unit": "short", + "decimals": 0, + "min": 0, + "max": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "probe_success{job=\"blackbox_icmp\"}", + "legendFormat": "{{target}}", + "refId": "A" + } + ], + "description": "1 means the last ICMP probe succeeded." + }, + { + "id": 7, + "title": "ICMP 24h availability", + "type": "stat", + "datasource": null, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 13 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "unit": "percent", + "decimals": 2, + "min": 0, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 95 + }, + { + "color": "green", + "value": 99 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "avg_over_time(probe_success{job=\"blackbox_icmp\"}[24h]) * 100", + "legendFormat": "{{target}}", + "refId": "A" + } + ] + }, + { + "id": 8, + "title": "ICMP last latency", + "type": "stat", + "datasource": null, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 13 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "unit": "s", + "decimals": 3, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.1 + }, + { + "color": "red", + "value": 0.3 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "probe_duration_seconds{job=\"blackbox_icmp\"}", + "legendFormat": "{{target}}", + "refId": "A" + } + ] + }, + { + "id": 9, + "title": "ICMP latency", + "type": "timeseries", + "datasource": null, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "unit": "s", + "decimals": 3, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.1 + }, + { + "color": "red", + "value": 0.3 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "expr": "probe_duration_seconds{job=\"blackbox_icmp\"}", + "legendFormat": "{{target}}", + "refId": "A" + } + ] + }, + { + "id": 10, + "title": "Probe success timeline", + "type": "timeseries", + "datasource": null, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "unit": "short", + "decimals": 0, + "min": 0, + "max": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "expr": "probe_success{job=\"blackbox_http\"}", + "legendFormat": "HTTP {{target}}", + "refId": "A" + }, + { + "expr": "probe_success{job=\"blackbox_icmp\"}", + "legendFormat": "ICMP {{target}}", + "refId": "B" + } + ] + }, + { + "id": 11, + "title": "DNS + connect + TLS phase timing", + "type": "timeseries", + "datasource": null, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 26 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "unit": "s", + "decimals": 3, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "expr": "probe_http_duration_seconds{job=\"blackbox_http\", phase=\"resolve\"}", + "legendFormat": "{{target}} resolve", + "refId": "A" + }, + { + "expr": "probe_http_duration_seconds{job=\"blackbox_http\", phase=\"connect\"}", + "legendFormat": "{{target}} connect", + "refId": "B" + }, + { + "expr": "probe_http_duration_seconds{job=\"blackbox_http\", phase=\"tls\"}", + "legendFormat": "{{target}} tls", + "refId": "C" + }, + { + "expr": "probe_http_duration_seconds{job=\"blackbox_http\", phase=\"processing\"}", + "legendFormat": "{{target}} processing", + "refId": "D" + } + ] + } + ] +} diff --git a/hosts/AZ-PRM-1/services/monitoring/dashboards/az-node-fleet-demo.json b/hosts/AZ-PRM-1/services/monitoring/dashboards/az-node-fleet-demo.json new file mode 100644 index 0000000..5a87963 --- /dev/null +++ b/hosts/AZ-PRM-1/services/monitoring/dashboards/az-node-fleet-demo.json @@ -0,0 +1,731 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "liveNow": false, + "schemaVersion": 39, + "style": "dark", + "tags": [ + "az", + "demo", + "provisioned" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h" + ] + }, + "timezone": "browser", + "version": 1, + "weekStart": "", + "uid": "az-node-fleet-demo", + "title": "AZ Node Fleet Demo", + "refresh": "30s", + "description": "Demo dashboard for node_exporter metrics on AZ-PRM-1 and AZ-CLD-1. Uses the default Prometheus datasource provisioned by NixOS.", + "panels": [ + { + "id": 1, + "title": "Scrape up", + "type": "stat", + "datasource": null, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "unit": "short", + "decimals": 0, + "min": 0, + "max": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "up{job=~\"node|node-cld\"}", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "description": "1 means Prometheus can scrape the host." + }, + { + "id": 2, + "title": "Uptime", + "type": "stat", + "datasource": null, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "unit": "d", + "decimals": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "(time() - node_boot_time_seconds) / 86400", + "legendFormat": "{{instance}}", + "refId": "A" + } + ] + }, + { + "id": 3, + "title": "CPU busy", + "type": "gauge", + "datasource": null, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "unit": "percent", + "decimals": 1, + "min": 0, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "red", + "value": 90 + } + ] + } + }, + "overrides": [] + }, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "targets": [ + { + "expr": "100 - (avg by(instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval])) * 100)", + "legendFormat": "{{instance}}", + "refId": "A" + } + ] + }, + { + "id": 4, + "title": "Memory used", + "type": "gauge", + "datasource": null, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "unit": "percent", + "decimals": 1, + "min": 0, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + } + }, + "overrides": [] + }, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "targets": [ + { + "expr": "100 * (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes))", + "legendFormat": "{{instance}}", + "refId": "A" + } + ] + }, + { + "id": 5, + "title": "CPU busy by host", + "type": "timeseries", + "datasource": null, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 4 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "unit": "percent", + "decimals": 1, + "min": 0, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "red", + "value": 90 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "expr": "100 - (avg by(instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval])) * 100)", + "legendFormat": "{{instance}}", + "refId": "A" + } + ] + }, + { + "id": 6, + "title": "Load average (5m)", + "type": "timeseries", + "datasource": null, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 4 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "unit": "short", + "decimals": 2, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "expr": "node_load5", + "legendFormat": "{{instance}}", + "refId": "A" + } + ] + }, + { + "id": 7, + "title": "Memory used", + "type": "timeseries", + "datasource": null, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "unit": "percent", + "decimals": 1, + "min": 0, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "expr": "100 * (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes))", + "legendFormat": "{{instance}}", + "refId": "A" + } + ] + }, + { + "id": 8, + "title": "Root filesystem used", + "type": "timeseries", + "datasource": null, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "unit": "percent", + "decimals": 1, + "min": 0, + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 80 + }, + { + "color": "red", + "value": 92 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "expr": "100 * (1 - (node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay|squashfs|ramfs\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay|squashfs|ramfs\"}))", + "legendFormat": "{{instance}}", + "refId": "A" + } + ] + }, + { + "id": 9, + "title": "Network throughput", + "type": "timeseries", + "datasource": null, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 20 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "unit": "Bps", + "decimals": 1, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "expr": "sum by(instance) (rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|br-.*\"}[$__rate_interval]))", + "legendFormat": "{{instance}} RX", + "refId": "A" + }, + { + "expr": "sum by(instance) (rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|br-.*\"}[$__rate_interval]))", + "legendFormat": "{{instance}} TX", + "refId": "B" + } + ] + }, + { + "id": 10, + "title": "Disk IO", + "type": "timeseries", + "datasource": null, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 20 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "unit": "Bps", + "decimals": 1, + "min": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "expr": "sum by(instance) (rate(node_disk_read_bytes_total[$__rate_interval]))", + "legendFormat": "{{instance}} read", + "refId": "A" + }, + { + "expr": "sum by(instance) (rate(node_disk_written_bytes_total[$__rate_interval]))", + "legendFormat": "{{instance}} write", + "refId": "B" + } + ] + }, + { + "id": 11, + "title": "Failed systemd units", + "type": "stat", + "datasource": null, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 28 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "unit": "short", + "decimals": 0, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "sum by(instance) (node_systemd_units{state=\"failed\"})", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "description": "Requires node_exporter systemd collector." + }, + { + "id": 12, + "title": "Filesystem free bytes", + "type": "stat", + "datasource": null, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 28 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "unit": "bytes", + "decimals": 1, + "min": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "expr": "node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay|squashfs|ramfs\"}", + "legendFormat": "{{instance}}", + "refId": "A" + } + ] + } + ] +} diff --git a/hosts/AZ-PRM-1/services/monitoring/default.nix b/hosts/AZ-PRM-1/services/monitoring/default.nix new file mode 100644 index 0000000..ce6ee4d --- /dev/null +++ b/hosts/AZ-PRM-1/services/monitoring/default.nix @@ -0,0 +1,10 @@ +{...}: { + imports = [ + ./alerting.nix + ./blackbox.nix + ./cld-scrape.nix + ./exporters.nix + ./grafana.nix + ./prometheus.nix + ]; +} diff --git a/hosts/AZ-PRM-1/services/monitoring/exporters.nix b/hosts/AZ-PRM-1/services/monitoring/exporters.nix new file mode 100644 index 0000000..0fe13f2 --- /dev/null +++ b/hosts/AZ-PRM-1/services/monitoring/exporters.nix @@ -0,0 +1,15 @@ +{config, ...}: let + nodeExporterPort = config.m3ta.ports.get "node-exporter"; +in { + services.prometheus.exporters.node = { + enable = true; + port = nodeExporterPort; + listenAddress = "127.0.0.1"; + enabledCollectors = [ + "systemd" + "diskstats" + "filesystem" + ]; + openFirewall = false; # localhost only; prometheus scrapes via 127.0.0.1 + }; +} diff --git a/hosts/AZ-PRM-1/services/monitoring/grafana.nix b/hosts/AZ-PRM-1/services/monitoring/grafana.nix new file mode 100644 index 0000000..b86d475 --- /dev/null +++ b/hosts/AZ-PRM-1/services/monitoring/grafana.nix @@ -0,0 +1,80 @@ +{config, ...}: let + serviceName = "grafana"; + servicePort = config.m3ta.ports.get serviceName; + prometheusPort = config.m3ta.ports.get "prometheus"; +in { + services.grafana = { + enable = true; + settings = { + server = { + http_addr = "127.0.0.1"; + http_port = servicePort; + domain = "g.l.az-gruppe.com"; + root_url = "https://g.l.az-gruppe.com"; + serve_from_sub_path = false; + }; + database = { + type = "postgres"; + host = "127.0.0.1"; + name = "grafana"; + user = "grafana"; + password = "$__file{${config.age.secrets.grafana-db-password.path}}"; + ssl_mode = "disable"; + }; + security = { + admin_user = "admin"; + # Grafana file-provider: $__file{} reads the raw secret from the file. + # The agenix secrets must contain ONLY the raw value (no KEY= prefix). + admin_password = "$__file{${config.age.secrets.grafana-admin-pw.path}}"; + secret_key = "$__file{${config.age.secrets.grafana-secret-key.path}}"; + disable_gravatar = true; + }; + }; + provision = { + datasources.settings.datasources = [ + { + name = "Prometheus"; + uid = "prometheus"; + type = "prometheus"; + url = "http://localhost:${toString prometheusPort}"; + isDefault = true; + access = "proxy"; + jsonData.timeInterval = "15s"; + } + # Phase 3: add Loki datasource here once services.loki is enabled. + ]; + dashboards.settings.providers = [ + { + name = "default"; + options.path = "/etc/grafana-dashboards"; + } + ]; + }; + }; + + # Dashboards directory (Phase 1: empty placeholder, JSON files added later) + environment.etc."grafana-dashboards".source = ./. + "/dashboards"; + + systemd.services.grafana = { + after = ["postgresql.service"]; + wants = ["postgresql.service"]; + }; + + # Traefik configuration specific to grafana + services.traefik.dynamicConfigOptions.http = { + services.${serviceName}.loadBalancer.servers = [ + { + url = "http://localhost:${toString servicePort}/"; + } + ]; + + routers.${serviceName} = { + rule = "Host(`g.l.az-gruppe.com`)"; + tls = { + certResolver = "ionos"; + }; + service = serviceName; + entrypoints = "websecure"; + }; + }; +} diff --git a/hosts/AZ-PRM-1/services/monitoring/prometheus.nix b/hosts/AZ-PRM-1/services/monitoring/prometheus.nix new file mode 100644 index 0000000..054a529 --- /dev/null +++ b/hosts/AZ-PRM-1/services/monitoring/prometheus.nix @@ -0,0 +1,28 @@ +{config, ...}: let + prometheusPort = config.m3ta.ports.get "prometheus"; + nodeExporterPort = config.m3ta.ports.get "node-exporter"; +in { + services.prometheus = { + enable = true; + port = prometheusPort; + listenAddress = "127.0.0.1"; + retentionTime = "30d"; + + scrapeConfigs = [ + { + job_name = "node"; + static_configs = [ + { + targets = ["localhost:${toString nodeExporterPort}"]; + labels = { + instance = "AZ-PRM-1"; + }; + } + ]; + } + # Phase 2: add AZ-CLD-1 node target (scrape over netbird 100.x). + # Phase 2: add blackbox_exporter, podman/cadvisor targets. + # Phase 2: enable remote-write receiver (extraFlags) for Windows clients. + ]; + }; +} diff --git a/hosts/AZ-PRM-1/services/postgres.nix b/hosts/AZ-PRM-1/services/postgres.nix index e651041..1caa6d5 100644 --- a/hosts/AZ-PRM-1/services/postgres.nix +++ b/hosts/AZ-PRM-1/services/postgres.nix @@ -31,6 +31,10 @@ CREATE USER vaultwarden WITH ENCRYPTED PASSWORD 'n8n'; CREATE DATABASE vaultwarden; ALTER DATABASE vaultwarden OWNER to vaultwarden; + + CREATE USER grafana WITH ENCRYPTED PASSWORD 'grafana'; + CREATE DATABASE grafana; + ALTER DATABASE grafana OWNER to grafana; ''; authentication = pkgs.lib.mkOverride 10 '' # Local connections (Unix socket) @@ -48,6 +52,10 @@ host baserow baserow 10.89.0.0/24 scram-sha-256 host kestra kestra 10.89.0.0/24 scram-sha-256 + # Grafana (local socket / localhost only) + host grafana grafana 127.0.0.1/32 scram-sha-256 + host grafana grafana ::1/128 scram-sha-256 + # Deny all other connections host all all 0.0.0.0/0 reject host all all ::/0 reject diff --git a/hosts/common/ports.nix b/hosts/common/ports.nix index a1aa8c1..2b7ca51 100644 --- a/hosts/common/ports.nix +++ b/hosts/common/ports.nix @@ -24,6 +24,7 @@ baserow = 3050; frappe-lms = 3052; snipe-it = 3053; + azess = 3054; librechat = 3040; librechat-dev = 3141; @@ -41,6 +42,12 @@ postgres = 5432; pgadmin = 5050; mysql = 3306; + + # Observability stack (AZ-PRM-1) + grafana = 3060; + prometheus = 9090; + node-exporter = 9100; + blackbox-exporter = 9115; }; hostOverrides = { diff --git a/overlays/mods/n8n.nix b/overlays/mods/n8n.nix index 1b1ae3d..8b813f1 100644 --- a/overlays/mods/n8n.nix +++ b/overlays/mods/n8n.nix @@ -1,26 +1,40 @@ -# {prev}: -# prev.n8n.overrideAttrs (oldAttrs: rec { -# version = "1.112.6"; +# Adds `libphonenumber-js` to n8n's node_modules so it can be require()'d +# from Code nodes running in the Task Runner. +# +# Prerequisite on the n8n service: +# N8N_RUNNERS_EXTERNAL_ALLOW = "libphonenumber-js"; +# +# libphonenumber-js has zero runtime + peer dependencies, so a plain tarball +# unpack into the shared node_modules hierarchy is sufficient. +{prev}: let + libphonenumber-js = prev.stdenv.mkDerivation rec { + pname = "libphonenumber-js"; + version = "1.13.8"; -# src = prev.fetchFromGitHub { -# owner = "n8n-io"; -# repo = "n8n"; -# rev = "n8n@${version}"; -# hash = "sha256-r/MCU/S1kkKQPkhmp9ZHTtgZxMu5TFCl5Yejp73gATw="; -# }; + src = prev.fetchurl { + url = "https://registry.npmjs.org/${pname}/-/${pname}-${version}.tgz"; + hash = "sha256-SysWDKlbXgbe441Sd4pO+k+F1y/UC49a1KL+DcFWBIA="; + }; -# pnpmDeps = prev.pnpm_10.fetchDeps { -# pname = oldAttrs.pname; -# inherit version src; -# fetcherVersion = 1; -# hash = "sha256-j+HJhvzrcu8JsezcFJxfgteOgTspWQb2ZSN2fEl7Voo="; -# }; + dontConfigure = true; + dontBuild = true; -# nativeBuildInputs = -# builtins.map -# (input: -# if input == prev.pnpm_9.configHook -# then prev.pnpm_10.configHook -# else input) -# oldAttrs.nativeBuildInputs; -# }) + installPhase = '' + runHook preInstall + mkdir -p $out/lib/node_modules/${pname} + cp -r * $out/lib/node_modules/${pname}/ + runHook postInstall + ''; + }; +in + prev.n8n.overrideAttrs (oldAttrs: { + postInstall = + (oldAttrs.postInstall or "") + + '' + # n8n ships a pnpm stub symlink (libphonenumber-js -> empty-npm-package). + # Remove it and place the real package there instead. + rm -rf $out/lib/n8n/node_modules/libphonenumber-js + cp -r ${libphonenumber-js}/lib/node_modules/libphonenumber-js \ + $out/lib/n8n/node_modules/ + ''; + }) diff --git a/secrets.nix b/secrets.nix index 6eec521..ea2051b 100644 --- a/secrets.nix +++ b/secrets.nix @@ -53,4 +53,8 @@ in { "secrets/outline-key.age".publicKeys = systems ++ users; "secrets/ref-key.age".publicKeys = systems ++ users; "secrets/exa-key.age".publicKeys = systems ++ users; + "secrets/grafana-admin-pw.age".publicKeys = systems ++ users; + "secrets/grafana-db-password.age".publicKeys = systems ++ users; + "secrets/grafana-secret-key.age".publicKeys = systems ++ users; + "secrets/ntfy-grafana-webhook.age".publicKeys = systems ++ users; } diff --git a/secrets/grafana-admin-pw.age b/secrets/grafana-admin-pw.age new file mode 100644 index 0000000..0344160 --- /dev/null +++ b/secrets/grafana-admin-pw.age @@ -0,0 +1,16 @@ +-----BEGIN AGE ENCRYPTED FILE----- +YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHNzaC1lZDI1NTE5IFpoVnNlZyBEQ1Ax +R2hMZkt2a0cwYlZHYjFhVG5kaHB5MlEwMVVaK2ZQdWdVTEx0cUdjCmtXbUgyRTJt +SXFvQ1F2WWxoU1RYcG1NSmd4QnU4Y3UxaFVsZkZZc0RINU0KLT4gc3NoLWVkMjU1 +MTkgU3JIYXFBIEdZa0hicVVxbExVOHdkNFNROUlkSmg2ak5UVVREK1ZMZkt1bEpD +ejdOaE0KYjBrcjk0UG5LV2orL3ZISTZXV0h6cnZ3OThIYjQ0Q1RZUVd6L2FEeERa +cwotPiBzc2gtZWQyNTUxOSBsR3FWWmcgZ21JMlRGUW81U3BhbUYrUjU3Y1NGNm03 +RElwbUpVMmhFalF5ZjZMYmx5Ywovc2ErZS85U0V0T2diNlVYM2VIR3lkZnVrRUY4 +N0oyQUlNTzB6Zng3ODNnCi0+IHNzaC1lZDI1NTE5IENTTXloZyBBVEVYeStOZ1hM +K2d0S3ZqRmZ2TVpUejJwd0dCVEw5ZE5WWkRlcXhKbVEwCjFKRU5DRDd3aHN1a1VY +VzVaMU01S21YMVZ6YndKUUQ4emg3TFN4djVOREUKLT4gdDctZ3JlYXNlIHw9VCBj +dGUgRk03NSdtUXwgSQoza3BNSnI2eThMc3F4eVBSY0dhVi92K3BKcC9EaWVZRWxu +N0VKZGVLWG1SYWhXWTZ4TGQ1Y21VTFY0ZmNlQQotLS0gcHdhMHcyT2x1V2RmRWV5 +OG1yTFhnVFdlS2RqUkd0YlVFczRzWmwyYjJwNAp0z9rmy4jPvnkW6kA1OBLpvXby +Zt/VxkA0aj8fgsVoNRFnGWaVDFcynLR0ZJMNYQ== +-----END AGE ENCRYPTED FILE----- diff --git a/secrets/grafana-db-password.age b/secrets/grafana-db-password.age new file mode 100644 index 0000000..3706c9b --- /dev/null +++ b/secrets/grafana-db-password.age @@ -0,0 +1,16 @@ +-----BEGIN AGE ENCRYPTED FILE----- +YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHNzaC1lZDI1NTE5IFpoVnNlZyA5aVNn +TUliejFZc2tnZExIWjRDWE9NTWJVVWlOWGtFMUF0L0U4TkNSUnh3Ck9Ock1IVW8x +Zk4wVG5pczY4M09ISnhuYU1lS2NyNkxzZnpPdFI1SmN6REkKLT4gc3NoLWVkMjU1 +MTkgU3JIYXFBIEQrckhQTWJzM1lrVTlOeGdoQ3BWN0djd3Y1Q2E5aXd2VU1scGF1 +UzM1aHMKdzNPQWZpeE5DSGllcXJSUzZEN2JGYitxQjhXa3B2NVRIeHNNbTMxamFm +OAotPiBzc2gtZWQyNTUxOSBsR3FWWmcgUG55RWc4Y0RlOHI0bFM4akdIYjhwTCt2 +R09ydXBCSm44bTMvak84TTFVRQorWmVzVXBsaFNaVFk1enRWaTdGNmQ2eGt2ZFpy +UDVhYWllSkJ6RGJhVGxNCi0+IHNzaC1lZDI1NTE5IENTTXloZyBFcDF4dEhlUjhG +alVMUVlScjFsQys5VjhoMzlMNzNJOG9GOXp1bnQxdjM4CnJYQmoxanpnRW1YWFJN +Q28ybGcvc1YwWjFZNFc3eHlaRjBlTDIvcXlXM0UKLT4gTlotdnotZ3JlYXNlIGhT +Z0gxViA7NkUjMWhRIE5rc35pcVA/IEtrQG8zen18CjVRM0cwV2Z3bXlpS2JGb3lO +WUNRRDdpNngrZEJBdGVQcW40cjFiTGlMdFFSYmh1QlJyTjIKLS0tIFl1YlloL0tC +ZVBZbVJqOFp2OUtMTnUvUWRSL3hUc0VSZ0tCb2lYeU4wMW8KB077DPDfkB1bje7z +blXS/m1slMnw11sdLOY5FnQV7m5JDuK15e0iruhqPvm0TADR +-----END AGE ENCRYPTED FILE----- diff --git a/secrets/grafana-secret-key.age b/secrets/grafana-secret-key.age new file mode 100644 index 0000000..7fd612d --- /dev/null +++ b/secrets/grafana-secret-key.age @@ -0,0 +1,18 @@ +-----BEGIN AGE ENCRYPTED FILE----- +YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHNzaC1lZDI1NTE5IFpoVnNlZyBqUUd1 +VXo5ZGo0bnczUGd1cW54ZEdaLzdIcHdNcWo1N3B3YUZBUXo2eTJJCnZMSDQ1WnNF +UHF1c251cExHc3U2bzgvSzZLMGdySUdMcDNveVJRMDd1UkEKLT4gc3NoLWVkMjU1 +MTkgU3JIYXFBIHFkeC9jTTB0MUo0OExabHdyb0ZScTRXcUJmNjRUVG5EMEljVHlo +TkdtRUEKS2lxTU5rTW9tSVpzNFgxK2k3dllNNk9kNmIzU3J4OC83YktFZ3VEbkVQ +ZwotPiBzc2gtZWQyNTUxOSBsR3FWWmcgUTAzQWkvSUYxcldNUjlRd3VnalprTFZT +UGFGckFYT3h2bjUzeEFUZWdqSQpGQS9WRGFoOUtwTXpQdG9NUThNNkUvWDIxZ1hC +ZDlKMUxtUEdVU3Zzb3dRCi0+IHNzaC1lZDI1NTE5IENTTXloZyBqelcyRlBFMUh0 +ZDNtZWx0SkdsNTVlL1c0aDloVXJxSUlHbEtnVGN3RUMwCmhxUS9zcjNVQ0NkRElO +YzVzUFlrQktRelBRYmt3anVOL1lSLytNaEVnVlkKLT4gXnBlI34iJVItZ3JlYXNl +IDYxR3dbSlsrCmg5QUhLS3AzUHdpWFFUYjgvQ2VTSGlTWkVqUWRrdDRUaVQ4T3ZI +eWN0Mnp2OFJMa1ZOT2VFTWMrOHhjY09sWG4KUGlCOXpYeTVsOGVyd1NkY0RqeU5o +MzNHcy9nS1BNR0orUXo1UDdySllwRG1HbS9GUlRmYmNoc1YKLS0tIGZNemhmcE4x +N0tyUzg2VHJaajEydnp6SFdCQXV2WlNNYyswNVJETm90VVUKdtdcfrVdawFwMKZ1 +FUlQvH+ef1iBjuDIX2zuy3Fvtwu8lo3qaOGJTT7lP+bPXXjtKdiz02DkhhD2HM12 +i16mfLP3bvMms+W68r87cxx/zFq7O1D44cvYY2a1Z9ZmU0LZsw== +-----END AGE ENCRYPTED FILE----- diff --git a/secrets/ntfy-grafana-webhook.age b/secrets/ntfy-grafana-webhook.age new file mode 100644 index 0000000..9f46e52 --- /dev/null +++ b/secrets/ntfy-grafana-webhook.age @@ -0,0 +1,17 @@ +-----BEGIN AGE ENCRYPTED FILE----- +YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHNzaC1lZDI1NTE5IFpoVnNlZyBMTnFZ +NkozaWQvUklzemlpaHhCeVU3YmZ2OW5BRUY2Z1k5RW91WC84MzJ3Cmk1VTdoSVB3 +MXdFKzF3cEVoaVF0UlUwaERkWmJkSDAxbGt0SGJ3RXltcDQKLT4gc3NoLWVkMjU1 +MTkgU3JIYXFBIDFCZmo1NTJQRVlpZCt4MlRMbVV5anZnUUlUWUp6aDhDV3FlYmxO +RnhCVXcKLzhyWFE4R2VDWkhzVzBpaCtQVFowN0YxenNIWjNKYld4RkRzaFNZb0tv +TQotPiBzc2gtZWQyNTUxOSBsR3FWWmcgSDlQdElQcXI1MXRrMVY0cVppc2VEN1ZQ +eVJrSkM5MlVjdzF2TTN6QmR5SQpEUFpEY0trRXB2RHFUZEcrVzUvdW5JWFZFOGNP +THVrZTlXMWVQaUlCaU00Ci0+IHNzaC1lZDI1NTE5IENTTXloZyA1V1V4VmRHSTNP +cGJqK3F6ckwzSEhMMUQxZEJPVUYxRTI3TGdjMzRZdGhJClpja3FPNm4wQ2VSTlZu +ak1zcUljUlF3enNCMy9XdVpxSThxTEN3RlRVODAKLT4gJmBDT1QpLi1ncmVhc2Ug +VHpTSUJnIDJtYihWIEx9aXBJemEKSStWQnBkdTRvakkzUFRVCi0tLSBybVMxYVNR +b3NDc0Z5NXUrNnNDM2p6aGdMMjFkbzlGTXZJZjVlWGkvZVZvCmPEPGsFFpaUVOfU +3uk2FMi20oSgKSFrVCz0YkhJXXmSVgA2qX7QGnkVqb8ffzAJeYS1xlgj/cAjezux +RTDP4urrTRBnR8egcn6j9/8YoHFJVPmKVXXndIt0MqAsYjeyqWYfKIvaJMA1GSMU +2Eila212uzU= +-----END AGE ENCRYPTED FILE-----