feat: prometheus + grafana
This commit is contained in:
Generated
+70
-49
@@ -84,11 +84,11 @@
|
||||
"nixpkgs": "nixpkgs_4"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1780133320,
|
||||
"narHash": "sha256-8AiN9tV9PBb5xblJiPlhumBbKj61qLjzqXXFtkj3vvY=",
|
||||
"lastModified": 1782789853,
|
||||
"narHash": "sha256-LEmctqYRQRq0wB1MoS+IVr/0/uu/0nKahqNeJBGTjJ8=",
|
||||
"ref": "refs/heads/master",
|
||||
"rev": "920c00313ae242bd93275c30131b9ab1e52ee2fb",
|
||||
"revCount": 88,
|
||||
"rev": "cd36a91440b971582fcf2d4eedf9a46fb755f6e1",
|
||||
"revCount": 92,
|
||||
"type": "git",
|
||||
"url": "ssh://gitea@code.m3ta.dev/m3tam3re/AGENTS"
|
||||
},
|
||||
@@ -145,6 +145,26 @@
|
||||
"url": "https://code.m3ta.dev/m3tam3re/AGENTS"
|
||||
}
|
||||
},
|
||||
"azess": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1783575830,
|
||||
"narHash": "sha256-JQY3gB7d7YYgEBigcg7FDmuJ825v8HUK14GU0h0oBY4=",
|
||||
"ref": "refs/heads/main",
|
||||
"rev": "3a745690a57fd9c8588682808a0d64328c8b81e3",
|
||||
"revCount": 3,
|
||||
"type": "git",
|
||||
"url": "https://git.az-gruppe.com/AZ-Intec-GmbH/AZess"
|
||||
},
|
||||
"original": {
|
||||
"type": "git",
|
||||
"url": "https://git.az-gruppe.com/AZ-Intec-GmbH/AZess"
|
||||
}
|
||||
},
|
||||
"azion-scheduler": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
@@ -287,11 +307,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1778446047,
|
||||
"narHash": "sha256-oQvcadh2BCkrog+SGrG6YffKJrveYpjj3TdQJWaKhaM=",
|
||||
"lastModified": 1782772816,
|
||||
"narHash": "sha256-s9BuFv0mRuZx9C1MF8qPHRdcAK14ONi0A5m6E2wqOoM=",
|
||||
"owner": "nix-community",
|
||||
"repo": "bun2nix",
|
||||
"rev": "f2bc12af1a6369648aac41041ceeaa0b866599c6",
|
||||
"rev": "5a39d717029e94163ac223aee8d5c9946cafed1c",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -418,11 +438,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1781317940,
|
||||
"narHash": "sha256-uMVOhV6pVPgm2hn1WGEbIcJRWjnsyWKy8PHCUn0++iI=",
|
||||
"lastModified": 1782780764,
|
||||
"narHash": "sha256-Q2wPFsuDo1y6neZ3ttxxOOItupg/4+mPkyAZbC0+wfw=",
|
||||
"owner": "AvengeMedia",
|
||||
"repo": "dms-plugin-registry",
|
||||
"rev": "4ab59f3da3df33bf106045b856db8de875cc42c6",
|
||||
"rev": "9fa716427ab5905845b2a10a01d67bee0b1e4c4b",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -439,11 +459,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1778716662,
|
||||
"narHash": "sha256-m1Yf0wZ8j1OHjTc2UwHwyQRSnNeSgLJOd7q5Y45hzi4=",
|
||||
"lastModified": 1782949081,
|
||||
"narHash": "sha256-vp6Y/Grm98ESt6ceOkWiHWyZRDV3J1RID4w+6NWK9yA=",
|
||||
"owner": "hercules-ci",
|
||||
"repo": "flake-parts",
|
||||
"rev": "f7c1a2d347e4c52d5fb8d10cb4d94b5884e546fb",
|
||||
"rev": "17c9d6cdfc60c64f4ee8d306f9bc0b4ccb51481e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -548,11 +568,11 @@
|
||||
"uv2nix": "uv2nix_2"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1781346807,
|
||||
"narHash": "sha256-ytT4ojx0qFW4b/oYeW+MkmaA3b/BZ9pqkPmpAg8j1gg=",
|
||||
"lastModified": 1782794050,
|
||||
"narHash": "sha256-+qKEwIhTkml3DNSmIvGFJyssW9ZXTp78KGclFdcDvX8=",
|
||||
"owner": "NousResearch",
|
||||
"repo": "hermes-agent",
|
||||
"rev": "2a5dc0ef3df433a36abed9ee544ea067d807c438",
|
||||
"rev": "972b1620906a1b80772c2f67492ad50b3c83f048",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -633,11 +653,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1781305496,
|
||||
"narHash": "sha256-g8Vv4Qfc7n+lgov97REu3X6BeJtvYY0hlSUZR1GrGQQ=",
|
||||
"lastModified": 1782749631,
|
||||
"narHash": "sha256-slFTUgDy0KTPA4LBAmC/9SngDq8GCPdX+ZR0yQHHN1E=",
|
||||
"owner": "nix-community",
|
||||
"repo": "home-manager",
|
||||
"rev": "c87a39aa979acc4848016d2220c6238390d84779",
|
||||
"rev": "5d72a29fc36ac21adae6ae35568fe5ee6700850f",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -656,11 +676,11 @@
|
||||
"treefmt-nix": "treefmt-nix"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1782091854,
|
||||
"narHash": "sha256-mnwUTV0WzFm3XH3fR/JKjh7e1WLxQw84ImPMm6CDURQ=",
|
||||
"lastModified": 1783567605,
|
||||
"narHash": "sha256-LyF84yqWppXAAEOAkL325Lt+DrVmkbbeXGzDgX9zMzI=",
|
||||
"owner": "numtide",
|
||||
"repo": "llm-agents.nix",
|
||||
"rev": "7c8390eaa41343b3c0c107a426520fb1efcff5b1",
|
||||
"rev": "a4c847460f0e773d02d0655ce28dc6b532dd65d6",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -686,11 +706,11 @@
|
||||
"nur": "nur"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1781945902,
|
||||
"narHash": "sha256-CxhK2GmZVttHJ1ltN+MgOvPPP8Faok6xoz42s6s7YEE=",
|
||||
"lastModified": 1783536074,
|
||||
"narHash": "sha256-oftWmLYtGzP81WdaCOPN0KxPp5rWdsnYcXfjh2h24AM=",
|
||||
"ref": "refs/heads/master",
|
||||
"rev": "9cd60383f885e877892e60a3f6034ffd1734c64c",
|
||||
"revCount": 74,
|
||||
"rev": "32fdaddf7ca5018154bb97813c3492101ed9aa98",
|
||||
"revCount": 88,
|
||||
"type": "git",
|
||||
"url": "ssh://gitea@code.m3ta.dev/m3tam3re/m3ta-home"
|
||||
},
|
||||
@@ -711,11 +731,11 @@
|
||||
"openspec": "openspec"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1781500279,
|
||||
"narHash": "sha256-810qVHwu6jVhu01FWj9dXuruK84Gw2smAlbu6FPBfeY=",
|
||||
"lastModified": 1783016129,
|
||||
"narHash": "sha256-Uxie6uBNLUFJ8Tts7e6AYk5cOER5/EnCqKNL77Tzgc4=",
|
||||
"ref": "refs/heads/master",
|
||||
"rev": "050c273c65b36dc03c34a7547d80e88afff5ac48",
|
||||
"revCount": 327,
|
||||
"rev": "fc5092d72dda7db13f81e5910a159de5108590d5",
|
||||
"revCount": 350,
|
||||
"type": "git",
|
||||
"url": "ssh://gitea@code.m3ta.dev/m3tam3re/nixpkgs"
|
||||
},
|
||||
@@ -733,11 +753,11 @@
|
||||
"openspec": "openspec_2"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1782800249,
|
||||
"narHash": "sha256-jslSMVQf3sn9wC+DkXUW245He/64JxVX4j4y7y2HS+U=",
|
||||
"lastModified": 1783269665,
|
||||
"narHash": "sha256-bA4PooGV1x3vtmhs6NEprZbWErwaEJdgxoKfAWaH0do=",
|
||||
"ref": "refs/heads/master",
|
||||
"rev": "e695a8c6364c118ea60534a7be8eb2097e804e24",
|
||||
"revCount": 333,
|
||||
"rev": "f279ae9c89d59f3e0deb0fd6307800f4cb993815",
|
||||
"revCount": 356,
|
||||
"type": "git",
|
||||
"url": "https://code.m3ta.dev/m3tam3re/nixpkgs"
|
||||
},
|
||||
@@ -939,11 +959,11 @@
|
||||
},
|
||||
"nixpkgs-master": {
|
||||
"locked": {
|
||||
"lastModified": 1781153468,
|
||||
"narHash": "sha256-ZBRmjFtJn/XmHBV230OSabKQqxOoOJunJmBtSt1sLs0=",
|
||||
"lastModified": 1782540244,
|
||||
"narHash": "sha256-3skOZJEfAUG7LSB/Ok2AQUiqeagYLSq+8wKjtS1hOyc=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "cd265fd6b43f2ec1257c2e400f648895d2ad7ccd",
|
||||
"rev": "a65184fc373636356fcaf460dc09655a919c66ea",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -1003,11 +1023,11 @@
|
||||
},
|
||||
"nixpkgs_2": {
|
||||
"locked": {
|
||||
"lastModified": 1781607440,
|
||||
"narHash": "sha256-rxO+uc/KFbSJp+pgyXRuAX6QlG9hJdnt0BXpEQRXY+U=",
|
||||
"lastModified": 1783279667,
|
||||
"narHash": "sha256-/NAkDSsve+GNM0Bt6tleJdCGfsTlK89nPjkVOzZMo0s=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "3e41b24abd260e8f71dbe2f5737d24122f972158",
|
||||
"rev": "f205b5574fd0cb7da5b702a2da51507b7f4fdd1b",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -1051,11 +1071,11 @@
|
||||
},
|
||||
"nixpkgs_5": {
|
||||
"locked": {
|
||||
"lastModified": 1781074563,
|
||||
"narHash": "sha256-md8WlXOlfnIeHeOScMTTHFyf2d6iaTwPl2apR5EQ3P4=",
|
||||
"lastModified": 1782723713,
|
||||
"narHash": "sha256-oPXCU/SSUokcGaJREHibG1CBX3+s/W7orDWQOZDsEeQ=",
|
||||
"owner": "nixos",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "9ae611a455b90cf061d8f332b977e387bda8e1ca",
|
||||
"rev": "b5aa0fbd538984f6e3d201be0005b4463d8b09f8",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -1157,11 +1177,11 @@
|
||||
"nixpkgs": "nixpkgs_5"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1781346642,
|
||||
"narHash": "sha256-o92OOSMAB08HQgG7pW2BZVIO53Pkv4oAjLk4Iol3Iko=",
|
||||
"lastModified": 1782796354,
|
||||
"narHash": "sha256-zHZEX+twYRO4n369Nrk21DBb519JZCGJLHBo/trhHO0=",
|
||||
"owner": "nix-community",
|
||||
"repo": "NUR",
|
||||
"rev": "6e4e8d731fbb3831296607d5f88de727cf7bf6de",
|
||||
"rev": "df4f4323fe161a8bab3aafa2d067ff9d7e223a67",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -1223,11 +1243,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1780479100,
|
||||
"narHash": "sha256-VZZ/ukjciXqiebwei2JizyOnxx0T3IeoowFWElKec4o=",
|
||||
"lastModified": 1782291243,
|
||||
"narHash": "sha256-0pVn5pDqlKpqSgzf4eG9TFCjkjzOtnmGTpbILso7GwI=",
|
||||
"owner": "Fission-AI",
|
||||
"repo": "OpenSpec",
|
||||
"rev": "1b06fddd59d8e592d5b5794a1970b22867e85b1f",
|
||||
"rev": "737518b36fe4b6fdb09c83eeaf8d873a428c92e6",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -1376,6 +1396,7 @@
|
||||
"inputs": {
|
||||
"agenix": "agenix",
|
||||
"agents": "agents",
|
||||
"azess": "azess",
|
||||
"azion-scheduler": "azion-scheduler",
|
||||
"disko": "disko",
|
||||
"home-manager": "home-manager_2",
|
||||
|
||||
@@ -61,6 +61,11 @@
|
||||
url = "git+https://git.az-gruppe.com/AZ-Intec-GmbH/AZion";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
|
||||
azess = {
|
||||
url = "git+https://git.az-gruppe.com/AZ-Intec-GmbH/AZess";
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
};
|
||||
};
|
||||
|
||||
outputs = {
|
||||
@@ -162,6 +167,7 @@
|
||||
inputs.disko.nixosModules.disko
|
||||
inputs.azion-scheduler.nixosModules.default
|
||||
inputs.zugferd-service.nixosModules.default
|
||||
inputs.azess.nixosModules.default
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
{
|
||||
imports = [
|
||||
./containers
|
||||
./monitoring
|
||||
|
||||
./gitea.nix
|
||||
# ./gotenberg.nix
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
# Phase 2 — activate by adding `./monitoring` to hosts/AZ-CLD-1/services/default.nix imports.
|
||||
{...}: {
|
||||
imports = [
|
||||
./node-exporter.nix
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
# Phase 2 — not active until imported.
|
||||
# Activate by:
|
||||
# 1. Create hosts/AZ-CLD-1/services/monitoring/default.nix with `imports = [ ./node-exporter.nix ];`
|
||||
# 2. Add `./monitoring` to hosts/AZ-CLD-1/services/default.nix imports.
|
||||
#
|
||||
# binds to netbird interface so PRM prometheus can scrape it over VPN.
|
||||
{config, ...}: let
|
||||
nodeExporterPort = config.m3ta.ports.get "node-exporter";
|
||||
prmNetbirdIP = "100.91.49.26";
|
||||
in {
|
||||
services.prometheus.exporters.node = {
|
||||
enable = true;
|
||||
port = nodeExporterPort;
|
||||
listenAddress = "100.91.203.184"; # CLD netbird IP — overwrite if changed
|
||||
enabledCollectors = [
|
||||
"systemd"
|
||||
"diskstats"
|
||||
"filesystem"
|
||||
];
|
||||
openFirewall = false;
|
||||
};
|
||||
|
||||
networking.firewall.extraCommands = ''
|
||||
iptables -A INPUT -p tcp -s ${prmNetbirdIP} --dport ${toString nodeExporterPort} -j ACCEPT
|
||||
'';
|
||||
}
|
||||
@@ -9,8 +9,15 @@ in {
|
||||
N8N_RUNNERS_ENABLED = true;
|
||||
N8N_NATIVE_PYTHON_RUNNER = true;
|
||||
N8N_RUNNERS_AUTH_TOKEN_FILE = config.age.secrets.n8n-runner-auth-token.path;
|
||||
NODE_FUNCTION_ALLOW_EXTERNAL = "*";
|
||||
N8N_RUNNERS_STDLIB_ALLOW = "*";
|
||||
};
|
||||
taskRunners = {
|
||||
enable = true;
|
||||
environment = {
|
||||
N8N_RUNNERS_STDLIB_ALLOW = "*";
|
||||
};
|
||||
};
|
||||
taskRunners.enable = true;
|
||||
};
|
||||
|
||||
systemd.services.${serviceName}.serviceConfig = {
|
||||
|
||||
@@ -98,7 +98,7 @@ in {
|
||||
local all jannik_mueller scram-sha-256
|
||||
local az_test az_test scram-sha-256
|
||||
local metabase,az_kpi_raw metabase scram-sha-256
|
||||
local n8n n8n scram-sha-256
|
||||
local all n8n scram-sha-256
|
||||
local outline outline scram-sha-256
|
||||
local vaultwarden vaultwarden scram-sha-256
|
||||
local zammad zammad scram-sha-256
|
||||
@@ -122,8 +122,8 @@ in {
|
||||
host metabase,az_kpi_raw metabase 127.0.0.1/32 scram-sha-256
|
||||
host metabase,az_kpi_raw metabase ::1/128 scram-sha-256
|
||||
|
||||
host n8n n8n 127.0.0.1/32 scram-sha-256
|
||||
host n8n n8n ::1/128 scram-sha-256
|
||||
host all n8n 127.0.0.1/32 scram-sha-256
|
||||
host all n8n ::1/128 scram-sha-256
|
||||
|
||||
host vaultwarden vaultwarden 127.0.0.1/32 scram-sha-256
|
||||
host vaultwarden vaultwarden ::1/128 scram-sha-256
|
||||
|
||||
+60
-39
@@ -1,43 +1,64 @@
|
||||
{
|
||||
{lib, ...}: let
|
||||
ntfyGrafanaWebhookSecret = ../../secrets/ntfy-grafana-webhook.age;
|
||||
in {
|
||||
age = {
|
||||
secrets = {
|
||||
azion-env = {
|
||||
file = ../../secrets/azion-env.age;
|
||||
secrets =
|
||||
{
|
||||
azion-env = {
|
||||
file = ../../secrets/azion-env.age;
|
||||
};
|
||||
grafana-admin-pw = {
|
||||
file = ../../secrets/grafana-admin-pw.age;
|
||||
owner = "grafana";
|
||||
};
|
||||
grafana-db-password = {
|
||||
file = ../../secrets/grafana-db-password.age;
|
||||
owner = "grafana";
|
||||
};
|
||||
grafana-secret-key = {
|
||||
file = ../../secrets/grafana-secret-key.age;
|
||||
owner = "grafana";
|
||||
};
|
||||
traefik-env = {
|
||||
file = ../../secrets/traefik-env.age;
|
||||
};
|
||||
kestra-config = {
|
||||
file = ../../secrets/kestra-config.age;
|
||||
mode = "644";
|
||||
};
|
||||
kestra-env = {file = ../../secrets/kestra-env.age;};
|
||||
kestra-secrets = {file = ../../secrets/kestra-secrets.age;};
|
||||
n8n-env = {
|
||||
file = ../../secrets/n8n-env-prm.age;
|
||||
};
|
||||
n8n-runner-auth-token = {
|
||||
file = ../../secrets/n8n-runner-auth-token-prm.age;
|
||||
};
|
||||
pgadmin-pw = {
|
||||
file = ../../secrets/pgadmin-pw.age;
|
||||
owner = "pgadmin";
|
||||
};
|
||||
pg-cert = {
|
||||
file = ../../secrets/server.crt.age;
|
||||
owner = "postgres";
|
||||
group = "postgres";
|
||||
mode = "0644";
|
||||
};
|
||||
pg-key = {
|
||||
file = ../../secrets/server.key.age;
|
||||
owner = "postgres";
|
||||
group = "postgres";
|
||||
mode = "0600";
|
||||
};
|
||||
smb-autoablage = {
|
||||
file = ../../secrets/smb-autoablage.age;
|
||||
};
|
||||
}
|
||||
// lib.optionalAttrs (builtins.pathExists ntfyGrafanaWebhookSecret) {
|
||||
ntfy-grafana-webhook = {
|
||||
file = ntfyGrafanaWebhookSecret;
|
||||
mode = "0400";
|
||||
};
|
||||
};
|
||||
traefik-env = {
|
||||
file = ../../secrets/traefik-env.age;
|
||||
};
|
||||
kestra-config = {
|
||||
file = ../../secrets/kestra-config.age;
|
||||
mode = "644";
|
||||
};
|
||||
kestra-env = {file = ../../secrets/kestra-env.age;};
|
||||
kestra-secrets = {file = ../../secrets/kestra-secrets.age;};
|
||||
n8n-env = {
|
||||
file = ../../secrets/n8n-env-prm.age;
|
||||
};
|
||||
n8n-runner-auth-token = {
|
||||
file = ../../secrets/n8n-runner-auth-token-prm.age;
|
||||
};
|
||||
pgadmin-pw = {
|
||||
file = ../../secrets/pgadmin-pw.age;
|
||||
owner = "pgadmin";
|
||||
};
|
||||
pg-cert = {
|
||||
file = ../../secrets/server.crt.age;
|
||||
owner = "postgres";
|
||||
group = "postgres";
|
||||
mode = "0644";
|
||||
};
|
||||
pg-key = {
|
||||
file = ../../secrets/server.key.age;
|
||||
owner = "postgres";
|
||||
group = "postgres";
|
||||
mode = "0600";
|
||||
};
|
||||
smb-autoablage = {
|
||||
file = ../../secrets/smb-autoablage.age;
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
{config, ...}: let
|
||||
serviceName = "azess";
|
||||
servicePort = config.m3ta.ports.get serviceName;
|
||||
in {
|
||||
services.azess = {
|
||||
enable = true;
|
||||
host = "127.0.0.1";
|
||||
port = servicePort;
|
||||
workers = 4;
|
||||
};
|
||||
|
||||
services.traefik.dynamicConfigOptions.http = {
|
||||
services.${serviceName}.loadBalancer.servers = [
|
||||
{url = "http://localhost:${toString servicePort}/";}
|
||||
];
|
||||
|
||||
routers.${serviceName} = {
|
||||
rule = "Host(`azess.l.az-gruppe.com`)";
|
||||
tls = {
|
||||
certResolver = "ionos";
|
||||
};
|
||||
service = serviceName;
|
||||
entrypoints = "websecure";
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -2,8 +2,10 @@
|
||||
imports = [
|
||||
./containers
|
||||
./backup-ingest.nix
|
||||
./azess.nix
|
||||
./azion-scheduler.nix
|
||||
./bpi.nix
|
||||
./monitoring
|
||||
./n8n.nix
|
||||
./netbird.nix
|
||||
./pgadmin.nix
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
...
|
||||
}: let
|
||||
prometheusDatasourceUid = "prometheus";
|
||||
ntfySecretAvailable = builtins.hasAttr "ntfy-grafana-webhook" config.age.secrets;
|
||||
|
||||
prometheusQuery = refId: expr: {
|
||||
inherit refId;
|
||||
datasourceUid = prometheusDatasourceUid;
|
||||
relativeTimeRange = {
|
||||
from = 600;
|
||||
to = 0;
|
||||
};
|
||||
model = {
|
||||
datasource = {
|
||||
type = "prometheus";
|
||||
uid = prometheusDatasourceUid;
|
||||
};
|
||||
editorMode = "code";
|
||||
inherit expr refId;
|
||||
hide = false;
|
||||
instant = true;
|
||||
intervalMs = 1000;
|
||||
legendFormat = "__auto";
|
||||
maxDataPoints = 43200;
|
||||
range = false;
|
||||
};
|
||||
};
|
||||
|
||||
thresholdExpression = {
|
||||
refId,
|
||||
expressionRefId,
|
||||
evaluator,
|
||||
}: {
|
||||
inherit refId;
|
||||
datasourceUid = "__expr__";
|
||||
model = {
|
||||
conditions = [
|
||||
{
|
||||
inherit evaluator;
|
||||
operator.type = "and";
|
||||
query.params = [];
|
||||
reducer = {
|
||||
params = [];
|
||||
type = "avg";
|
||||
};
|
||||
type = "query";
|
||||
}
|
||||
];
|
||||
datasource = {
|
||||
name = "Expression";
|
||||
type = "__expr__";
|
||||
uid = "__expr__";
|
||||
};
|
||||
expression = expressionRefId;
|
||||
hide = false;
|
||||
inherit refId;
|
||||
type = "threshold";
|
||||
};
|
||||
};
|
||||
|
||||
mkAlertRule = {
|
||||
uid,
|
||||
title,
|
||||
expr,
|
||||
for ? "5m",
|
||||
noDataState ? "Alerting",
|
||||
execErrState ? "Error",
|
||||
evaluatorType ? "gt",
|
||||
evaluatorParams ? [0 0],
|
||||
labels ? {},
|
||||
annotations ? {},
|
||||
}: {
|
||||
inherit uid title for noDataState execErrState;
|
||||
condition = "B";
|
||||
data = [
|
||||
(prometheusQuery "A" expr)
|
||||
(thresholdExpression {
|
||||
refId = "B";
|
||||
expressionRefId = "A";
|
||||
evaluator = {
|
||||
type = evaluatorType;
|
||||
params = evaluatorParams;
|
||||
};
|
||||
})
|
||||
];
|
||||
annotations =
|
||||
{
|
||||
summary = title;
|
||||
}
|
||||
// annotations;
|
||||
labels =
|
||||
{
|
||||
service = "monitoring";
|
||||
}
|
||||
// labels;
|
||||
isPaused = false;
|
||||
};
|
||||
in {
|
||||
warnings = lib.optional (!ntfySecretAvailable) ''
|
||||
Grafana alert rules are provisioned, but ntfy notifications are disabled because secrets/ntfy-grafana-webhook.age is missing.
|
||||
Create it as an EnvironmentFile containing: GRAFANA_NTFY_WEBHOOK_URL=https://<ntfy-webhook-url>
|
||||
'';
|
||||
|
||||
systemd.services.grafana.serviceConfig.EnvironmentFile = lib.mkIf ntfySecretAvailable [
|
||||
config.age.secrets."ntfy-grafana-webhook".path
|
||||
];
|
||||
|
||||
services.grafana.provision.alerting = {
|
||||
rules.settings = {
|
||||
apiVersion = 1;
|
||||
groups = [
|
||||
{
|
||||
orgId = 1;
|
||||
name = "az-infrastructure";
|
||||
folder = "Infrastructure";
|
||||
interval = "60s";
|
||||
rules = [
|
||||
(mkAlertRule {
|
||||
uid = "az_host_down";
|
||||
title = "Host down";
|
||||
expr = ''up{job=~"node|node-cld"}'';
|
||||
for = "2m";
|
||||
evaluatorType = "lt";
|
||||
evaluatorParams = [1 0];
|
||||
labels.severity = "critical";
|
||||
annotations.description = "Prometheus cannot scrape a node_exporter target.";
|
||||
})
|
||||
(mkAlertRule {
|
||||
uid = "az_cpu_high";
|
||||
title = "CPU usage high";
|
||||
expr = ''100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)'';
|
||||
for = "10m";
|
||||
evaluatorType = "gt";
|
||||
evaluatorParams = [90 0];
|
||||
labels.severity = "warning";
|
||||
annotations.description = "Average CPU usage has been above 90% for 10 minutes.";
|
||||
})
|
||||
(mkAlertRule {
|
||||
uid = "az_memory_high";
|
||||
title = "Memory usage high";
|
||||
expr = ''100 * (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes))'';
|
||||
for = "10m";
|
||||
evaluatorType = "gt";
|
||||
evaluatorParams = [90 0];
|
||||
labels.severity = "warning";
|
||||
annotations.description = "Memory usage has been above 90% for 10 minutes.";
|
||||
})
|
||||
(mkAlertRule {
|
||||
uid = "az_rootfs_high";
|
||||
title = "Root filesystem usage high";
|
||||
expr = ''100 * (1 - (node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs|ramfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay|squashfs|ramfs"}))'';
|
||||
for = "15m";
|
||||
evaluatorType = "gt";
|
||||
evaluatorParams = [90 0];
|
||||
labels.severity = "warning";
|
||||
annotations.description = "Root filesystem usage has been above 90% for 15 minutes.";
|
||||
})
|
||||
(mkAlertRule {
|
||||
uid = "az_systemd_failed";
|
||||
title = "Systemd units failed";
|
||||
expr = ''sum by(instance) (node_systemd_units{state="failed"})'';
|
||||
for = "5m";
|
||||
evaluatorType = "gt";
|
||||
evaluatorParams = [0 0];
|
||||
labels.severity = "warning";
|
||||
annotations.description = "One or more systemd units are failed on the host.";
|
||||
})
|
||||
(mkAlertRule {
|
||||
uid = "az_http_probe_failed";
|
||||
title = "HTTP probe failed";
|
||||
expr = ''probe_success{job="blackbox_http"}'';
|
||||
for = "2m";
|
||||
evaluatorType = "lt";
|
||||
evaluatorParams = [1 0];
|
||||
labels.severity = "critical";
|
||||
annotations.description = "A blackbox HTTP probe is failing.";
|
||||
})
|
||||
(mkAlertRule {
|
||||
uid = "az_icmp_probe_failed";
|
||||
title = "ICMP probe failed";
|
||||
expr = ''probe_success{job="blackbox_icmp"}'';
|
||||
for = "2m";
|
||||
evaluatorType = "lt";
|
||||
evaluatorParams = [1 0];
|
||||
labels.severity = "warning";
|
||||
annotations.description = "A blackbox ICMP probe is failing.";
|
||||
})
|
||||
(mkAlertRule {
|
||||
uid = "az_tls_cert_expiring";
|
||||
title = "TLS certificate expires soon";
|
||||
expr = ''(probe_ssl_earliest_cert_expiry{job="blackbox_http"} - time()) / 86400'';
|
||||
for = "1h";
|
||||
noDataState = "OK";
|
||||
evaluatorType = "lt";
|
||||
evaluatorParams = [14 0];
|
||||
labels.severity = "warning";
|
||||
annotations.description = "A probed TLS certificate expires in less than 14 days.";
|
||||
})
|
||||
];
|
||||
}
|
||||
];
|
||||
};
|
||||
|
||||
contactPoints.settings = lib.mkIf ntfySecretAvailable {
|
||||
apiVersion = 1;
|
||||
contactPoints = [
|
||||
{
|
||||
orgId = 1;
|
||||
name = "ntfy";
|
||||
receivers = [
|
||||
{
|
||||
uid = "ntfy-webhook";
|
||||
type = "webhook";
|
||||
disableResolveMessage = false;
|
||||
settings = {
|
||||
url = "$GRAFANA_NTFY_WEBHOOK_URL";
|
||||
httpMethod = "POST";
|
||||
};
|
||||
}
|
||||
];
|
||||
}
|
||||
];
|
||||
};
|
||||
|
||||
policies.settings = lib.mkIf ntfySecretAvailable {
|
||||
apiVersion = 1;
|
||||
policies = [
|
||||
{
|
||||
orgId = 1;
|
||||
receiver = "ntfy";
|
||||
group_by = ["alertname" "instance" "target" "severity"];
|
||||
group_wait = "30s";
|
||||
group_interval = "5m";
|
||||
repeat_interval = "4h";
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
# Phase 2 — not active until imported in services/default.nix.
|
||||
# Activate by adding `./blackbox.nix` to hosts/AZ-PRM-1/services/monitoring/default.nix imports.
|
||||
{config, ...}: let
|
||||
blackboxPort = config.m3ta.ports.get "blackbox-exporter";
|
||||
prometheusPort = config.m3ta.ports.get "prometheus";
|
||||
in {
|
||||
services.prometheus.exporters.blackbox = {
|
||||
enable = true;
|
||||
port = blackboxPort;
|
||||
listenAddress = "127.0.0.1";
|
||||
openFirewall = false;
|
||||
configFile = (builtins.toFile "blackbox.yml" ''
|
||||
modules:
|
||||
http_2xx:
|
||||
prober: http
|
||||
timeout: 5s
|
||||
http_post_2xx:
|
||||
prober: http
|
||||
timeout: 5s
|
||||
http:
|
||||
method: POST
|
||||
icmp_ping:
|
||||
prober: icmp
|
||||
timeout: 5s
|
||||
tcp_connect:
|
||||
prober: tcp
|
||||
timeout: 5s
|
||||
'');
|
||||
};
|
||||
|
||||
services.prometheus.scrapeConfigs = [
|
||||
{
|
||||
job_name = "blackbox_http";
|
||||
metrics_path = "/probe";
|
||||
params.module = ["http_2xx"];
|
||||
static_configs = [
|
||||
{
|
||||
targets = [
|
||||
"https://g.l.az-gruppe.com"
|
||||
"https://wf.l.az-gruppe.com"
|
||||
"https://k.l.az-gruppe.com"
|
||||
"https://git.az-gruppe.com"
|
||||
"https://ping.az-gruppe.com"
|
||||
"https://llm.az-gruppe.com"
|
||||
"https://r.az-gruppe.com"
|
||||
];
|
||||
labels = {
|
||||
instance = "AZ-PRM-1-blackbox";
|
||||
};
|
||||
}
|
||||
];
|
||||
relabel_configs = [
|
||||
{
|
||||
source_labels = ["__address__"];
|
||||
target_label = "__param_target";
|
||||
}
|
||||
{
|
||||
source_labels = ["__param_target"];
|
||||
target_label = "target";
|
||||
}
|
||||
{
|
||||
target_label = "__address__";
|
||||
replacement = "localhost:${toString blackboxPort}";
|
||||
}
|
||||
];
|
||||
}
|
||||
{
|
||||
job_name = "blackbox_icmp";
|
||||
metrics_path = "/probe";
|
||||
params.module = ["icmp_ping"];
|
||||
static_configs = [
|
||||
{
|
||||
targets = [
|
||||
"100.91.203.184" # AZ-CLD-1 netbird
|
||||
"192.168.152.97" # SMB/DMS share
|
||||
"192.168.152.98" # SkriptHelper
|
||||
"192.168.152.102" # legacy PRTG (until decommissioned)
|
||||
"1.1.1.1" # upstream DNS reachability
|
||||
"8.8.8.8" # upstream DNS reachability
|
||||
];
|
||||
}
|
||||
];
|
||||
relabel_configs = [
|
||||
{
|
||||
source_labels = ["__address__"];
|
||||
target_label = "__param_target";
|
||||
}
|
||||
{
|
||||
source_labels = ["__param_target"];
|
||||
target_label = "target";
|
||||
}
|
||||
{
|
||||
target_label = "__address__";
|
||||
replacement = "localhost:${toString blackboxPort}";
|
||||
}
|
||||
];
|
||||
}
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
{config, ...}: let
|
||||
nodeExporterPort = config.m3ta.ports.get "node-exporter";
|
||||
cldNetbirdIP = "100.91.203.184";
|
||||
in {
|
||||
services.prometheus.scrapeConfigs = [
|
||||
{
|
||||
job_name = "node-cld";
|
||||
static_configs = [
|
||||
{
|
||||
targets = ["${cldNetbirdIP}:${toString nodeExporterPort}"];
|
||||
labels = {
|
||||
instance = "AZ-CLD-1";
|
||||
};
|
||||
}
|
||||
];
|
||||
}
|
||||
];
|
||||
|
||||
# Allow CLD to reach PRM prometheus scrapes (prometheus initiates connection TO CLD exporter)
|
||||
networking.firewall.extraCommands = ''
|
||||
# No PRM-side firewall change needed: PRM scrapes outbound to CLD:9100.
|
||||
# CLD-side must allow inbound from 100.91.49.26 (PRM netbird IP) — see CLD config.
|
||||
'';
|
||||
}
|
||||
@@ -0,0 +1,722 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"target": {
|
||||
"limit": 100,
|
||||
"matchAny": false,
|
||||
"tags": [],
|
||||
"type": "dashboard"
|
||||
},
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
"links": [],
|
||||
"liveNow": false,
|
||||
"schemaVersion": 39,
|
||||
"style": "dark",
|
||||
"tags": [
|
||||
"az",
|
||||
"demo",
|
||||
"provisioned"
|
||||
],
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"time": {
|
||||
"from": "now-24h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {
|
||||
"refresh_intervals": [
|
||||
"10s",
|
||||
"30s",
|
||||
"1m",
|
||||
"5m",
|
||||
"15m",
|
||||
"30m",
|
||||
"1h"
|
||||
]
|
||||
},
|
||||
"timezone": "browser",
|
||||
"version": 1,
|
||||
"weekStart": "",
|
||||
"uid": "az-blackbox-demo",
|
||||
"title": "AZ Blackbox Probe Demo",
|
||||
"refresh": "30s",
|
||||
"description": "Demo dashboard for HTTP and ICMP probes from Prometheus blackbox_exporter.",
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "HTTP probe success",
|
||||
"type": "stat",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 5,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "short",
|
||||
"decimals": 0,
|
||||
"min": 0,
|
||||
"max": 1,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "red",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "green",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "probe_success{job=\"blackbox_http\"}",
|
||||
"legendFormat": "{{target}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"description": "1 means the last HTTP probe matched the http_2xx module."
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "HTTP 24h availability",
|
||||
"type": "stat",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 5,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 0
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "percent",
|
||||
"decimals": 2,
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "red",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 95
|
||||
},
|
||||
{
|
||||
"color": "green",
|
||||
"value": 99
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "avg_over_time(probe_success{job=\"blackbox_http\"}[24h]) * 100",
|
||||
"legendFormat": "{{target}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "TLS cert days left",
|
||||
"type": "stat",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 5,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 0
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "d",
|
||||
"decimals": 0,
|
||||
"min": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "red",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 14
|
||||
},
|
||||
{
|
||||
"color": "green",
|
||||
"value": 30
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(probe_ssl_earliest_cert_expiry{job=\"blackbox_http\"} - time()) / 86400",
|
||||
"legendFormat": "{{target}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "HTTP latency",
|
||||
"type": "timeseries",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 5
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "s",
|
||||
"decimals": 3,
|
||||
"min": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 1
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 3
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "probe_duration_seconds{job=\"blackbox_http\"}",
|
||||
"legendFormat": "{{target}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "HTTP status code",
|
||||
"type": "timeseries",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 5
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "short",
|
||||
"decimals": 0,
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "probe_http_status_code{job=\"blackbox_http\"}",
|
||||
"legendFormat": "{{target}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "ICMP probe success",
|
||||
"type": "stat",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 5,
|
||||
"w": 8,
|
||||
"x": 0,
|
||||
"y": 13
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "short",
|
||||
"decimals": 0,
|
||||
"min": 0,
|
||||
"max": 1,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "red",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "green",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "probe_success{job=\"blackbox_icmp\"}",
|
||||
"legendFormat": "{{target}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"description": "1 means the last ICMP probe succeeded."
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "ICMP 24h availability",
|
||||
"type": "stat",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 5,
|
||||
"w": 8,
|
||||
"x": 8,
|
||||
"y": 13
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "percent",
|
||||
"decimals": 2,
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "red",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 95
|
||||
},
|
||||
{
|
||||
"color": "green",
|
||||
"value": 99
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "avg_over_time(probe_success{job=\"blackbox_icmp\"}[24h]) * 100",
|
||||
"legendFormat": "{{target}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"title": "ICMP last latency",
|
||||
"type": "stat",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 5,
|
||||
"w": 8,
|
||||
"x": 16,
|
||||
"y": 13
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "s",
|
||||
"decimals": 3,
|
||||
"min": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 0.1
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 0.3
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "probe_duration_seconds{job=\"blackbox_icmp\"}",
|
||||
"legendFormat": "{{target}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"title": "ICMP latency",
|
||||
"type": "timeseries",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 18
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "s",
|
||||
"decimals": 3,
|
||||
"min": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 0.1
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 0.3
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "probe_duration_seconds{job=\"blackbox_icmp\"}",
|
||||
"legendFormat": "{{target}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"title": "Probe success timeline",
|
||||
"type": "timeseries",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 18
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "short",
|
||||
"decimals": 0,
|
||||
"min": 0,
|
||||
"max": 1,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "red",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "green",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "probe_success{job=\"blackbox_http\"}",
|
||||
"legendFormat": "HTTP {{target}}",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "probe_success{job=\"blackbox_icmp\"}",
|
||||
"legendFormat": "ICMP {{target}}",
|
||||
"refId": "B"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"title": "DNS + connect + TLS phase timing",
|
||||
"type": "timeseries",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 26
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "s",
|
||||
"decimals": 3,
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "probe_http_duration_seconds{job=\"blackbox_http\", phase=\"resolve\"}",
|
||||
"legendFormat": "{{target}} resolve",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "probe_http_duration_seconds{job=\"blackbox_http\", phase=\"connect\"}",
|
||||
"legendFormat": "{{target}} connect",
|
||||
"refId": "B"
|
||||
},
|
||||
{
|
||||
"expr": "probe_http_duration_seconds{job=\"blackbox_http\", phase=\"tls\"}",
|
||||
"legendFormat": "{{target}} tls",
|
||||
"refId": "C"
|
||||
},
|
||||
{
|
||||
"expr": "probe_http_duration_seconds{job=\"blackbox_http\", phase=\"processing\"}",
|
||||
"legendFormat": "{{target}} processing",
|
||||
"refId": "D"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,731 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"target": {
|
||||
"limit": 100,
|
||||
"matchAny": false,
|
||||
"tags": [],
|
||||
"type": "dashboard"
|
||||
},
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
"links": [],
|
||||
"liveNow": false,
|
||||
"schemaVersion": 39,
|
||||
"style": "dark",
|
||||
"tags": [
|
||||
"az",
|
||||
"demo",
|
||||
"provisioned"
|
||||
],
|
||||
"templating": {
|
||||
"list": []
|
||||
},
|
||||
"time": {
|
||||
"from": "now-6h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {
|
||||
"refresh_intervals": [
|
||||
"10s",
|
||||
"30s",
|
||||
"1m",
|
||||
"5m",
|
||||
"15m",
|
||||
"30m",
|
||||
"1h"
|
||||
]
|
||||
},
|
||||
"timezone": "browser",
|
||||
"version": 1,
|
||||
"weekStart": "",
|
||||
"uid": "az-node-fleet-demo",
|
||||
"title": "AZ Node Fleet Demo",
|
||||
"refresh": "30s",
|
||||
"description": "Demo dashboard for node_exporter metrics on AZ-PRM-1 and AZ-CLD-1. Uses the default Prometheus datasource provisioned by NixOS.",
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"title": "Scrape up",
|
||||
"type": "stat",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "short",
|
||||
"decimals": 0,
|
||||
"min": 0,
|
||||
"max": 1,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "red",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "green",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "up{job=~\"node|node-cld\"}",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"description": "1 means Prometheus can scrape the host."
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"title": "Uptime",
|
||||
"type": "stat",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 6,
|
||||
"y": 0
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "d",
|
||||
"decimals": 1,
|
||||
"min": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(time() - node_boot_time_seconds) / 86400",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"title": "CPU busy",
|
||||
"type": "gauge",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 12,
|
||||
"y": 0
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "percent",
|
||||
"decimals": 1,
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 70
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 90
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showThresholdLabels": false,
|
||||
"showThresholdMarkers": true
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "100 - (avg by(instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval])) * 100)",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"title": "Memory used",
|
||||
"type": "gauge",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 18,
|
||||
"y": 0
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "percent",
|
||||
"decimals": 1,
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 75
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 90
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showThresholdLabels": false,
|
||||
"showThresholdMarkers": true
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "100 * (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes))",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"title": "CPU busy by host",
|
||||
"type": "timeseries",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 4
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "percent",
|
||||
"decimals": 1,
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 70
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 90
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "100 - (avg by(instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval])) * 100)",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"title": "Load average (5m)",
|
||||
"type": "timeseries",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 4
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "short",
|
||||
"decimals": 2,
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "node_load5",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "Memory used",
|
||||
"type": "timeseries",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 12
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "percent",
|
||||
"decimals": 1,
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 75
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 90
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "100 * (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes))",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"title": "Root filesystem used",
|
||||
"type": "timeseries",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 12
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "percent",
|
||||
"decimals": 1,
|
||||
"min": 0,
|
||||
"max": 100,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 80
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 92
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "100 * (1 - (node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay|squashfs|ramfs\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay|squashfs|ramfs\"}))",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"title": "Network throughput",
|
||||
"type": "timeseries",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 20
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "Bps",
|
||||
"decimals": 1,
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by(instance) (rate(node_network_receive_bytes_total{device!~\"lo|veth.*|docker.*|br-.*\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{instance}} RX",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "sum by(instance) (rate(node_network_transmit_bytes_total{device!~\"lo|veth.*|docker.*|br-.*\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{instance}} TX",
|
||||
"refId": "B"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"title": "Disk IO",
|
||||
"type": "timeseries",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 20
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "Bps",
|
||||
"decimals": 1,
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"legend": {
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "multi",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by(instance) (rate(node_disk_read_bytes_total[$__rate_interval]))",
|
||||
"legendFormat": "{{instance}} read",
|
||||
"refId": "A"
|
||||
},
|
||||
{
|
||||
"expr": "sum by(instance) (rate(node_disk_written_bytes_total[$__rate_interval]))",
|
||||
"legendFormat": "{{instance}} write",
|
||||
"refId": "B"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"title": "Failed systemd units",
|
||||
"type": "stat",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 28
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "short",
|
||||
"decimals": 0,
|
||||
"min": 0,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum by(instance) (node_systemd_units{state=\"failed\"})",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"description": "Requires node_exporter systemd collector."
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"title": "Filesystem free bytes",
|
||||
"type": "stat",
|
||||
"datasource": null,
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 28
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {},
|
||||
"mappings": [],
|
||||
"unit": "bytes",
|
||||
"decimals": 1,
|
||||
"min": 0
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay|squashfs|ramfs\"}",
|
||||
"legendFormat": "{{instance}}",
|
||||
"refId": "A"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
{...}: {
|
||||
imports = [
|
||||
./alerting.nix
|
||||
./blackbox.nix
|
||||
./cld-scrape.nix
|
||||
./exporters.nix
|
||||
./grafana.nix
|
||||
./prometheus.nix
|
||||
];
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{config, ...}: let
|
||||
nodeExporterPort = config.m3ta.ports.get "node-exporter";
|
||||
in {
|
||||
services.prometheus.exporters.node = {
|
||||
enable = true;
|
||||
port = nodeExporterPort;
|
||||
listenAddress = "127.0.0.1";
|
||||
enabledCollectors = [
|
||||
"systemd"
|
||||
"diskstats"
|
||||
"filesystem"
|
||||
];
|
||||
openFirewall = false; # localhost only; prometheus scrapes via 127.0.0.1
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
{config, ...}: let
|
||||
serviceName = "grafana";
|
||||
servicePort = config.m3ta.ports.get serviceName;
|
||||
prometheusPort = config.m3ta.ports.get "prometheus";
|
||||
in {
|
||||
services.grafana = {
|
||||
enable = true;
|
||||
settings = {
|
||||
server = {
|
||||
http_addr = "127.0.0.1";
|
||||
http_port = servicePort;
|
||||
domain = "g.l.az-gruppe.com";
|
||||
root_url = "https://g.l.az-gruppe.com";
|
||||
serve_from_sub_path = false;
|
||||
};
|
||||
database = {
|
||||
type = "postgres";
|
||||
host = "127.0.0.1";
|
||||
name = "grafana";
|
||||
user = "grafana";
|
||||
password = "$__file{${config.age.secrets.grafana-db-password.path}}";
|
||||
ssl_mode = "disable";
|
||||
};
|
||||
security = {
|
||||
admin_user = "admin";
|
||||
# Grafana file-provider: $__file{<path>} reads the raw secret from the file.
|
||||
# The agenix secrets must contain ONLY the raw value (no KEY= prefix).
|
||||
admin_password = "$__file{${config.age.secrets.grafana-admin-pw.path}}";
|
||||
secret_key = "$__file{${config.age.secrets.grafana-secret-key.path}}";
|
||||
disable_gravatar = true;
|
||||
};
|
||||
};
|
||||
provision = {
|
||||
datasources.settings.datasources = [
|
||||
{
|
||||
name = "Prometheus";
|
||||
uid = "prometheus";
|
||||
type = "prometheus";
|
||||
url = "http://localhost:${toString prometheusPort}";
|
||||
isDefault = true;
|
||||
access = "proxy";
|
||||
jsonData.timeInterval = "15s";
|
||||
}
|
||||
# Phase 3: add Loki datasource here once services.loki is enabled.
|
||||
];
|
||||
dashboards.settings.providers = [
|
||||
{
|
||||
name = "default";
|
||||
options.path = "/etc/grafana-dashboards";
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
# Dashboards directory (Phase 1: empty placeholder, JSON files added later)
|
||||
environment.etc."grafana-dashboards".source = ./. + "/dashboards";
|
||||
|
||||
systemd.services.grafana = {
|
||||
after = ["postgresql.service"];
|
||||
wants = ["postgresql.service"];
|
||||
};
|
||||
|
||||
# Traefik configuration specific to grafana
|
||||
services.traefik.dynamicConfigOptions.http = {
|
||||
services.${serviceName}.loadBalancer.servers = [
|
||||
{
|
||||
url = "http://localhost:${toString servicePort}/";
|
||||
}
|
||||
];
|
||||
|
||||
routers.${serviceName} = {
|
||||
rule = "Host(`g.l.az-gruppe.com`)";
|
||||
tls = {
|
||||
certResolver = "ionos";
|
||||
};
|
||||
service = serviceName;
|
||||
entrypoints = "websecure";
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
{config, ...}: let
|
||||
prometheusPort = config.m3ta.ports.get "prometheus";
|
||||
nodeExporterPort = config.m3ta.ports.get "node-exporter";
|
||||
in {
|
||||
services.prometheus = {
|
||||
enable = true;
|
||||
port = prometheusPort;
|
||||
listenAddress = "127.0.0.1";
|
||||
retentionTime = "30d";
|
||||
|
||||
scrapeConfigs = [
|
||||
{
|
||||
job_name = "node";
|
||||
static_configs = [
|
||||
{
|
||||
targets = ["localhost:${toString nodeExporterPort}"];
|
||||
labels = {
|
||||
instance = "AZ-PRM-1";
|
||||
};
|
||||
}
|
||||
];
|
||||
}
|
||||
# Phase 2: add AZ-CLD-1 node target (scrape over netbird 100.x).
|
||||
# Phase 2: add blackbox_exporter, podman/cadvisor targets.
|
||||
# Phase 2: enable remote-write receiver (extraFlags) for Windows clients.
|
||||
];
|
||||
};
|
||||
}
|
||||
@@ -31,6 +31,10 @@
|
||||
CREATE USER vaultwarden WITH ENCRYPTED PASSWORD 'n8n';
|
||||
CREATE DATABASE vaultwarden;
|
||||
ALTER DATABASE vaultwarden OWNER to vaultwarden;
|
||||
|
||||
CREATE USER grafana WITH ENCRYPTED PASSWORD 'grafana';
|
||||
CREATE DATABASE grafana;
|
||||
ALTER DATABASE grafana OWNER to grafana;
|
||||
'';
|
||||
authentication = pkgs.lib.mkOverride 10 ''
|
||||
# Local connections (Unix socket)
|
||||
@@ -48,6 +52,10 @@
|
||||
host baserow baserow 10.89.0.0/24 scram-sha-256
|
||||
host kestra kestra 10.89.0.0/24 scram-sha-256
|
||||
|
||||
# Grafana (local socket / localhost only)
|
||||
host grafana grafana 127.0.0.1/32 scram-sha-256
|
||||
host grafana grafana ::1/128 scram-sha-256
|
||||
|
||||
# Deny all other connections
|
||||
host all all 0.0.0.0/0 reject
|
||||
host all all ::/0 reject
|
||||
|
||||
@@ -24,6 +24,7 @@
|
||||
baserow = 3050;
|
||||
frappe-lms = 3052;
|
||||
snipe-it = 3053;
|
||||
azess = 3054;
|
||||
|
||||
librechat = 3040;
|
||||
librechat-dev = 3141;
|
||||
@@ -41,6 +42,12 @@
|
||||
postgres = 5432;
|
||||
pgadmin = 5050;
|
||||
mysql = 3306;
|
||||
|
||||
# Observability stack (AZ-PRM-1)
|
||||
grafana = 3060;
|
||||
prometheus = 9090;
|
||||
node-exporter = 9100;
|
||||
blackbox-exporter = 9115;
|
||||
};
|
||||
|
||||
hostOverrides = {
|
||||
|
||||
+37
-23
@@ -1,26 +1,40 @@
|
||||
# {prev}:
|
||||
# prev.n8n.overrideAttrs (oldAttrs: rec {
|
||||
# version = "1.112.6";
|
||||
# Adds `libphonenumber-js` to n8n's node_modules so it can be require()'d
|
||||
# from Code nodes running in the Task Runner.
|
||||
#
|
||||
# Prerequisite on the n8n service:
|
||||
# N8N_RUNNERS_EXTERNAL_ALLOW = "libphonenumber-js";
|
||||
#
|
||||
# libphonenumber-js has zero runtime + peer dependencies, so a plain tarball
|
||||
# unpack into the shared node_modules hierarchy is sufficient.
|
||||
{prev}: let
|
||||
libphonenumber-js = prev.stdenv.mkDerivation rec {
|
||||
pname = "libphonenumber-js";
|
||||
version = "1.13.8";
|
||||
|
||||
# src = prev.fetchFromGitHub {
|
||||
# owner = "n8n-io";
|
||||
# repo = "n8n";
|
||||
# rev = "n8n@${version}";
|
||||
# hash = "sha256-r/MCU/S1kkKQPkhmp9ZHTtgZxMu5TFCl5Yejp73gATw=";
|
||||
# };
|
||||
src = prev.fetchurl {
|
||||
url = "https://registry.npmjs.org/${pname}/-/${pname}-${version}.tgz";
|
||||
hash = "sha256-SysWDKlbXgbe441Sd4pO+k+F1y/UC49a1KL+DcFWBIA=";
|
||||
};
|
||||
|
||||
# pnpmDeps = prev.pnpm_10.fetchDeps {
|
||||
# pname = oldAttrs.pname;
|
||||
# inherit version src;
|
||||
# fetcherVersion = 1;
|
||||
# hash = "sha256-j+HJhvzrcu8JsezcFJxfgteOgTspWQb2ZSN2fEl7Voo=";
|
||||
# };
|
||||
dontConfigure = true;
|
||||
dontBuild = true;
|
||||
|
||||
# nativeBuildInputs =
|
||||
# builtins.map
|
||||
# (input:
|
||||
# if input == prev.pnpm_9.configHook
|
||||
# then prev.pnpm_10.configHook
|
||||
# else input)
|
||||
# oldAttrs.nativeBuildInputs;
|
||||
# })
|
||||
installPhase = ''
|
||||
runHook preInstall
|
||||
mkdir -p $out/lib/node_modules/${pname}
|
||||
cp -r * $out/lib/node_modules/${pname}/
|
||||
runHook postInstall
|
||||
'';
|
||||
};
|
||||
in
|
||||
prev.n8n.overrideAttrs (oldAttrs: {
|
||||
postInstall =
|
||||
(oldAttrs.postInstall or "")
|
||||
+ ''
|
||||
# n8n ships a pnpm stub symlink (libphonenumber-js -> empty-npm-package).
|
||||
# Remove it and place the real package there instead.
|
||||
rm -rf $out/lib/n8n/node_modules/libphonenumber-js
|
||||
cp -r ${libphonenumber-js}/lib/node_modules/libphonenumber-js \
|
||||
$out/lib/n8n/node_modules/
|
||||
'';
|
||||
})
|
||||
|
||||
@@ -53,4 +53,8 @@ in {
|
||||
"secrets/outline-key.age".publicKeys = systems ++ users;
|
||||
"secrets/ref-key.age".publicKeys = systems ++ users;
|
||||
"secrets/exa-key.age".publicKeys = systems ++ users;
|
||||
"secrets/grafana-admin-pw.age".publicKeys = systems ++ users;
|
||||
"secrets/grafana-db-password.age".publicKeys = systems ++ users;
|
||||
"secrets/grafana-secret-key.age".publicKeys = systems ++ users;
|
||||
"secrets/ntfy-grafana-webhook.age".publicKeys = systems ++ users;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHNzaC1lZDI1NTE5IFpoVnNlZyBEQ1Ax
|
||||
R2hMZkt2a0cwYlZHYjFhVG5kaHB5MlEwMVVaK2ZQdWdVTEx0cUdjCmtXbUgyRTJt
|
||||
SXFvQ1F2WWxoU1RYcG1NSmd4QnU4Y3UxaFVsZkZZc0RINU0KLT4gc3NoLWVkMjU1
|
||||
MTkgU3JIYXFBIEdZa0hicVVxbExVOHdkNFNROUlkSmg2ak5UVVREK1ZMZkt1bEpD
|
||||
ejdOaE0KYjBrcjk0UG5LV2orL3ZISTZXV0h6cnZ3OThIYjQ0Q1RZUVd6L2FEeERa
|
||||
cwotPiBzc2gtZWQyNTUxOSBsR3FWWmcgZ21JMlRGUW81U3BhbUYrUjU3Y1NGNm03
|
||||
RElwbUpVMmhFalF5ZjZMYmx5Ywovc2ErZS85U0V0T2diNlVYM2VIR3lkZnVrRUY4
|
||||
N0oyQUlNTzB6Zng3ODNnCi0+IHNzaC1lZDI1NTE5IENTTXloZyBBVEVYeStOZ1hM
|
||||
K2d0S3ZqRmZ2TVpUejJwd0dCVEw5ZE5WWkRlcXhKbVEwCjFKRU5DRDd3aHN1a1VY
|
||||
VzVaMU01S21YMVZ6YndKUUQ4emg3TFN4djVOREUKLT4gdDctZ3JlYXNlIHw9VCBj
|
||||
dGUgRk03NSdtUXwgSQoza3BNSnI2eThMc3F4eVBSY0dhVi92K3BKcC9EaWVZRWxu
|
||||
N0VKZGVLWG1SYWhXWTZ4TGQ1Y21VTFY0ZmNlQQotLS0gcHdhMHcyT2x1V2RmRWV5
|
||||
OG1yTFhnVFdlS2RqUkd0YlVFczRzWmwyYjJwNAp0z9rmy4jPvnkW6kA1OBLpvXby
|
||||
Zt/VxkA0aj8fgsVoNRFnGWaVDFcynLR0ZJMNYQ==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
@@ -0,0 +1,16 @@
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHNzaC1lZDI1NTE5IFpoVnNlZyA5aVNn
|
||||
TUliejFZc2tnZExIWjRDWE9NTWJVVWlOWGtFMUF0L0U4TkNSUnh3Ck9Ock1IVW8x
|
||||
Zk4wVG5pczY4M09ISnhuYU1lS2NyNkxzZnpPdFI1SmN6REkKLT4gc3NoLWVkMjU1
|
||||
MTkgU3JIYXFBIEQrckhQTWJzM1lrVTlOeGdoQ3BWN0djd3Y1Q2E5aXd2VU1scGF1
|
||||
UzM1aHMKdzNPQWZpeE5DSGllcXJSUzZEN2JGYitxQjhXa3B2NVRIeHNNbTMxamFm
|
||||
OAotPiBzc2gtZWQyNTUxOSBsR3FWWmcgUG55RWc4Y0RlOHI0bFM4akdIYjhwTCt2
|
||||
R09ydXBCSm44bTMvak84TTFVRQorWmVzVXBsaFNaVFk1enRWaTdGNmQ2eGt2ZFpy
|
||||
UDVhYWllSkJ6RGJhVGxNCi0+IHNzaC1lZDI1NTE5IENTTXloZyBFcDF4dEhlUjhG
|
||||
alVMUVlScjFsQys5VjhoMzlMNzNJOG9GOXp1bnQxdjM4CnJYQmoxanpnRW1YWFJN
|
||||
Q28ybGcvc1YwWjFZNFc3eHlaRjBlTDIvcXlXM0UKLT4gTlotdnotZ3JlYXNlIGhT
|
||||
Z0gxViA7NkUjMWhRIE5rc35pcVA/IEtrQG8zen18CjVRM0cwV2Z3bXlpS2JGb3lO
|
||||
WUNRRDdpNngrZEJBdGVQcW40cjFiTGlMdFFSYmh1QlJyTjIKLS0tIFl1YlloL0tC
|
||||
ZVBZbVJqOFp2OUtMTnUvUWRSL3hUc0VSZ0tCb2lYeU4wMW8KB077DPDfkB1bje7z
|
||||
blXS/m1slMnw11sdLOY5FnQV7m5JDuK15e0iruhqPvm0TADR
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
@@ -0,0 +1,18 @@
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHNzaC1lZDI1NTE5IFpoVnNlZyBqUUd1
|
||||
VXo5ZGo0bnczUGd1cW54ZEdaLzdIcHdNcWo1N3B3YUZBUXo2eTJJCnZMSDQ1WnNF
|
||||
UHF1c251cExHc3U2bzgvSzZLMGdySUdMcDNveVJRMDd1UkEKLT4gc3NoLWVkMjU1
|
||||
MTkgU3JIYXFBIHFkeC9jTTB0MUo0OExabHdyb0ZScTRXcUJmNjRUVG5EMEljVHlo
|
||||
TkdtRUEKS2lxTU5rTW9tSVpzNFgxK2k3dllNNk9kNmIzU3J4OC83YktFZ3VEbkVQ
|
||||
ZwotPiBzc2gtZWQyNTUxOSBsR3FWWmcgUTAzQWkvSUYxcldNUjlRd3VnalprTFZT
|
||||
UGFGckFYT3h2bjUzeEFUZWdqSQpGQS9WRGFoOUtwTXpQdG9NUThNNkUvWDIxZ1hC
|
||||
ZDlKMUxtUEdVU3Zzb3dRCi0+IHNzaC1lZDI1NTE5IENTTXloZyBqelcyRlBFMUh0
|
||||
ZDNtZWx0SkdsNTVlL1c0aDloVXJxSUlHbEtnVGN3RUMwCmhxUS9zcjNVQ0NkRElO
|
||||
YzVzUFlrQktRelBRYmt3anVOL1lSLytNaEVnVlkKLT4gXnBlI34iJVItZ3JlYXNl
|
||||
IDYxR3dbSlsrCmg5QUhLS3AzUHdpWFFUYjgvQ2VTSGlTWkVqUWRrdDRUaVQ4T3ZI
|
||||
eWN0Mnp2OFJMa1ZOT2VFTWMrOHhjY09sWG4KUGlCOXpYeTVsOGVyd1NkY0RqeU5o
|
||||
MzNHcy9nS1BNR0orUXo1UDdySllwRG1HbS9GUlRmYmNoc1YKLS0tIGZNemhmcE4x
|
||||
N0tyUzg2VHJaajEydnp6SFdCQXV2WlNNYyswNVJETm90VVUKdtdcfrVdawFwMKZ1
|
||||
FUlQvH+ef1iBjuDIX2zuy3Fvtwu8lo3qaOGJTT7lP+bPXXjtKdiz02DkhhD2HM12
|
||||
i16mfLP3bvMms+W68r87cxx/zFq7O1D44cvYY2a1Z9ZmU0LZsw==
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
@@ -0,0 +1,17 @@
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHNzaC1lZDI1NTE5IFpoVnNlZyBMTnFZ
|
||||
NkozaWQvUklzemlpaHhCeVU3YmZ2OW5BRUY2Z1k5RW91WC84MzJ3Cmk1VTdoSVB3
|
||||
MXdFKzF3cEVoaVF0UlUwaERkWmJkSDAxbGt0SGJ3RXltcDQKLT4gc3NoLWVkMjU1
|
||||
MTkgU3JIYXFBIDFCZmo1NTJQRVlpZCt4MlRMbVV5anZnUUlUWUp6aDhDV3FlYmxO
|
||||
RnhCVXcKLzhyWFE4R2VDWkhzVzBpaCtQVFowN0YxenNIWjNKYld4RkRzaFNZb0tv
|
||||
TQotPiBzc2gtZWQyNTUxOSBsR3FWWmcgSDlQdElQcXI1MXRrMVY0cVppc2VEN1ZQ
|
||||
eVJrSkM5MlVjdzF2TTN6QmR5SQpEUFpEY0trRXB2RHFUZEcrVzUvdW5JWFZFOGNP
|
||||
THVrZTlXMWVQaUlCaU00Ci0+IHNzaC1lZDI1NTE5IENTTXloZyA1V1V4VmRHSTNP
|
||||
cGJqK3F6ckwzSEhMMUQxZEJPVUYxRTI3TGdjMzRZdGhJClpja3FPNm4wQ2VSTlZu
|
||||
ak1zcUljUlF3enNCMy9XdVpxSThxTEN3RlRVODAKLT4gJmBDT1QpLi1ncmVhc2Ug
|
||||
VHpTSUJnIDJtYihWIEx9aXBJemEKSStWQnBkdTRvakkzUFRVCi0tLSBybVMxYVNR
|
||||
b3NDc0Z5NXUrNnNDM2p6aGdMMjFkbzlGTXZJZjVlWGkvZVZvCmPEPGsFFpaUVOfU
|
||||
3uk2FMi20oSgKSFrVCz0YkhJXXmSVgA2qX7QGnkVqb8ffzAJeYS1xlgj/cAjezux
|
||||
RTDP4urrTRBnR8egcn6j9/8YoHFJVPmKVXXndIt0MqAsYjeyqWYfKIvaJMA1GSMU
|
||||
2Eila212uzU=
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
Reference in New Issue
Block a user