diff --git a/hosts/AZ-CLD-1/secrets.nix b/hosts/AZ-CLD-1/secrets.nix index c83cd2a..d8d9a7f 100644 --- a/hosts/AZ-CLD-1/secrets.nix +++ b/hosts/AZ-CLD-1/secrets.nix @@ -4,6 +4,12 @@ traefik-env = { file = ../../secrets/traefik-env.age; }; + monitoring-loki-remote-env = { + file = ../../secrets/monitoring-loki-remote-env.age; + owner = "root"; + group = "root"; + mode = "0400"; + }; baserow-env = { file = ../../secrets/baserow-env.age; }; diff --git a/hosts/AZ-CLD-1/services/containers/litellm.nix b/hosts/AZ-CLD-1/services/containers/litellm.nix index 32c2389..69c0796 100644 --- a/hosts/AZ-CLD-1/services/containers/litellm.nix +++ b/hosts/AZ-CLD-1/services/containers/litellm.nix @@ -12,6 +12,7 @@ in { DO_NOT_TRACK = "True"; SCARF_NO_ANALYTICS = "True"; STORE_MODEL_IN_DB = "True"; + LITELLM_LOG = "json"; }; volumes = ["/var/lib/litellm/config.yaml:/app/config.yaml"]; extraOptions = ["--add-host=postgres:10.89.0.1" "--ip=10.89.0.30" "--network=web"]; diff --git a/hosts/AZ-CLD-1/services/monitoring/alloy.nix b/hosts/AZ-CLD-1/services/monitoring/alloy.nix new file mode 100644 index 0000000..87fe669 --- /dev/null +++ b/hosts/AZ-CLD-1/services/monitoring/alloy.nix @@ -0,0 +1,100 @@ +{ + config, + lib, + ... +}: let + lokiPort = config.m3ta.ports.get "loki"; + alloyPort = config.m3ta.ports.get "alloy"; + prmNetbirdIP = "100.91.49.26"; +in { + services.alloy = { + enable = true; + configPath = "/etc/alloy"; + environmentFile = config.age.secrets.monitoring-loki-remote-env.path; + extraFlags = [ + "--server.http.listen-addr=127.0.0.1:${toString alloyPort}" + "--disable-reporting" + ]; + }; + + environment.etc."alloy/config.alloy".text = '' + loki.relabel "journal" { + forward_to = [] + + rule { + source_labels = ["__journal__systemd_unit"] + target_label = "unit" + } + + rule { + source_labels = ["__journal_priority_keyword"] + target_label = "level" + } + + rule { + source_labels = ["__journal_syslog_identifier"] + target_label = "syslog_identifier" + } + } + + loki.source.journal "system" { + forward_to = [loki.process.journal.receiver] + relabel_rules = loki.relabel.journal.rules + labels = { + host = "AZ-CLD-1", + environment = "prod", + } + } + + loki.process "journal" { + forward_to = [loki.write.prm.receiver] + + stage.json { + expressions = { + app = "app", + service = "service", + level = "level", + trace_id = "trace_id", + session_id = "session_id", + request_id = "request_id", + model = "model", + message = "message", + } + drop_malformed = false + } + + stage.labels { + values = { + app = "", + service = "", + level = "", + } + } + + stage.structured_metadata { + values = { + trace_id = "", + session_id = "", + request_id = "", + model = "", + } + } + } + + loki.write "prm" { + endpoint { + url = "http://${prmNetbirdIP}:${toString lokiPort}/loki/api/v1/push" + basic_auth { + username = "alloy" + password = sys.env("LOKI_BASIC_AUTH_PASSWORD") + } + } + } + ''; + + systemd.services.alloy = { + wants = ["network-online.target"]; + after = ["network-online.target"]; + serviceConfig.SupplementaryGroups = lib.mkAfter ["adm"]; + }; +} diff --git a/hosts/AZ-CLD-1/services/monitoring/default.nix b/hosts/AZ-CLD-1/services/monitoring/default.nix index b438dda..6e4b6b4 100644 --- a/hosts/AZ-CLD-1/services/monitoring/default.nix +++ b/hosts/AZ-CLD-1/services/monitoring/default.nix @@ -1,6 +1,7 @@ # Phase 2 — activate by adding `./monitoring` to hosts/AZ-CLD-1/services/default.nix imports. {...}: { imports = [ + ./alloy.nix ./node-exporter.nix ]; } diff --git a/hosts/AZ-PRM-1/secrets.nix b/hosts/AZ-PRM-1/secrets.nix index bfc46d6..ee3b2a2 100644 --- a/hosts/AZ-PRM-1/secrets.nix +++ b/hosts/AZ-PRM-1/secrets.nix @@ -19,6 +19,11 @@ in { file = ../../secrets/grafana-secret-key.age; owner = "grafana"; }; + monitoring-loki-htpasswd = { + file = ../../secrets/monitoring-loki-htpasswd.age; + owner = "traefik"; + mode = "0400"; + }; traefik-env = { file = ../../secrets/traefik-env.age; }; diff --git a/hosts/AZ-PRM-1/services/monitoring/alloy.nix b/hosts/AZ-PRM-1/services/monitoring/alloy.nix new file mode 100644 index 0000000..3656c1c --- /dev/null +++ b/hosts/AZ-PRM-1/services/monitoring/alloy.nix @@ -0,0 +1,92 @@ +{ + config, + lib, + ... +}: let + lokiPort = config.m3ta.ports.get "loki"; + alloyPort = config.m3ta.ports.get "alloy"; +in { + services.alloy = { + enable = true; + configPath = "/etc/alloy"; + extraFlags = [ + "--server.http.listen-addr=127.0.0.1:${toString alloyPort}" + "--disable-reporting" + ]; + }; + + environment.etc."alloy/config.alloy".text = '' + loki.relabel "journal" { + forward_to = [] + + rule { + source_labels = ["__journal__systemd_unit"] + target_label = "unit" + } + + rule { + source_labels = ["__journal_priority_keyword"] + target_label = "level" + } + + rule { + source_labels = ["__journal_syslog_identifier"] + target_label = "syslog_identifier" + } + } + + loki.source.journal "system" { + forward_to = [loki.process.journal.receiver] + relabel_rules = loki.relabel.journal.rules + labels = { + host = "AZ-PRM-1", + environment = "prod", + } + } + + loki.process "journal" { + forward_to = [loki.write.local.receiver] + + stage.json { + expressions = { + app = "app", + service = "service", + level = "level", + trace_id = "trace_id", + session_id = "session_id", + n8n_execution_id = "execution_id", + workflow_id = "workflow_id", + execution_id = "execution_id", + message = "message", + } + drop_malformed = false + } + + stage.labels { + values = { + app = "", + service = "", + level = "", + } + } + + stage.structured_metadata { + values = { + trace_id = "", + session_id = "", + n8n_execution_id = "", + workflow_id = "", + execution_id = "", + } + } + } + + loki.write "local" { + endpoint { + url = "http://127.0.0.1:${toString lokiPort}/loki/api/v1/push" + } + } + ''; + + systemd.services.alloy.serviceConfig.SupplementaryGroups = lib.mkAfter ["adm"]; +} diff --git a/hosts/AZ-PRM-1/services/monitoring/blackbox.nix b/hosts/AZ-PRM-1/services/monitoring/blackbox.nix index 255e91b..45abc1a 100644 --- a/hosts/AZ-PRM-1/services/monitoring/blackbox.nix +++ b/hosts/AZ-PRM-1/services/monitoring/blackbox.nix @@ -9,7 +9,7 @@ in { port = blackboxPort; listenAddress = "127.0.0.1"; openFirewall = false; - configFile = (builtins.toFile "blackbox.yml" '' + configFile = builtins.toFile "blackbox.yml" '' modules: http_2xx: prober: http @@ -25,7 +25,7 @@ in { tcp_connect: prober: tcp timeout: 5s - ''); + ''; }; services.prometheus.scrapeConfigs = [ diff --git a/hosts/AZ-PRM-1/services/monitoring/dashboards/az-loki-usage-demo.json b/hosts/AZ-PRM-1/services/monitoring/dashboards/az-loki-usage-demo.json new file mode 100644 index 0000000..6b91609 --- /dev/null +++ b/hosts/AZ-PRM-1/services/monitoring/dashboards/az-loki-usage-demo.json @@ -0,0 +1,764 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "liveNow": false, + "schemaVersion": 39, + "style": "dark", + "tags": [ + "az", + "demo", + "loki", + "logs", + "provisioned" + ], + "templating": { + "list": [ + { + "name": "host", + "label": "Host", + "type": "query", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "definition": "label_values(host)", + "query": "label_values(host)", + "refresh": 1, + "sort": 1, + "includeAll": true, + "multi": true, + "allValue": ".+", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "hide": 0, + "skipUrlSync": false + }, + { + "name": "unit", + "label": "Systemd Unit", + "type": "query", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "definition": "label_values({host=~\"$host\"}, unit)", + "query": "label_values({host=~\"$host\"}, unit)", + "refresh": 1, + "sort": 1, + "includeAll": true, + "multi": true, + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "hide": 0, + "skipUrlSync": false + }, + { + "name": "app", + "label": "App", + "type": "query", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "definition": "label_values({host=~\"$host\", unit=~\"$unit\"}, app)", + "query": "label_values({host=~\"$host\", unit=~\"$unit\"}, app)", + "refresh": 1, + "sort": 1, + "includeAll": true, + "multi": true, + "allValue": ".*", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "hide": 0, + "skipUrlSync": false + }, + { + "name": "level", + "label": "Level", + "type": "query", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "definition": "label_values(level)", + "query": "label_values(level)", + "refresh": 1, + "sort": 1, + "includeAll": true, + "multi": true, + "allValue": ".+", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "hide": 0, + "skipUrlSync": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h" + ] + }, + "timezone": "browser", + "version": 1, + "weekStart": "", + "uid": "az-loki-usage-demo", + "title": "AZ Loki Usage Demo", + "refresh": "30s", + "description": "Provisioned demo dashboard for Loki ingestion health, log volume, noisy units/apps, and recent warnings/errors from Alloy journald streams.", + "panels": [ + { + "id": 1, + "title": "Log ingest rate", + "type": "stat", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "unit": "logs/s", + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 100 + }, + { + "color": "red", + "value": 500 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum(rate({host=~\"$host\", unit=~\"$unit\", app=~\"$app\"}[5m]))", + "legendFormat": "logs/sec", + "queryType": "range", + "refId": "A" + } + ], + "description": "Total Loki log ingestion rate for the selected hosts." + }, + { + "id": 2, + "title": "Warn/Error rate", + "type": "stat", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "unit": "logs/s", + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.1 + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum(rate({host=~\"$host\", unit=~\"$unit\", app=~\"$app\", level=~\"warning|err|error|crit|alert|emerg\"}[5m]))", + "legendFormat": "warn+error/sec", + "queryType": "range", + "refId": "A" + } + ], + "description": "Rate of warning and higher priority logs." + }, + { + "id": 3, + "title": "Logs last hour", + "type": "stat", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "unit": "short", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum(count_over_time({host=~\"$host\", unit=~\"$unit\", app=~\"$app\"}[1h]))", + "legendFormat": "logs", + "queryType": "range", + "refId": "A" + } + ], + "description": "Total log lines ingested in the last hour." + }, + { + "id": 4, + "title": "Noisy units", + "type": "stat", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "unit": "short", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "count(topk(10, sum by (unit) (count_over_time({host=~\"$host\", unit=~\"$unit\", app=~\"$app\"}[1h]))))", + "legendFormat": "units", + "queryType": "range", + "refId": "A" + } + ], + "description": "How many systemd units are present in the top-10 noisy-unit set." + }, + { + "id": 5, + "title": "Log rate by host", + "type": "timeseries", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 4 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "unit": "logs/s", + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (host) (rate({host=~\"$host\", unit=~\"$unit\", app=~\"$app\"}[5m]))", + "legendFormat": "{{host}}", + "queryType": "range", + "refId": "A" + } + ], + "description": "Compares PRM and CLD log volume over time." + }, + { + "id": 6, + "title": "Warnings/errors by host and level", + "type": "timeseries", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 4 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "unit": "logs/s", + "decimals": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (host, level) (rate({host=~\"$host\", unit=~\"$unit\", app=~\"$app\", level=~\"warning|err|error|crit|alert|emerg\"}[5m]))", + "legendFormat": "{{host}} {{level}}", + "queryType": "range", + "refId": "A" + } + ], + "description": "Highlights noisy warning/error streams before they become incidents." + }, + { + "id": 7, + "title": "Top systemd units by log volume (1h)", + "type": "table", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "unit": "short", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "topk(10, sum by (unit) (count_over_time({host=~\"$host\", unit=~\"$unit\", app=~\"$app\"}[1h])))", + "legendFormat": "{{unit}}", + "queryType": "range", + "refId": "A" + } + ], + "description": "Top systemd units by raw log-line count in the last hour." + }, + { + "id": 8, + "title": "Top warning/error units (1h)", + "type": "table", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "unit": "short", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "topk(10, sum by (unit, level) (count_over_time({host=~\"$host\", unit=~\"$unit\", app=~\"$app\", level=~\"warning|err|error|crit|alert|emerg\"}[1h])))", + "legendFormat": "{{unit}} {{level}}", + "queryType": "range", + "refId": "A" + } + ], + "description": "Top warning/error-producing units in the last hour." + }, + { + "id": 9, + "title": "Recent warnings and errors", + "type": "logs", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 20 + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{host=~\"$host\", unit=~\"$unit\", app=~\"$app\", level=~\"warning|err|error|crit|alert|emerg\"}", + "queryType": "range", + "refId": "A", + "maxLines": 500 + } + ], + "description": "Latest warning and higher priority logs. Use host/level variables to narrow the stream." + }, + { + "id": 10, + "title": "Live filtered log stream", + "type": "logs", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 28 + }, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{host=~\"$host\", unit=~\"$unit\", app=~\"$app\", level=~\"$level\"}", + "queryType": "range", + "refId": "A", + "maxLines": 1000 + } + ], + "description": "General Loki stream for selected host and level values." + } + ] +} diff --git a/hosts/AZ-PRM-1/services/monitoring/default.nix b/hosts/AZ-PRM-1/services/monitoring/default.nix index ce6ee4d..69f55b8 100644 --- a/hosts/AZ-PRM-1/services/monitoring/default.nix +++ b/hosts/AZ-PRM-1/services/monitoring/default.nix @@ -4,6 +4,8 @@ ./blackbox.nix ./cld-scrape.nix ./exporters.nix + ./loki.nix + ./alloy.nix ./grafana.nix ./prometheus.nix ]; diff --git a/hosts/AZ-PRM-1/services/monitoring/grafana.nix b/hosts/AZ-PRM-1/services/monitoring/grafana.nix index b86d475..e6467cb 100644 --- a/hosts/AZ-PRM-1/services/monitoring/grafana.nix +++ b/hosts/AZ-PRM-1/services/monitoring/grafana.nix @@ -2,6 +2,7 @@ serviceName = "grafana"; servicePort = config.m3ta.ports.get serviceName; prometheusPort = config.m3ta.ports.get "prometheus"; + lokiPort = config.m3ta.ports.get "loki"; in { services.grafana = { enable = true; @@ -41,7 +42,16 @@ in { access = "proxy"; jsonData.timeInterval = "15s"; } - # Phase 3: add Loki datasource here once services.loki is enabled. + { + name = "Loki"; + uid = "loki"; + type = "loki"; + url = "http://localhost:${toString lokiPort}"; + access = "proxy"; + jsonData = { + maxLines = 1000; + }; + } ]; dashboards.settings.providers = [ { diff --git a/hosts/AZ-PRM-1/services/monitoring/loki.nix b/hosts/AZ-PRM-1/services/monitoring/loki.nix new file mode 100644 index 0000000..de5101c --- /dev/null +++ b/hosts/AZ-PRM-1/services/monitoring/loki.nix @@ -0,0 +1,91 @@ +{config, ...}: let + lokiPort = config.m3ta.ports.get "loki"; + prmNetbirdIP = "100.91.49.26"; + cldNetbirdIP = "100.91.203.184"; +in { + services.loki = { + enable = true; + dataDir = "/var/lib/loki"; + configuration = { + auth_enabled = false; + + server = { + http_listen_address = "127.0.0.1"; + http_listen_port = lokiPort; + }; + + common = { + path_prefix = config.services.loki.dataDir; + replication_factor = 1; + ring = { + instance_addr = "127.0.0.1"; + kvstore.store = "inmemory"; + }; + }; + + schema_config.configs = [ + { + from = "2024-04-01"; + store = "tsdb"; + object_store = "filesystem"; + schema = "v13"; + index = { + prefix = "index_"; + period = "24h"; + }; + } + ]; + + storage_config = { + filesystem.directory = "${config.services.loki.dataDir}/chunks"; + tsdb_shipper = { + active_index_directory = "${config.services.loki.dataDir}/tsdb-index"; + cache_location = "${config.services.loki.dataDir}/tsdb-cache"; + }; + }; + + limits_config = { + retention_period = "336h"; + max_query_lookback = "336h"; + allow_structured_metadata = true; + volume_enabled = true; + reject_old_samples = true; + reject_old_samples_max_age = "168h"; + }; + + compactor = { + working_directory = "${config.services.loki.dataDir}/compactor"; + compaction_interval = "10m"; + retention_enabled = true; + retention_delete_delay = "2h"; + retention_delete_worker_count = 50; + delete_request_store = "filesystem"; + }; + + analytics.reporting_enabled = false; + }; + }; + + services.traefik.staticConfigOptions.entryPoints.loki = { + address = "${prmNetbirdIP}:${toString lokiPort}"; + }; + + services.traefik.dynamicConfigOptions.http = { + middlewares.loki-basic-auth.basicAuth.usersFile = config.age.secrets.monitoring-loki-htpasswd.path; + + services.loki.loadBalancer.servers = [ + {url = "http://127.0.0.1:${toString lokiPort}/";} + ]; + + routers.loki-push = { + rule = "PathPrefix(`/loki/api/v1/push`)"; + entrypoints = ["loki"]; + service = "loki"; + middlewares = ["loki-basic-auth"]; + }; + }; + + networking.firewall.extraCommands = '' + iptables -A INPUT -p tcp -s ${cldNetbirdIP} --dport ${toString lokiPort} -j ACCEPT + ''; +} diff --git a/hosts/AZ-PRM-1/services/n8n.nix b/hosts/AZ-PRM-1/services/n8n.nix index b90d93b..ac1389c 100644 --- a/hosts/AZ-PRM-1/services/n8n.nix +++ b/hosts/AZ-PRM-1/services/n8n.nix @@ -11,6 +11,8 @@ in { N8N_RUNNERS_ENABLED = true; N8N_NATIVE_PYTHON_RUNNER = true; N8N_RUNNERS_AUTH_TOKEN_FILE = config.age.secrets.n8n-runner-auth-token.path; + N8N_LOG_OUTPUT = "json"; + N8N_LOG_LEVEL = "info"; }; taskRunners.enable = true; }; diff --git a/hosts/common/ports.nix b/hosts/common/ports.nix index 372b5da..da8de83 100644 --- a/hosts/common/ports.nix +++ b/hosts/common/ports.nix @@ -47,6 +47,8 @@ # Observability stack (AZ-PRM-1) grafana = 3060; prometheus = 9090; + loki = 3100; + alloy = 12345; node-exporter = 9100; blackbox-exporter = 9115; }; diff --git a/secrets.nix b/secrets.nix index ea2051b..56cdd34 100644 --- a/secrets.nix +++ b/secrets.nix @@ -56,5 +56,7 @@ in { "secrets/grafana-admin-pw.age".publicKeys = systems ++ users; "secrets/grafana-db-password.age".publicKeys = systems ++ users; "secrets/grafana-secret-key.age".publicKeys = systems ++ users; + "secrets/monitoring-loki-htpasswd.age".publicKeys = [AZ-PRM-1] ++ users; + "secrets/monitoring-loki-remote-env.age".publicKeys = [AZ-CLD-1] ++ users; "secrets/ntfy-grafana-webhook.age".publicKeys = systems ++ users; } diff --git a/secrets/monitoring-loki-htpasswd.age b/secrets/monitoring-loki-htpasswd.age new file mode 100644 index 0000000..e54a7fe --- /dev/null +++ b/secrets/monitoring-loki-htpasswd.age @@ -0,0 +1,7 @@ +age-encryption.org/v1 +-> ssh-ed25519 lGqVZg W/Od/uVGTUKo69Vgv5O/PDs0vYBInTaZMoA6iAz3pAU +kML873cHcA+ciN+6TwZG8EBB0OlomKVXVM3lQZ4GZtY +-> ssh-ed25519 CSMyhg pbV76BK+MtO0r5P0KfnJ3/r//UM1FfB721kPUNgJEzs +Sly0hTN9GDPK424HyKLbFxx9h3C9zd4OSr10608SkWc +--- JCsfQn/i+WzVfNsmaBoiyQcrsdEmL+oZ6y4Xrm6LQuI +Jk~62@vY(k&DMǂs% \ No newline at end of file diff --git a/secrets/monitoring-loki-remote-env.age b/secrets/monitoring-loki-remote-env.age new file mode 100644 index 0000000..be9bce1 --- /dev/null +++ b/secrets/monitoring-loki-remote-env.age @@ -0,0 +1,7 @@ +age-encryption.org/v1 +-> ssh-ed25519 ZhVseg lxegZPY5/q3bTKvJNYWHqqIiXu3LG/SQWFLf5E+e5SU +y/36OFpvNqZb1Xr9qUJRp90hQdgsVFI6Oqygk25Ko2k +-> ssh-ed25519 CSMyhg cItqZpxjFQYw5t575rcC5myW+SkElckvZ8LUZXzrn0g +eSWhz8bWG59MNAX5y92FjpilKddR6EmYyldASnKZctM +--- +Q+6qL6eqzVI7IP7Vt45t3x08RdKe/XFqnmrYl8XC5Q +HeăotA&RQS17pS&*(fI0J;ޝγme-luxTOE4