feat: grafana loki

hosts/AZ-CLD-1/secrets.nix

@@ -4,6 +4,12 @@
       traefik-env = {
         file = ../../secrets/traefik-env.age;
       };
+      monitoring-loki-remote-env = {
+        file = ../../secrets/monitoring-loki-remote-env.age;
+        owner = "root";
+        group = "root";
+        mode = "0400";
+      };
       baserow-env = {
         file = ../../secrets/baserow-env.age;
       };

hosts/AZ-CLD-1/services/containers/litellm.nix

@@ -12,6 +12,7 @@ in {
       DO_NOT_TRACK = "True";
       SCARF_NO_ANALYTICS = "True";
       STORE_MODEL_IN_DB = "True";
+      LITELLM_LOG = "json";
     };
     volumes = ["/var/lib/litellm/config.yaml:/app/config.yaml"];
     extraOptions = ["--add-host=postgres:10.89.0.1" "--ip=10.89.0.30" "--network=web"];

hosts/AZ-CLD-1/services/monitoring/alloy.nix

@@ -0,0 +1,100 @@
+{
+  config,
+  lib,
+  ...
+}: let
+  lokiPort = config.m3ta.ports.get "loki";
+  alloyPort = config.m3ta.ports.get "alloy";
+  prmNetbirdIP = "100.91.49.26";
+in {
+  services.alloy = {
+    enable = true;
+    configPath = "/etc/alloy";
+    environmentFile = config.age.secrets.monitoring-loki-remote-env.path;
+    extraFlags = [
+      "--server.http.listen-addr=127.0.0.1:${toString alloyPort}"
+      "--disable-reporting"
+    ];
+  };
+
+  environment.etc."alloy/config.alloy".text = ''
+    loki.relabel "journal" {
+      forward_to = []
+
+      rule {
+        source_labels = ["__journal__systemd_unit"]
+        target_label  = "unit"
+      }
+
+      rule {
+        source_labels = ["__journal_priority_keyword"]
+        target_label  = "level"
+      }
+
+      rule {
+        source_labels = ["__journal_syslog_identifier"]
+        target_label  = "syslog_identifier"
+      }
+    }
+
+    loki.source.journal "system" {
+      forward_to    = [loki.process.journal.receiver]
+      relabel_rules = loki.relabel.journal.rules
+      labels        = {
+        host        = "AZ-CLD-1",
+        environment = "prod",
+      }
+    }
+
+    loki.process "journal" {
+      forward_to = [loki.write.prm.receiver]
+
+      stage.json {
+        expressions = {
+          app        = "app",
+          service    = "service",
+          level      = "level",
+          trace_id   = "trace_id",
+          session_id = "session_id",
+          request_id = "request_id",
+          model      = "model",
+          message    = "message",
+        }
+        drop_malformed = false
+      }
+
+      stage.labels {
+        values = {
+          app     = "",
+          service = "",
+          level   = "",
+        }
+      }
+
+      stage.structured_metadata {
+        values = {
+          trace_id   = "",
+          session_id = "",
+          request_id = "",
+          model      = "",
+        }
+      }
+    }
+
+    loki.write "prm" {
+      endpoint {
+        url = "http://${prmNetbirdIP}:${toString lokiPort}/loki/api/v1/push"
+        basic_auth {
+          username = "alloy"
+          password = sys.env("LOKI_BASIC_AUTH_PASSWORD")
+        }
+      }
+    }
+  '';
+
+  systemd.services.alloy = {
+    wants = ["network-online.target"];
+    after = ["network-online.target"];
+    serviceConfig.SupplementaryGroups = lib.mkAfter ["adm"];
+  };
+}

hosts/AZ-CLD-1/services/monitoring/default.nix

@@ -1,6 +1,7 @@
 # Phase 2 — activate by adding `./monitoring` to hosts/AZ-CLD-1/services/default.nix imports.
 {...}: {
   imports = [
+    ./alloy.nix
     ./node-exporter.nix
   ];
 }

hosts/AZ-PRM-1/secrets.nix

@@ -19,6 +19,11 @@ in {
           file = ../../secrets/grafana-secret-key.age;
           owner = "grafana";
         };
+        monitoring-loki-htpasswd = {
+          file = ../../secrets/monitoring-loki-htpasswd.age;
+          owner = "traefik";
+          mode = "0400";
+        };
         traefik-env = {
           file = ../../secrets/traefik-env.age;
         };

hosts/AZ-PRM-1/services/monitoring/alloy.nix

@@ -0,0 +1,92 @@
+{
+  config,
+  lib,
+  ...
+}: let
+  lokiPort = config.m3ta.ports.get "loki";
+  alloyPort = config.m3ta.ports.get "alloy";
+in {
+  services.alloy = {
+    enable = true;
+    configPath = "/etc/alloy";
+    extraFlags = [
+      "--server.http.listen-addr=127.0.0.1:${toString alloyPort}"
+      "--disable-reporting"
+    ];
+  };
+
+  environment.etc."alloy/config.alloy".text = ''
+    loki.relabel "journal" {
+      forward_to = []
+
+      rule {
+        source_labels = ["__journal__systemd_unit"]
+        target_label  = "unit"
+      }
+
+      rule {
+        source_labels = ["__journal_priority_keyword"]
+        target_label  = "level"
+      }
+
+      rule {
+        source_labels = ["__journal_syslog_identifier"]
+        target_label  = "syslog_identifier"
+      }
+    }
+
+    loki.source.journal "system" {
+      forward_to    = [loki.process.journal.receiver]
+      relabel_rules = loki.relabel.journal.rules
+      labels        = {
+        host        = "AZ-PRM-1",
+        environment = "prod",
+      }
+    }
+
+    loki.process "journal" {
+      forward_to = [loki.write.local.receiver]
+
+      stage.json {
+        expressions = {
+          app              = "app",
+          service          = "service",
+          level            = "level",
+          trace_id         = "trace_id",
+          session_id       = "session_id",
+          n8n_execution_id = "execution_id",
+          workflow_id      = "workflow_id",
+          execution_id     = "execution_id",
+          message          = "message",
+        }
+        drop_malformed = false
+      }
+
+      stage.labels {
+        values = {
+          app     = "",
+          service = "",
+          level   = "",
+        }
+      }
+
+      stage.structured_metadata {
+        values = {
+          trace_id         = "",
+          session_id       = "",
+          n8n_execution_id = "",
+          workflow_id      = "",
+          execution_id     = "",
+        }
+      }
+    }
+
+    loki.write "local" {
+      endpoint {
+        url = "http://127.0.0.1:${toString lokiPort}/loki/api/v1/push"
+      }
+    }
+  '';
+
+  systemd.services.alloy.serviceConfig.SupplementaryGroups = lib.mkAfter ["adm"];
+}

hosts/AZ-PRM-1/services/monitoring/blackbox.nix

@@ -9,7 +9,7 @@ in {
     port = blackboxPort;
     listenAddress = "127.0.0.1";
     openFirewall = false;
-    configFile = (builtins.toFile "blackbox.yml" ''
+    configFile = builtins.toFile "blackbox.yml" ''
       modules:
         http_2xx:
           prober: http
@@ -25,7 +25,7 @@ in {
         tcp_connect:
           prober: tcp
           timeout: 5s
-    '');
+    '';
   };
 
   services.prometheus.scrapeConfigs = [

hosts/AZ-PRM-1/services/monitoring/dashboards/az-loki-usage-demo.json

@@ -0,0 +1,764 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "links": [],
+  "liveNow": false,
+  "schemaVersion": 39,
+  "style": "dark",
+  "tags": [
+    "az",
+    "demo",
+    "loki",
+    "logs",
+    "provisioned"
+  ],
+  "templating": {
+    "list": [
+      {
+        "name": "host",
+        "label": "Host",
+        "type": "query",
+        "datasource": {
+          "type": "loki",
+          "uid": "loki"
+        },
+        "definition": "label_values(host)",
+        "query": "label_values(host)",
+        "refresh": 1,
+        "sort": 1,
+        "includeAll": true,
+        "multi": true,
+        "allValue": ".+",
+        "current": {
+          "selected": true,
+          "text": "All",
+          "value": "$__all"
+        },
+        "hide": 0,
+        "skipUrlSync": false
+      },
+      {
+        "name": "unit",
+        "label": "Systemd Unit",
+        "type": "query",
+        "datasource": {
+          "type": "loki",
+          "uid": "loki"
+        },
+        "definition": "label_values({host=~\"$host\"}, unit)",
+        "query": "label_values({host=~\"$host\"}, unit)",
+        "refresh": 1,
+        "sort": 1,
+        "includeAll": true,
+        "multi": true,
+        "allValue": ".*",
+        "current": {
+          "selected": true,
+          "text": "All",
+          "value": "$__all"
+        },
+        "hide": 0,
+        "skipUrlSync": false
+      },
+      {
+        "name": "app",
+        "label": "App",
+        "type": "query",
+        "datasource": {
+          "type": "loki",
+          "uid": "loki"
+        },
+        "definition": "label_values({host=~\"$host\", unit=~\"$unit\"}, app)",
+        "query": "label_values({host=~\"$host\", unit=~\"$unit\"}, app)",
+        "refresh": 1,
+        "sort": 1,
+        "includeAll": true,
+        "multi": true,
+        "allValue": ".*",
+        "current": {
+          "selected": true,
+          "text": "All",
+          "value": "$__all"
+        },
+        "hide": 0,
+        "skipUrlSync": false
+      },
+      {
+        "name": "level",
+        "label": "Level",
+        "type": "query",
+        "datasource": {
+          "type": "loki",
+          "uid": "loki"
+        },
+        "definition": "label_values(level)",
+        "query": "label_values(level)",
+        "refresh": 1,
+        "sort": 1,
+        "includeAll": true,
+        "multi": true,
+        "allValue": ".+",
+        "current": {
+          "selected": true,
+          "text": "All",
+          "value": "$__all"
+        },
+        "hide": 0,
+        "skipUrlSync": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h"
+    ]
+  },
+  "timezone": "browser",
+  "version": 1,
+  "weekStart": "",
+  "uid": "az-loki-usage-demo",
+  "title": "AZ Loki Usage Demo",
+  "refresh": "30s",
+  "description": "Provisioned demo dashboard for Loki ingestion health, log volume, noisy units/apps, and recent warnings/errors from Alloy journald streams.",
+  "panels": [
+    {
+      "id": 1,
+      "title": "Log ingest rate",
+      "type": "stat",
+      "datasource": {
+        "type": "loki",
+        "uid": "loki"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 0,
+        "y": 0
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {},
+          "mappings": [],
+          "unit": "logs/s",
+          "decimals": 2,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 100
+              },
+              {
+                "color": "red",
+                "value": 500
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "background",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "loki"
+          },
+          "expr": "sum(rate({host=~\"$host\", unit=~\"$unit\", app=~\"$app\"}[5m]))",
+          "legendFormat": "logs/sec",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "description": "Total Loki log ingestion rate for the selected hosts."
+    },
+    {
+      "id": 2,
+      "title": "Warn/Error rate",
+      "type": "stat",
+      "datasource": {
+        "type": "loki",
+        "uid": "loki"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 6,
+        "y": 0
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {},
+          "mappings": [],
+          "unit": "logs/s",
+          "decimals": 2,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "yellow",
+                "value": 0.1
+              },
+              {
+                "color": "red",
+                "value": 1
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "background",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "loki"
+          },
+          "expr": "sum(rate({host=~\"$host\", unit=~\"$unit\", app=~\"$app\", level=~\"warning|err|error|crit|alert|emerg\"}[5m]))",
+          "legendFormat": "warn+error/sec",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "description": "Rate of warning and higher priority logs."
+    },
+    {
+      "id": 3,
+      "title": "Logs last hour",
+      "type": "stat",
+      "datasource": {
+        "type": "loki",
+        "uid": "loki"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 12,
+        "y": 0
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {},
+          "mappings": [],
+          "unit": "short",
+          "decimals": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "background",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "loki"
+          },
+          "expr": "sum(count_over_time({host=~\"$host\", unit=~\"$unit\", app=~\"$app\"}[1h]))",
+          "legendFormat": "logs",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "description": "Total log lines ingested in the last hour."
+    },
+    {
+      "id": 4,
+      "title": "Noisy units",
+      "type": "stat",
+      "datasource": {
+        "type": "loki",
+        "uid": "loki"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 18,
+        "y": 0
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {},
+          "mappings": [],
+          "unit": "short",
+          "decimals": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "background",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "loki"
+          },
+          "expr": "count(topk(10, sum by (unit) (count_over_time({host=~\"$host\", unit=~\"$unit\", app=~\"$app\"}[1h]))))",
+          "legendFormat": "units",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "description": "How many systemd units are present in the top-10 noisy-unit set."
+    },
+    {
+      "id": 5,
+      "title": "Log rate by host",
+      "type": "timeseries",
+      "datasource": {
+        "type": "loki",
+        "uid": "loki"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 4
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {},
+          "mappings": [],
+          "unit": "logs/s",
+          "decimals": 2,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "loki"
+          },
+          "expr": "sum by (host) (rate({host=~\"$host\", unit=~\"$unit\", app=~\"$app\"}[5m]))",
+          "legendFormat": "{{host}}",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "description": "Compares PRM and CLD log volume over time."
+    },
+    {
+      "id": 6,
+      "title": "Warnings/errors by host and level",
+      "type": "timeseries",
+      "datasource": {
+        "type": "loki",
+        "uid": "loki"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 4
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {},
+          "mappings": [],
+          "unit": "logs/s",
+          "decimals": 2,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "loki"
+          },
+          "expr": "sum by (host, level) (rate({host=~\"$host\", unit=~\"$unit\", app=~\"$app\", level=~\"warning|err|error|crit|alert|emerg\"}[5m]))",
+          "legendFormat": "{{host}} {{level}}",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "description": "Highlights noisy warning/error streams before they become incidents."
+    },
+    {
+      "id": 7,
+      "title": "Top systemd units by log volume (1h)",
+      "type": "table",
+      "datasource": {
+        "type": "loki",
+        "uid": "loki"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 12
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {},
+          "mappings": [],
+          "unit": "short",
+          "decimals": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "cellHeight": "sm",
+        "footer": {
+          "countRows": false,
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
+        "showHeader": true
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "loki"
+          },
+          "expr": "topk(10, sum by (unit) (count_over_time({host=~\"$host\", unit=~\"$unit\", app=~\"$app\"}[1h])))",
+          "legendFormat": "{{unit}}",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "description": "Top systemd units by raw log-line count in the last hour."
+    },
+    {
+      "id": 8,
+      "title": "Top warning/error units (1h)",
+      "type": "table",
+      "datasource": {
+        "type": "loki",
+        "uid": "loki"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 12
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {},
+          "mappings": [],
+          "unit": "short",
+          "decimals": 0,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "options": {
+        "cellHeight": "sm",
+        "footer": {
+          "countRows": false,
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
+        "showHeader": true
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "loki"
+          },
+          "expr": "topk(10, sum by (unit, level) (count_over_time({host=~\"$host\", unit=~\"$unit\", app=~\"$app\", level=~\"warning|err|error|crit|alert|emerg\"}[1h])))",
+          "legendFormat": "{{unit}} {{level}}",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "description": "Top warning/error-producing units in the last hour."
+    },
+    {
+      "id": 9,
+      "title": "Recent warnings and errors",
+      "type": "logs",
+      "datasource": {
+        "type": "loki",
+        "uid": "loki"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 20
+      },
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "options": {
+        "dedupStrategy": "none",
+        "enableLogDetails": true,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showLabels": false,
+        "showTime": true,
+        "sortOrder": "Descending",
+        "wrapLogMessage": true
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "loki"
+          },
+          "expr": "{host=~\"$host\", unit=~\"$unit\", app=~\"$app\", level=~\"warning|err|error|crit|alert|emerg\"}",
+          "queryType": "range",
+          "refId": "A",
+          "maxLines": 500
+        }
+      ],
+      "description": "Latest warning and higher priority logs. Use host/level variables to narrow the stream."
+    },
+    {
+      "id": 10,
+      "title": "Live filtered log stream",
+      "type": "logs",
+      "datasource": {
+        "type": "loki",
+        "uid": "loki"
+      },
+      "gridPos": {
+        "h": 10,
+        "w": 24,
+        "x": 0,
+        "y": 28
+      },
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "options": {
+        "dedupStrategy": "none",
+        "enableLogDetails": true,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showLabels": false,
+        "showTime": true,
+        "sortOrder": "Descending",
+        "wrapLogMessage": true
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "loki"
+          },
+          "expr": "{host=~\"$host\", unit=~\"$unit\", app=~\"$app\", level=~\"$level\"}",
+          "queryType": "range",
+          "refId": "A",
+          "maxLines": 1000
+        }
+      ],
+      "description": "General Loki stream for selected host and level values."
+    }
+  ]
+}

hosts/AZ-PRM-1/services/monitoring/default.nix

@@ -4,6 +4,8 @@
     ./blackbox.nix
     ./cld-scrape.nix
     ./exporters.nix
+    ./loki.nix
+    ./alloy.nix
     ./grafana.nix
     ./prometheus.nix
   ];

hosts/AZ-PRM-1/services/monitoring/grafana.nix

@@ -2,6 +2,7 @@
   serviceName = "grafana";
   servicePort = config.m3ta.ports.get serviceName;
   prometheusPort = config.m3ta.ports.get "prometheus";
+  lokiPort = config.m3ta.ports.get "loki";
 in {
   services.grafana = {
     enable = true;
@@ -41,7 +42,16 @@ in {
           access = "proxy";
           jsonData.timeInterval = "15s";
         }
-        # Phase 3: add Loki datasource here once services.loki is enabled.
+        {
+          name = "Loki";
+          uid = "loki";
+          type = "loki";
+          url = "http://localhost:${toString lokiPort}";
+          access = "proxy";
+          jsonData = {
+            maxLines = 1000;
+          };
+        }
       ];
       dashboards.settings.providers = [
         {

hosts/AZ-PRM-1/services/monitoring/loki.nix

@@ -0,0 +1,91 @@
+{config, ...}: let
+  lokiPort = config.m3ta.ports.get "loki";
+  prmNetbirdIP = "100.91.49.26";
+  cldNetbirdIP = "100.91.203.184";
+in {
+  services.loki = {
+    enable = true;
+    dataDir = "/var/lib/loki";
+    configuration = {
+      auth_enabled = false;
+
+      server = {
+        http_listen_address = "127.0.0.1";
+        http_listen_port = lokiPort;
+      };
+
+      common = {
+        path_prefix = config.services.loki.dataDir;
+        replication_factor = 1;
+        ring = {
+          instance_addr = "127.0.0.1";
+          kvstore.store = "inmemory";
+        };
+      };
+
+      schema_config.configs = [
+        {
+          from = "2024-04-01";
+          store = "tsdb";
+          object_store = "filesystem";
+          schema = "v13";
+          index = {
+            prefix = "index_";
+            period = "24h";
+          };
+        }
+      ];
+
+      storage_config = {
+        filesystem.directory = "${config.services.loki.dataDir}/chunks";
+        tsdb_shipper = {
+          active_index_directory = "${config.services.loki.dataDir}/tsdb-index";
+          cache_location = "${config.services.loki.dataDir}/tsdb-cache";
+        };
+      };
+
+      limits_config = {
+        retention_period = "336h";
+        max_query_lookback = "336h";
+        allow_structured_metadata = true;
+        volume_enabled = true;
+        reject_old_samples = true;
+        reject_old_samples_max_age = "168h";
+      };
+
+      compactor = {
+        working_directory = "${config.services.loki.dataDir}/compactor";
+        compaction_interval = "10m";
+        retention_enabled = true;
+        retention_delete_delay = "2h";
+        retention_delete_worker_count = 50;
+        delete_request_store = "filesystem";
+      };
+
+      analytics.reporting_enabled = false;
+    };
+  };
+
+  services.traefik.staticConfigOptions.entryPoints.loki = {
+    address = "${prmNetbirdIP}:${toString lokiPort}";
+  };
+
+  services.traefik.dynamicConfigOptions.http = {
+    middlewares.loki-basic-auth.basicAuth.usersFile = config.age.secrets.monitoring-loki-htpasswd.path;
+
+    services.loki.loadBalancer.servers = [
+      {url = "http://127.0.0.1:${toString lokiPort}/";}
+    ];
+
+    routers.loki-push = {
+      rule = "PathPrefix(`/loki/api/v1/push`)";
+      entrypoints = ["loki"];
+      service = "loki";
+      middlewares = ["loki-basic-auth"];
+    };
+  };
+
+  networking.firewall.extraCommands = ''
+    iptables -A INPUT -p tcp -s ${cldNetbirdIP} --dport ${toString lokiPort} -j ACCEPT
+  '';
+}

hosts/AZ-PRM-1/services/n8n.nix

@@ -11,6 +11,8 @@ in {
       N8N_RUNNERS_ENABLED = true;
       N8N_NATIVE_PYTHON_RUNNER = true;
       N8N_RUNNERS_AUTH_TOKEN_FILE = config.age.secrets.n8n-runner-auth-token.path;
+      N8N_LOG_OUTPUT = "json";
+      N8N_LOG_LEVEL = "info";
     };
     taskRunners.enable = true;
   };

hosts/common/ports.nix

@@ -47,6 +47,8 @@
       # Observability stack (AZ-PRM-1)
       grafana = 3060;
       prometheus = 9090;
+      loki = 3100;
+      alloy = 12345;
       node-exporter = 9100;
       blackbox-exporter = 9115;
     };

secrets.nix

@@ -56,5 +56,7 @@ in {
   "secrets/grafana-admin-pw.age".publicKeys = systems ++ users;
   "secrets/grafana-db-password.age".publicKeys = systems ++ users;
   "secrets/grafana-secret-key.age".publicKeys = systems ++ users;
+  "secrets/monitoring-loki-htpasswd.age".publicKeys = [AZ-PRM-1] ++ users;
+  "secrets/monitoring-loki-remote-env.age".publicKeys = [AZ-CLD-1] ++ users;
   "secrets/ntfy-grafana-webhook.age".publicKeys = systems ++ users;
 }

secrets/monitoring-loki-htpasswd.age

@@ -0,0 +1,7 @@
+age-encryption.org/v1
+-> ssh-ed25519 lGqVZg W/Od/uVGTUKo69Vgv5O/PDs0vYBInTaZMoA6iAz3pAU
+kML873cHcA+ciN+6TwZG8EBB0OlomKVXVM3lQZ4GZtY
+-> ssh-ed25519 CSMyhg pbV76BK+MtO0r5P0KfnJ3/r//UM1FfB721kPUNgJEzs
+Sly0hTN9GDPK424HyKLbFxx9h3C9zd4OSr10608SkWc
+--- JCsfQn/i+WzVfNsmaBoiyQcrsdEmL+oZ6y4Xrm6LQuI
+´ÄJ¿žkÈå~6¦ÉÄí€2ï@�vÜ
Y©¥(ká&ˆD·MÄ<Gò[Zm4ª%‘ͺ*Y¿ÃåËy ´ÎåË‹�`¦W�Ôúæ9ÓÃP!ÇßÀÞÒ)¤.�·*>ŠãÇ‚¡­sæ%

secrets/monitoring-loki-remote-env.age

@@ -0,0 +1,7 @@
+age-encryption.org/v1
+-> ssh-ed25519 ZhVseg lxegZPY5/q3bTKvJNYWHqqIiXu3LG/SQWFLf5E+e5SU
+y/36OFpvNqZb1Xr9qUJRp90hQdgsVFI6Oqygk25Ko2k
+-> ssh-ed25519 CSMyhg cItqZpxjFQYw5t575rcC5myW+SkElckvZ8LUZXzrn0g
+eSWhz8bWG59MNAX5y92FjpilKddR6EmYyldASnKZctM
+--- +Q+6qL6eqzVI7IP7Vt45t3x08RdKe/XFqnmrYl8XC5Q
+HÂÊe’ăoÀtAä&«RQÅS17ÄÆpS³Ò&*(fIþ0þJˆº;Þ�Ïγmëe-lÔuxTƒOE¾ÿÄ4çº<h¯‡€|¶‡ãlö˜ˆ2ãßÜ£4`�øçåuhû2pöH