feat: grafana loki

hosts/AZ-CLD-1/services/containers/litellm.nix

@@ -12,6 +12,7 @@ in {
       DO_NOT_TRACK = "True";
       SCARF_NO_ANALYTICS = "True";
       STORE_MODEL_IN_DB = "True";
+      LITELLM_LOG = "json";
     };
     volumes = ["/var/lib/litellm/config.yaml:/app/config.yaml"];
     extraOptions = ["--add-host=postgres:10.89.0.1" "--ip=10.89.0.30" "--network=web"];

hosts/AZ-CLD-1/services/monitoring/alloy.nix

@@ -0,0 +1,100 @@
+{
+  config,
+  lib,
+  ...
+}: let
+  lokiPort = config.m3ta.ports.get "loki";
+  alloyPort = config.m3ta.ports.get "alloy";
+  prmNetbirdIP = "100.91.49.26";
+in {
+  services.alloy = {
+    enable = true;
+    configPath = "/etc/alloy";
+    environmentFile = config.age.secrets.monitoring-loki-remote-env.path;
+    extraFlags = [
+      "--server.http.listen-addr=127.0.0.1:${toString alloyPort}"
+      "--disable-reporting"
+    ];
+  };
+
+  environment.etc."alloy/config.alloy".text = ''
+    loki.relabel "journal" {
+      forward_to = []
+
+      rule {
+        source_labels = ["__journal__systemd_unit"]
+        target_label  = "unit"
+      }
+
+      rule {
+        source_labels = ["__journal_priority_keyword"]
+        target_label  = "level"
+      }
+
+      rule {
+        source_labels = ["__journal_syslog_identifier"]
+        target_label  = "syslog_identifier"
+      }
+    }
+
+    loki.source.journal "system" {
+      forward_to    = [loki.process.journal.receiver]
+      relabel_rules = loki.relabel.journal.rules
+      labels        = {
+        host        = "AZ-CLD-1",
+        environment = "prod",
+      }
+    }
+
+    loki.process "journal" {
+      forward_to = [loki.write.prm.receiver]
+
+      stage.json {
+        expressions = {
+          app        = "app",
+          service    = "service",
+          level      = "level",
+          trace_id   = "trace_id",
+          session_id = "session_id",
+          request_id = "request_id",
+          model      = "model",
+          message    = "message",
+        }
+        drop_malformed = false
+      }
+
+      stage.labels {
+        values = {
+          app     = "",
+          service = "",
+          level   = "",
+        }
+      }
+
+      stage.structured_metadata {
+        values = {
+          trace_id   = "",
+          session_id = "",
+          request_id = "",
+          model      = "",
+        }
+      }
+    }
+
+    loki.write "prm" {
+      endpoint {
+        url = "http://${prmNetbirdIP}:${toString lokiPort}/loki/api/v1/push"
+        basic_auth {
+          username = "alloy"
+          password = sys.env("LOKI_BASIC_AUTH_PASSWORD")
+        }
+      }
+    }
+  '';
+
+  systemd.services.alloy = {
+    wants = ["network-online.target"];
+    after = ["network-online.target"];
+    serviceConfig.SupplementaryGroups = lib.mkAfter ["adm"];
+  };
+}

hosts/AZ-CLD-1/services/monitoring/default.nix

@@ -1,6 +1,7 @@
 # Phase 2 — activate by adding `./monitoring` to hosts/AZ-CLD-1/services/default.nix imports.
 {...}: {
   imports = [
+    ./alloy.nix
     ./node-exporter.nix
   ];
 }