AZ-NIX-CLIENTS/roles/thin-client/monitoring/default.nix

# roles/thin-client/monitoring/default.nix
#
# Q19 decisions: Prometheus node_exporter + Loki promtail (now: Alloy) +
# NetBird Inventory + Asset-Management-Tool (default: Snipe-IT).
{config, lib, pkgs, ...}: let
  inherit (lib) mkIf mkOption types;
  cfg = config.az.tc;
in {
  options.az.tc.monitoring = {
    prometheusPushGateway = mkOption {
      type = types.str;
      default = "pushgateway.az-group.local:9091"; # TODO: real
      description = ''
        Prometheus Pushgateway URL. node_exporter runs locally and pushes
        metrics here (thin clients are usually behind NetBird NAT, so
        pull-scraping isn't reliable).
      '';
    };

    lokiUrl = mkOption {
      type = types.str;
      default = "http://loki.az-group.local:3100"; # TODO: real
      description = "Loki URL for Alloy log shipping.";
    };

    assetTool = mkOption {
      type = types.enum ["snipe-it" "it-glue" "glpi" "none"];
      default = "snipe-it";
      description = ''
        Asset management tool to integrate with. snipe-it is the
        open-source default. Override if you use a different one.
      '';
    };

    snipeItUrl = mkOption {
      type = types.str;
      default = "https://snipeit.az-group.local"; # TODO: real
      description = "Snipe-IT base URL (no trailing slash).";
    };

    # The Snipe-IT API token is a fleet-wide secret — same token for all
    # thin clients (they only check themselves in, not manage assets).
    # Stored in agenix under secrets/snipeit-api-token.age.
  };

  config = mkIf cfg.enable {
    services.prometheus.exporters.node = {
      enable = true;
      enabledCollectors = ["systemd" "processes" "tcpstat"];
      listenAddress = "127.0.0.1";
      port = 9100;
    };

    # Push metrics to the central Pushgateway every 60s.
    # Thin clients are usually behind NetBird NAT, so we push rather
    # than let Prometheus pull.
    systemd.services."push-node-metrics" = {
      description = "Push node_exporter metrics to Pushgateway";
      wantedBy = ["multi-user.target"];
      after = ["network-online.target" "prometheus-node-exporter.service"];
      wants = ["network-online.target"];
      serviceConfig = {
        Type = "simple";
        ExecStart = pkgs.writeShellScript "push-node-metrics" ''
          set -euo pipefail
          while true; do
            # Scrape local node_exporter and forward to Pushgateway.
            # Use --data-binary to preserve Prometheus exposition format.
            ${pkgs.curl}/bin/curl -s --fail \
              --connect-timeout 5 \
              "http://127.0.0.1:9100/metrics" \
              | ${pkgs.curl}/bin/curl -s --fail \
                --connect-timeout 5 \
                -X POST \
                --data-binary @- \
                "http://${config.az.tc.monitoring.prometheusPushGateway}/metrics/job/${config.networking.hostName}/instance/${config.networking.hostName}" \
              || echo "push-node-metrics: push failed, will retry in 60s" >&2
            sleep 60
          done
        '';
        Restart = "always";
        RestartSec = "10s";
        User = "root";
      };
    };

    # Log shipping via Grafana Alloy (promtail is deprecated/EOL since
    # 2025). Alloy scrapes the systemd journal and ships to Loki.
    # Config is a real River config that ships all journal logs with
    # host + unit labels.
    services.alloy = {
      enable = true;
      extraFlags = [
        "--server.http.listen.address=127.0.0.1"
        "--server.http.listen.port=12345"
      ];
    };

    environment.etc."alloy/config.alloy".text = ''
      // Auto-generated by roles/thin-client/monitoring/default.nix
      // Ships all systemd journal entries to the central Loki.

      logging {
        level = "info"
        format = "logfmt"
      }

      loki.write "default" {
        endpoint {
          url = "${config.az.tc.monitoring.lokiUrl}/loki/api/v1/push"
        }
      }

      loki.source.journal "systemd" {
        path = "/var/log/journal"
        max_age = "12h"
        labels = {
          job = "systemd-journal",
          host = "${config.networking.hostName}",
        }
        forward_to = [loki.write.default.receiver]
        relabel_rules = {
          rule {
            source_labels = ["__journal__systemd_unit"]
            target_label  = "unit"
          }
          rule {
            source_labels = ["__journal__priority"]
            target_label  = "severity"
          }
          rule {
            source_labels = ["__journal__transport"]
            target_label  = "transport"
          }
        }
      }
    '';

    # Snipe-IT asset check-in — reports host presence + serial + asset tag
    # to the central asset management. Runs once at boot and daily after.
    # Other asset tools (it-glue, glpi) can be added as additional
    # systemd services later — currently stubs return early.
    systemd.services.snipe-it-checkin = mkIf (config.az.tc.monitoring.assetTool == "snipe-it") {
      description = "Report host presence to Snipe-IT asset management";
      wantedBy = ["multi-user.target"];
      after = ["network-online.target"];
      wants = ["network-online.target"];
      startAt = "*-*-* 03:30:00"; # daily at 03:30 (after auto-upgrade window)
      serviceConfig = {
        Type = "oneshot";
        ExecStart = pkgs.writeShellScript "snipe-it-checkin" ''
          set -euo pipefail

          API_URL="${config.az.tc.monitoring.snipeItUrl}/api/v1"
          TOKEN_FILE="/run/agenix/snipeit-api-token"

          if [ ! -s "$TOKEN_FILE" ]; then
            echo "snipe-it-checkin: $TOKEN_FILE missing or empty — skipping" >&2
            exit 0
          fi

          TOKEN="$(head -1 "$TOKEN_FILE")"
          HOSTNAME="${config.networking.hostName}"
          SERIAL="$(cat /sys/class/dmi/id/product_serial 2>/dev/null || echo unknown)"
          ASSET_TAG="$HOSTNAME"

          # Lookup by asset_tag — if it exists, PATCH; if not, POST.
          EXISTING="$(${pkgs.curl}/bin/curl -s --fail \
            --connect-timeout 5 \
            -H "Authorization: Bearer $TOKEN" \
            -H "Accept: application/json" \
            "$API_URL/hardware/byasset/$ASSET_TAG" || echo '{}')"

          PAYLOAD="$(cat <<JSON
          {
            "asset_tag": "$ASSET_TAG",
            "name": "$HOSTNAME",
            "serial": "$SERIAL",
            "model_id": 1,
            "status_id": 2,
            "notes": "AZ-NIX-CLIENTS thin client (auto-checked-in)"
          }
          JSON
          )"

          if echo "$EXISTING" | ${pkgs.jq}/bin/jq -e '.id' >/dev/null 2>&1; then
            ID="$(echo "$EXISTING" | ${pkgs.jq}/bin/jq -r '.id')"
            ${pkgs.curl}/bin/curl -s --fail \
              --connect-timeout 5 \
              -X PATCH \
              -H "Authorization: Bearer $TOKEN" \
              -H "Content-Type: application/json" \
              -H "Accept: application/json" \
              -d "$PAYLOAD" \
              "$API_URL/hardware/$ID" >/dev/null
            echo "snipe-it-checkin: PATCHed asset $ID ($ASSET_TAG)"
          else
            ${pkgs.curl}/bin/curl -s --fail \
              --connect-timeout 5 \
              -X POST \
              -H "Authorization: Bearer $TOKEN" \
              -H "Content-Type: application/json" \
              -H "Accept: application/json" \
              -d "$PAYLOAD" \
              "$API_URL/hardware" >/dev/null
            echo "snipe-it-checkin: POSTed new asset $ASSET_TAG"
          fi
        '';
        User = "root";
      };
    };

    # Snipe-IT API token (fleet-wide shared secret)
    age.secrets."snipeit-api-token" = mkIf (config.az.tc.monitoring.assetTool == "snipe-it") {
      file = ../../../secrets/snipeit-api-token.age;
      mode = "0400";
      owner = "root";
    };
  };
}