feat: mirror cld var backups to prm

hosts/AZ-CLD-1/secrets.nix

@@ -31,6 +31,12 @@
       n8n-runner-auth-token = {
         file = ../../secrets/n8n-runner-auth-token-cld.age;
       };
+      cld-var-backup-sync-ssh-key = {
+        file = ../../secrets/cld-var-backup-sync-ssh-key.age;
+        owner = "root";
+        group = "root";
+        mode = "0400";
+      };
       netbird-auth-secret = {
         file = ../../secrets/netbird-auth-secret.age;
       };

hosts/AZ-CLD-1/services/default.nix

@@ -8,6 +8,7 @@
     ./mysql.nix
     ./n8n.nix
     ./netbird.nix
+    ./var-backup-sync.nix
     ./ntfy.nix
     ./outline.nix
     ./postgres.nix

hosts/AZ-CLD-1/services/var-backup-sync.nix

@@ -0,0 +1,79 @@
+{
+  config,
+  pkgs,
+  ...
+}: let
+  serviceName = "cld-var-backup-sync";
+  sourceDir = "/var/backup/";
+  targetHost = "100.91.49.26";
+  targetPort = "2022";
+  targetUser = "backup-ingest";
+  targetDir = "/srv/veeam-ingest/AZ-CLD-1/var-backup/";
+  sshKey = config.age.secrets.cld-var-backup-sync-ssh-key.path;
+in {
+  environment.systemPackages = with pkgs; [
+    rsync
+    openssh
+  ];
+
+  systemd.tmpfiles.rules = [
+    "d /var/lib/${serviceName} 0700 root root - -"
+  ];
+
+  systemd.services.${serviceName} = {
+    description = "Mirror /var/backup from AZ-CLD-1 to AZ-PRM-1 for VEEAM";
+    after = ["network-online.target" "netbird.service"];
+    wants = ["network-online.target" "netbird.service"];
+
+    path = with pkgs; [
+      coreutils
+      openssh
+      rsync
+      util-linux
+    ];
+
+    serviceConfig = {
+      Type = "oneshot";
+      User = "root";
+      Group = "root";
+      RuntimeDirectory = serviceName;
+      RuntimeDirectoryMode = "0700";
+      LockPersonality = true;
+      NoNewPrivileges = true;
+      PrivateTmp = true;
+    };
+
+    script = ''
+      set -euo pipefail
+
+      lock_file="/run/${serviceName}/${serviceName}.lock"
+
+      flock -n "$lock_file" \
+        rsync \
+          -aH \
+          --no-owner \
+          --no-group \
+          --no-devices \
+          --no-specials \
+          --numeric-ids \
+          --delete \
+          --partial \
+          --delay-updates \
+          --human-readable \
+          --stats \
+          -e "ssh -i ${sshKey} -p ${targetPort} -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile=/var/lib/${serviceName}/known_hosts" \
+          ${sourceDir} \
+          ${targetUser}@${targetHost}:${targetDir}
+    '';
+  };
+
+  systemd.timers.${serviceName} = {
+    wantedBy = ["timers.target"];
+    timerConfig = {
+      OnCalendar = "*-*-* 05:00:00";
+      Persistent = true;
+      RandomizedDelaySec = "10m";
+      Unit = "${serviceName}.service";
+    };
+  };
+}

hosts/AZ-PRM-1/services/backup-ingest.nix

@@ -0,0 +1,25 @@
+{pkgs, ...}: let
+  backupUser = "backup-ingest";
+  backupGroup = "backup-ingest";
+  backupRoot = "/srv/veeam-ingest";
+  cldBackupTarget = "${backupRoot}/AZ-CLD-1/var-backup";
+in {
+  users.groups.${backupGroup} = {};
+
+  users.users.${backupUser} = {
+    isSystemUser = true;
+    group = backupGroup;
+    home = backupRoot;
+    createHome = false;
+    shell = pkgs.bashInteractive;
+    openssh.authorizedKeys.keys = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL6FhMvJqh7m+AfC5yeHUIxhEvalSoA5OzqtDVstuAX1 cld-var-backup-sync@AZ-CLD-1-to-AZ-PRM-1"
+    ];
+  };
+
+  systemd.tmpfiles.rules = [
+    "d ${backupRoot} 0750 ${backupUser} ${backupGroup} - -"
+    "d ${backupRoot}/AZ-CLD-1 0750 ${backupUser} ${backupGroup} - -"
+    "d ${cldBackupTarget} 0750 ${backupUser} ${backupGroup} - -"
+  ];
+}

hosts/AZ-PRM-1/services/default.nix

@@ -1,6 +1,7 @@
 {
   imports = [
     ./containers
+    ./backup-ingest.nix
     ./azion-scheduler.nix
     ./bpi.nix
     ./n8n.nix

secrets.nix

@@ -32,6 +32,7 @@ in {
   "secrets/n8n-db.age".publicKeys = systems ++ users;
   "secrets/n8n-runner-auth-token-cld.age".publicKeys = systems ++ users;
   "secrets/n8n-runner-auth-token-prm.age".publicKeys = systems ++ users;
+  "secrets/cld-var-backup-sync-ssh-key.age".publicKeys = [AZ-CLD-1] ++ users;
   "secrets/netbird-auth-secret.age".publicKeys = systems ++ users;
   "secrets/netbird-db-password.age".publicKeys = systems ++ users;
   "secrets/netbird-encryption-key.age".publicKeys = systems ++ users;

secrets/cld-var-backup-sync-ssh-key.age

@@ -0,0 +1,7 @@
+age-encryption.org/v1
+-> ssh-ed25519 ZhVseg FV2VZoyYS+MUzuliHGxvKwTiJXSrdhebOT1RazLS/AQ
+eM+0VpdFXVgIT4wgdG2XrqAs5RcVSyG6Njffcn7hOtM
+-> ssh-ed25519 CSMyhg wqLsziUzM1jSezZJz6FH3HrOC0KWCaW+Y9AhwckQ9BE
+lDPBOSS3LucDbMqGh6vqLIvq6mefMdJMOfZg773ds2g
+--- czPBWCwKI9y/1v1I/M5RASNYNqY3PKtdexdiTPR4csk
+¾b°ðyd	K€Hä<zØÄÄ"n°®@̺„b