feat: pgbounce, librechat upgrade, homarr

hosts/AZ-CLD-1/secrets.nix

@@ -13,6 +13,9 @@
       baserow-env = {
         file = ../../secrets/baserow-env.age;
       };
+      homarr-env = {
+        file = ../../secrets/homarr-env.age;
+      };
       librechat = {
         file = ../../secrets/librechat.age;
       };
@@ -81,6 +84,10 @@
         file = ../../secrets/pgadmin-pw.age;
         owner = "pgadmin";
       };
+      pgbouncer-userlist = {
+        file = ../../secrets/pgbouncer-userlist.age;
+        owner = "pgbouncer";
+      };
       vaultwarden-env = {
         file = ../../secrets/vaultwarden-env.age;
       };

hosts/AZ-CLD-1/services/containers/baserow.nix

@@ -7,7 +7,7 @@ in {
     environment = {
       # BASEROW_AMOUNT_OF_GUNICORN_WORKERS = "4";
       # BASEROW_AMOUNT_OF_WORKERS = "2";
-      DATABASE_CONN_MAX_AGE = "60";
+      DATABASE_CONN_MAX_AGE = "0";
       # Proxy: tell Django the connection is HTTPS so cookies get Secure flag
       BASEROW_ENABLE_SECURE_PROXY_SSL_HEADER = "yes";
       # Published apps run on different origins — allow cross-origin cookie delivery

hosts/AZ-CLD-1/services/containers/default.nix

@@ -1,6 +1,7 @@
 {lib, ...}: {
   imports = [
     ./baserow.nix
+    ./homarr.nix
     ./it-tools.nix
     ./librechat.nix
     ./litellm.nix

hosts/AZ-CLD-1/services/containers/homarr.nix

@@ -0,0 +1,48 @@
+{config, ...}: let
+  serviceName = "homarr";
+  servicePort = config.m3ta.ports.get serviceName;
+in {
+  virtualisation.oci-containers.containers.${serviceName} = {
+    image = "ghcr.io/homarr-labs/homarr:latest";
+    environment = {
+      TZ = "Europe/Berlin";
+
+      BASE_URL = "https://dash.az-gruppe.com";
+      NEXTAUTH_URL = "https://dash.az-gruppe.com";
+      AUTH_PROVIDERS = "oidc";
+      AUTH_OIDC_CLIENT_NAME = "Azure";
+      AUTH_OIDC_SCOPE_OVERWRITE = "openid email profile";
+      AUTH_OIDC_GROUPS_ATTRIBUTE = "roles";
+    };
+    environmentFiles = [config.age.secrets.homarr-env.path];
+    ports = ["127.0.0.1:${toString servicePort}:7575"];
+    volumes = ["homarr_data:/appdata"];
+    extraOptions = ["--ip=10.89.0.13" "--network=web" "--dns=8.8.8.8" "--dns=8.8.4.4"];
+  };
+
+  # Traefik configuration
+  services.traefik.dynamicConfigOptions.http = {
+    services.${serviceName}.loadBalancer.servers = [
+      {
+        url = "http://localhost:${toString servicePort}/";
+      }
+    ];
+
+    middlewares."${serviceName}-headers".headers = {
+      customRequestHeaders = {
+        X-Forwarded-Proto = "https";
+        X-Forwarded-Port = "443";
+      };
+    };
+
+    routers.${serviceName} = {
+      rule = "Host(`dash.az-gruppe.com`)";
+      tls = {
+        certResolver = "ionos";
+      };
+      service = serviceName;
+      entrypoints = "websecure";
+      middlewares = ["${serviceName}-headers"];
+    };
+  };
+}

hosts/AZ-CLD-1/services/containers/librechat-dev.nix

@@ -29,7 +29,7 @@ in {
       environment = {
         RAG_PORT = "8000";
         DB_HOST = "10.89.1.1";
-        DB_PORT = "5432";
+        DB_PORT = "6432";
       };
       environmentFiles = [envFileDev envFileCommon];
       dependsOn = ["meilisearch-dev"];

hosts/AZ-CLD-1/services/containers/librechat.nix

@@ -29,7 +29,7 @@ in {
       environment = {
         RAG_PORT = "8000";
         DB_HOST = "10.89.0.1";
-        DB_PORT = "5432";
+        DB_PORT = "6432";
       };
       environmentFiles = [envFileCommon envFileProd];
       dependsOn = ["meilisearch"];

hosts/AZ-CLD-1/services/containers/zammad-hr.nix

@@ -28,8 +28,8 @@
     POSTGRESQL_DB = "zammad_${instanceName}";
     POSTGRESQL_HOST = "10.89.0.1";
     POSTGRESQL_USER = "zammad_${instanceName}";
-    POSTGRESQL_PORT = "5432";
+    POSTGRESQL_PORT = "6433";
-    POSTGRESQL_OPTIONS = "?pool=50";
+    POSTGRESQL_OPTIONS = "?pool=5";
     REDIS_URL = "redis://zammad-redis:6379";
     TZ = "Europe/Berlin";
     BACKUP_DIR = "/var/tmp/zammad";
@@ -231,8 +231,8 @@ in {
         --env POSTGRESQL_DB=zammad_${instanceName} \
         --env POSTGRESQL_HOST=10.89.0.1 \
         --env POSTGRESQL_USER=zammad_${instanceName} \
-        --env POSTGRESQL_PORT=5432 \
+        --env POSTGRESQL_PORT=6433 \
-        --env POSTGRESQL_OPTIONS='?pool=50' \
+        --env POSTGRESQL_OPTIONS='?pool=5' \
         --env REDIS_URL=redis://zammad-redis:6379 \
         --env TZ=Europe/Berlin \
         --env ELASTICSEARCH_ENABLED=true \

hosts/AZ-CLD-1/services/default.nix

@@ -12,6 +12,7 @@
     ./var-backup-sync.nix
     ./ntfy.nix
     ./outline.nix
+    ./pgbouncer.nix
     ./postgres.nix
     ./snipe-it.nix
     ./traefik.nix

hosts/AZ-CLD-1/services/pgbouncer.nix

@@ -0,0 +1,136 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}: let
+  txPort = config.m3ta.ports.get "pgbouncer-tx";
+  sessionPort = config.m3ta.ports.get "pgbouncer-session";
+  pgPort = config.m3ta.ports.get "postgres";
+
+  # Podman-Bridge-IP des Hosts
+  hostBridgeIP = "10.89.0.1";
+
+  commonSettings = {
+    listen_addr = "127.0.0.1,${hostBridgeIP}";
+    auth_type = "scram-sha-256";
+    auth_file = config.age.secrets.pgbouncer-userlist.path;
+
+    admin_users = "sascha_koenig";
+    stats_users = "sascha_koenig";
+
+    log_connections = 1;
+    log_disconnections = 1;
+
+    server_reset_query = "DISCARD ALL";
+    server_check_query = "SELECT 1";
+    server_check_delay = 30;
+
+    server_tls_sslmode = "prefer";
+  };
+
+  mkDatabases = dbs:
+    lib.listToAttrs (map (db:
+      lib.nameValuePair db "host=127.0.0.1 port=${toString pgPort} dbname=${db}")
+    dbs);
+in {
+  services.pgbouncer = {
+    enable = true;
+
+    settings = {
+      pgbouncer =
+        commonSettings
+        // {
+          listen_port = txPort;
+          pool_mode = "transaction";
+
+          max_client_conn = 1000;
+          default_pool_size = 10;
+          min_pool_size = 2;
+          reserve_pool_size = 3;
+          reserve_pool_timeout = 3;
+
+          max_prepared_statements = 200;
+        };
+      databases = mkDatabases [
+        "baserow"
+        "litellm"
+        "librechat_rag"
+        "librechat_rag_dev"
+        "metabase"
+        "az_kpi_raw"
+      ];
+    };
+  };
+
+  systemd.services.pgbouncer = {
+    after = ["postgresql.service"];
+    requires = ["postgresql.service"];
+  };
+
+  environment.etc."pgbouncer/pgbouncer-session.ini".text = let
+    dbLines =
+      lib.concatStringsSep "\n"
+      (lib.mapAttrsToList (name: conn: "${name} = ${conn}")
+        (mkDatabases [
+          "outline"
+          "zammad"
+          "zammad_hr"
+          "vaultwarden"
+          "dash"
+        ]));
+  in ''
+    [databases]
+    ${dbLines}
+
+    [pgbouncer]
+    listen_addr = ${commonSettings.listen_addr}
+    listen_port = ${toString sessionPort}
+    pool_mode = session
+    auth_type = ${commonSettings.auth_type}
+    auth_file = ${config.age.secrets.pgbouncer-userlist.path}
+    admin_users = ${commonSettings.admin_users}
+    stats_users = ${commonSettings.stats_users}
+
+    max_client_conn = 300
+    # session-Mode: 1 Client-Conn = 1 Server-Conn. zammad_hr haelt via mehrere
+    # Prozesse (rails+scheduler+websocket) real ~15 Conns -> pro (user,db) 15.
+    # outline/vaultwarden schoepfen das nie aus (eigene Slots je user,db).
+    default_pool_size = 15
+    min_pool_size = 1
+    query_wait_timeout = 30
+
+    log_connections = 1
+    log_disconnections = 1
+    server_reset_query = DISCARD ALL
+    server_check_query = SELECT 1
+    server_check_delay = 30
+    server_tls_sslmode = prefer
+
+    # PID/Socket getrennt von der Haupt-Instanz halten.
+    pidfile = /run/pgbouncer-session/pgbouncer.pid
+    unix_socket_dir = /run/pgbouncer-session
+  '';
+
+  systemd.services.pgbouncer-session = {
+    description = "PgBouncer (session pool) for prepared-statement apps";
+    after = ["postgresql.service"];
+    requires = ["postgresql.service"];
+    wantedBy = ["multi-user.target"];
+    serviceConfig = {
+      User = "pgbouncer";
+      Group = "pgbouncer";
+      RuntimeDirectory = "pgbouncer-session";
+      ExecStart = "${pkgs.pgbouncer}/bin/pgbouncer /etc/pgbouncer/pgbouncer-session.ini";
+      Restart = "on-failure";
+      RestartSec = 5;
+    };
+  };
+
+  networking.firewall.extraCommands = ''
+    iptables -A INPUT -p tcp -s 127.0.0.1     --dport ${toString txPort}      -j ACCEPT
+    iptables -A INPUT -p tcp -s 10.89.0.0/24  --dport ${toString txPort}      -j ACCEPT
+    iptables -A INPUT -p tcp -s 127.0.0.1     --dport ${toString sessionPort} -j ACCEPT
+    iptables -A INPUT -p tcp -s 10.89.0.0/24  --dport ${toString sessionPort} -j ACCEPT
+  '';
+}

hosts/AZ-CLD-1/services/postgres.nix

@@ -12,8 +12,9 @@ in {
     package = pkgs.postgresql_17;
     settings = {
       ssl = true;
-      max_connections = 200;
+      max_connections = 100;
       shared_buffers = "4GB";
+      work_mem = "8MB";
       superuser_reserved_connections = 5;
 
       idle_in_transaction_session_timeout = "10min";
@@ -116,6 +117,17 @@ in {
       host    az_test             az_test             127.0.0.1/32       scram-sha-256
       host    az_test             az_test             ::1/128            scram-sha-256
 
+      host    baserow             baserow             127.0.0.1/32       scram-sha-256
+      host    baserow             baserow             ::1/128            scram-sha-256
+
+      host    librechat_rag       librechat_rag       127.0.0.1/32       scram-sha-256
+      host    librechat_rag       librechat_rag       ::1/128            scram-sha-256
+      host    librechat_rag_dev   librechat_rag_dev   127.0.0.1/32       scram-sha-256
+      host    librechat_rag_dev   librechat_rag_dev   ::1/128            scram-sha-256
+
+      host    litellm             litellm             127.0.0.1/32       scram-sha-256
+      host    litellm             litellm             ::1/128            scram-sha-256
+
       host    outline             outline             127.0.0.1/32       scram-sha-256
       host    outline             outline             ::1/128            scram-sha-256
 
@@ -128,16 +140,12 @@ in {
       host    vaultwarden         vaultwarden         127.0.0.1/32       scram-sha-256
       host    vaultwarden         vaultwarden         ::1/128            scram-sha-256
 
-      host    zammad              zammad              127.0.0.1/32       scram-sha-256
+      host    zammad_hr           zammad_hr           127.0.0.1/32       scram-sha-256
-      host    zammad              zammad              ::1/128            scram-sha-256
+      host    zammad_hr           zammad_hr           ::1/128            scram-sha-256
 
       # Podman network connections for Baserow
-      host    baserow             baserow             10.89.0.0/24       scram-sha-256
+      # host    baserow             baserow             10.89.0.0/24       scram-sha-256
       host    kestra              kestra              10.89.0.0/24       scram-sha-256
-      host    librechat_rag       librechat_rag       10.89.0.0/24       scram-sha-256
-      host    librechat_rag_dev   librechat_rag_dev   10.89.1.0/24       scram-sha-256
-      host    zammad_hr           zammad_hr           10.89.0.0/24       scram-sha-256
-      host    postgres            zammad_hr           10.89.0.0/24       scram-sha-256
       host    litellm             litellm             10.89.0.0/24       scram-sha-256
       host    netbird             netbird             10.89.0.0/24       scram-sha-256
 

hosts/common/ports.nix

@@ -20,6 +20,7 @@
       azion-scheduler = 3039;
       azion-scheduler-proxy = 3049;
       phishboard = 3055;
+      homarr = 3057;
 
       metabase = 3013;
       baserow = 3050;
@@ -42,6 +43,8 @@
       portainer = 9000;
 
       postgres = 5432;
+      pgbouncer-tx = 6432;
+      pgbouncer-session = 6433;
       pgadmin = 5050;
       mysql = 3306;
 

secrets.nix

@@ -44,8 +44,10 @@ in {
   "secrets/snipe-it-app-key.age".publicKeys = systems ++ users;
   "secrets/snipe-it-db-password.age".publicKeys = systems ++ users;
   "secrets/snipe-it-mail-password.age".publicKeys = systems ++ users;
+  "secrets/pgbouncer-userlist.age".publicKeys = systems ++ users;
   "secrets/pgadmin-pw.age".publicKeys = systems ++ users;
   "secrets/phishboard-env.age".publicKeys = systems ++ users;
+  "secrets/homarr-env.age".publicKeys = systems ++ users;
   "secrets/vaultwarden-env.age".publicKeys = systems ++ users;
   "secrets/vaultwarden-db.age".publicKeys = systems ++ users;
   "secrets/zammad-pw.age".publicKeys = systems ++ users;

secrets/litellm-env.age

@@ -1,20 +1,14 @@
------BEGIN AGE ENCRYPTED FILE-----
+age-encryption.org/v1
-YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHNzaC1lZDI1NTE5IFpoVnNlZyB0NW1T
+-> ssh-ed25519 ZhVseg P3zTtdI1xXB7fzI8tAyHrqu3smq5+uVn+BFUmkg77E4
-c0J6ZEVnaTVlMUh3c21GMmJ2ZStHQnErU2s3WWlCWERwUmNEUTNRCkFNSDhMa3BT
+0/FaClfU+bnyme+UIvtvt57Ah/7Mq+CZHUm38qU6qNY
-MTBNNzBKRm9tdW5jWlVmbHF6bEpGSHZ3aTRob2I1YmpTYmsKLT4gc3NoLWVkMjU1
+-> ssh-ed25519 SrHaqA 2lJ/qjvkmK2WbpIt0+lkYoJohV1ffL4JF4Tq1tKa8CY
-MTkgU3JIYXFBIHpCcXhSeE1hMll3cGJkb1ZhOWl1ZXA4L0RRV1ZNN1Mxa09paGRP
+yT5xn9AlfDdc9cuP0Ci0cSeXQZhc8F9rgGrleWCukH8
-YmliSG8KanJ1MTlDMExqdWgzVmVCTnFHeDBsNFRBdmtrTFJvcDhJdVp4Q0o5QkUv
+-> ssh-ed25519 lGqVZg 5fL5Akvl1qfSRvBiaXLvAbe1r++19onOzLbvrCL05CQ
-dwotPiBzc2gtZWQyNTUxOSBsR3FWWmcgNEVkYytqT05ZNjc3TFNKV1ZmS05zQkg2
+ldMBTTxpOboF/NYO0ZxCQUk+PzTtoRgpuBrit4F/zEs
-SmZlOTlVbzZObG5od3VDaGtEMAppSkdDMkZwUENDMzdDaGhHZ3FvUGhxREdiK3dk
+-> ssh-ed25519 CSMyhg NPvZ89Q1qRqFMNNkGoHMbSvJTMhvBMPLFkaXR783wl8
-MlU3a2VEOEU2NU5MMUdFCi0+IHNzaC1lZDI1NTE5IENTTXloZyBRR3l3MG0xa0Jz
+SMlZ5SpW5KKXAZUw+6U3wOmmfed83KJ/O6RJAi/6uWI
-U09rcWFIL2lEaCtvalVHVWtsclFNQ1BkM2lDN3grV2pZCll4dmp6SVJMSlJscjQx
+--- gDcEVCwfg5yURH9JLZbRXRP282xfSfC9c+3AWZs/Hjs
-WXFNMFRZYTMwa000aG8vWGx6TDd2QjM3ZEpmckUKLT4gMEAjLWdyZWFzZSBJWE5f
+möV9CO9¥p]ÙñײC‚9ðDd‡YVº.ˆØDoû„f)X„»ðJ�:írÅù•ˆL
-VXQKcjEzR0ZJeWJab2d4K3duN0owL01pQ1hpeHB3NXBzWFBYbWRtCi0tLSB4VnJx
+íµGÅþ®bî+^RÆ®3ÁâA‹æxQÉLûh–…€g™�‰³~Lm%
-TjFnQW1hc1U1emlJM1lIUncxejNzZUxwM0F1TG1zZUR2ZlIvUTdjCt0GcLJkRjMT
+ž°JY˜x(àÛr¨ðRr¸¯Æ
-C21Y89R2ob0cX8Th656sXau8Q2O/va78XNMKuYw7tFvk6ZBd4xd82vuy0SSxM+3f
+•¿Ýd–½sí–�þžZ:þZcXä°Hžêv#›ý[߇näY–�˜©}d‚eŪdnÖ©{7æcEa×ÈŠá[zeÅoXÆDÂ_¼;`¤f„v´_-L‘—	VBtTù�î±j
º£™•”ʈ%Aài'±y߀þ1�(Ñ›#¥44bÜ3k%FÇmîúº©k^
-3JTpbP7oexYsFDf5rg0tYVQbNvDU/sSyGzsLwIhiKNuyVavl268ptVGtYk+/oLOX
-+uogoOsBzcdRL9aUHAsWWwaaVx5wZrGHBlF6eAUlCY6JlN3Ofxt+wOAVq0I/xy9x
-iT7CO+qnCNJwH6e6kYY8GH0wXTCLDNmSver1yNUhtjt/T2qWcckRzkzLpt3wXpfU
-x38Sgisk0XBVbtHwRnZjWlZqCGLGk8nIg50EPKUfn73eNSDl6o3F4jT2RpWv+yLv
-TR4LjHI8wqlWsyKbMbE+5wb8mcI3EQ==
------END AGE ENCRYPTED FILE-----