138 lines
4.1 KiB
Nix
138 lines
4.1 KiB
Nix
{
|
|
config,
|
|
pkgs,
|
|
...
|
|
}: let
|
|
serviceName = "litellm";
|
|
servicePort = config.m3ta.ports.get serviceName;
|
|
prmNetbirdIP = "100.91.49.26";
|
|
cldNetbirdIP = "100.91.203.184";
|
|
|
|
python = pkgs.python3.withPackages (ps: [ps.pyyaml]);
|
|
litellmConfigGenerator = pkgs.writeText "litellm-config-generator.py" ''
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import yaml
|
|
|
|
base_path = Path(sys.argv[1])
|
|
target_path = Path(sys.argv[2])
|
|
|
|
with base_path.open("r", encoding="utf-8") as fh:
|
|
config = yaml.safe_load(fh) or {}
|
|
|
|
settings = config.get("litellm_settings")
|
|
if not isinstance(settings, dict):
|
|
settings = {}
|
|
config["litellm_settings"] = settings
|
|
|
|
def ensure_list(value):
|
|
if value is None:
|
|
return []
|
|
if isinstance(value, list):
|
|
return value
|
|
return [value]
|
|
|
|
callbacks = ensure_list(settings.get("callbacks"))
|
|
if "prometheus" not in callbacks:
|
|
callbacks.append("prometheus")
|
|
settings["callbacks"] = callbacks
|
|
|
|
service_callbacks = ensure_list(settings.get("service_callback"))
|
|
if "prometheus_system" not in service_callbacks:
|
|
service_callbacks.append("prometheus_system")
|
|
settings["service_callback"] = service_callbacks
|
|
|
|
# LiteLLM >= 1.85 protects /metrics by default. Keep auth enabled;
|
|
# Prometheus scrapes with a bearer token from an agenix secret on AZ-PRM-1.
|
|
settings["require_auth_for_metrics_endpoint"] = True
|
|
|
|
target_path.parent.mkdir(parents=True, exist_ok=True)
|
|
with target_path.open("w", encoding="utf-8") as fh:
|
|
yaml.safe_dump(config, fh, sort_keys=False)
|
|
'';
|
|
in {
|
|
systemd.services."podman-${serviceName}".preStart = ''
|
|
set -euo pipefail
|
|
|
|
base_config="/var/lib/${serviceName}/config.yaml"
|
|
target_config="/run/${serviceName}/config.yaml"
|
|
multiproc_dir="/var/lib/${serviceName}/prometheus-multiproc"
|
|
|
|
if [ ! -f "$base_config" ]; then
|
|
echo "Missing LiteLLM base config: $base_config" >&2
|
|
exit 1
|
|
fi
|
|
|
|
mkdir -p "$(dirname "$target_config")" "$multiproc_dir"
|
|
rm -f "$multiproc_dir"/*
|
|
chmod 0777 "$multiproc_dir"
|
|
|
|
${python}/bin/python ${litellmConfigGenerator} "$base_config" "$target_config"
|
|
chmod 0644 "$target_config"
|
|
'';
|
|
|
|
virtualisation.oci-containers.containers.${serviceName} = {
|
|
#image = "ghcr.io/berriai/litellm:v1.78.5-stable";
|
|
image = "docker.litellm.ai/berriai/litellm:1.92.0";
|
|
ports = [
|
|
"127.0.0.1:${toString servicePort}:4000"
|
|
"${cldNetbirdIP}:${toString servicePort}:4000"
|
|
];
|
|
cmd = ["--config" "/app/config.yaml" "--port" "4000"];
|
|
environmentFiles = [config.age.secrets.litellm-env.path];
|
|
environment = {
|
|
ANONYMIZED_TELEMETRY = "False";
|
|
DO_NOT_TRACK = "True";
|
|
SCARF_NO_ANALYTICS = "True";
|
|
STORE_MODEL_IN_DB = "True";
|
|
PROMETHEUS_MULTIPROC_DIR = "/prometheus_multiproc";
|
|
};
|
|
volumes = [
|
|
"/run/${serviceName}/config.yaml:/app/config.yaml:ro"
|
|
"/var/lib/${serviceName}/prometheus-multiproc:/prometheus_multiproc"
|
|
];
|
|
extraOptions = ["--add-host=postgres:10.89.0.1" "--ip=10.89.0.30" "--network=web"];
|
|
};
|
|
|
|
# Traefik configuration
|
|
services.traefik.dynamicConfigOptions.http = {
|
|
services.${serviceName}.loadBalancer.servers = [
|
|
{
|
|
url = "http://localhost:${toString servicePort}/";
|
|
}
|
|
];
|
|
|
|
middlewares."${serviceName}-metrics-local-only".ipAllowList.sourceRange = [
|
|
"127.0.0.1/32"
|
|
"::1/128"
|
|
"${prmNetbirdIP}/32"
|
|
];
|
|
|
|
routers."${serviceName}-metrics" = {
|
|
rule = "Host(`llm.az-gruppe.com`) && PathPrefix(`/metrics`)";
|
|
tls = {
|
|
certResolver = "ionos";
|
|
};
|
|
service = serviceName;
|
|
entrypoints = "websecure";
|
|
middlewares = ["${serviceName}-metrics-local-only"];
|
|
priority = 200;
|
|
};
|
|
|
|
routers.${serviceName} = {
|
|
rule = "Host(`llm.az-gruppe.com`)";
|
|
tls = {
|
|
certResolver = "ionos";
|
|
};
|
|
service = serviceName;
|
|
entrypoints = "websecure";
|
|
priority = 1;
|
|
};
|
|
};
|
|
|
|
networking.firewall.extraCommands = ''
|
|
iptables -A INPUT -p tcp -s ${prmNetbirdIP} --dport ${toString servicePort} -j ACCEPT
|
|
'';
|
|
}
|