fix: litellm -> grafana
This commit is contained in:
Generated
+81
-87
@@ -66,11 +66,11 @@
|
||||
"agents": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
"lastModified": 1780133320,
|
||||
"narHash": "sha256-8AiN9tV9PBb5xblJiPlhumBbKj61qLjzqXXFtkj3vvY=",
|
||||
"lastModified": 1783016143,
|
||||
"narHash": "sha256-3qejDAo4g2XfzTyWCaX8RVKH17JXnw/fujsbXQA+Mzc=",
|
||||
"ref": "refs/heads/master",
|
||||
"rev": "920c00313ae242bd93275c30131b9ab1e52ee2fb",
|
||||
"revCount": 88,
|
||||
"rev": "83284d752d2ac325d4bf3ab7b38acad1c2fffe3a",
|
||||
"revCount": 98,
|
||||
"type": "git",
|
||||
"url": "https://code.m3ta.dev/m3tam3re/AGENTS"
|
||||
},
|
||||
@@ -84,11 +84,11 @@
|
||||
"nixpkgs": "nixpkgs_4"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1782789853,
|
||||
"narHash": "sha256-LEmctqYRQRq0wB1MoS+IVr/0/uu/0nKahqNeJBGTjJ8=",
|
||||
"lastModified": 1783016143,
|
||||
"narHash": "sha256-3qejDAo4g2XfzTyWCaX8RVKH17JXnw/fujsbXQA+Mzc=",
|
||||
"ref": "refs/heads/master",
|
||||
"rev": "cd36a91440b971582fcf2d4eedf9a46fb755f6e1",
|
||||
"revCount": 92,
|
||||
"rev": "83284d752d2ac325d4bf3ab7b38acad1c2fffe3a",
|
||||
"revCount": 98,
|
||||
"type": "git",
|
||||
"url": "ssh://gitea@code.m3ta.dev/m3tam3re/AGENTS"
|
||||
},
|
||||
@@ -409,18 +409,18 @@
|
||||
},
|
||||
"dms": {
|
||||
"inputs": {
|
||||
"flake-compat": "flake-compat",
|
||||
"nixpkgs": [
|
||||
"m3ta-home",
|
||||
"nixpkgs"
|
||||
],
|
||||
"quickshell": "quickshell"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1777431599,
|
||||
"narHash": "sha256-g6r/Gx8PTDzO3jCNzzySA+Ff1lmLF9nDlMCNyyoQjoE=",
|
||||
"lastModified": 1783529634,
|
||||
"narHash": "sha256-zdHsPGPE5MVi/y+uIt548XScTfZjQzdF21dME7ISEJM=",
|
||||
"owner": "AvengeMedia",
|
||||
"repo": "DankMaterialShell",
|
||||
"rev": "eb5afcdc40ea5446c27e18552ff4a19f9daf9484",
|
||||
"rev": "bdfd565b72cbb416441a9ab828efcac4fd516c70",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -438,11 +438,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1782780764,
|
||||
"narHash": "sha256-Q2wPFsuDo1y6neZ3ttxxOOItupg/4+mPkyAZbC0+wfw=",
|
||||
"lastModified": 1783591397,
|
||||
"narHash": "sha256-e4DDmVAas6IPwwY+gfSDKh3PMzW/FXLCbjyWXOnnnbM=",
|
||||
"owner": "AvengeMedia",
|
||||
"repo": "dms-plugin-registry",
|
||||
"rev": "9fa716427ab5905845b2a10a01d67bee0b1e4c4b",
|
||||
"rev": "8b61e8fca730986e673737529fa56802dc9326bb",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -451,6 +451,22 @@
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"flake-compat": {
|
||||
"flake": false,
|
||||
"locked": {
|
||||
"lastModified": 1767039857,
|
||||
"narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=",
|
||||
"owner": "NixOS",
|
||||
"repo": "flake-compat",
|
||||
"rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"repo": "flake-compat",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"flake-parts": {
|
||||
"inputs": {
|
||||
"nixpkgs-lib": [
|
||||
@@ -568,15 +584,16 @@
|
||||
"uv2nix": "uv2nix_2"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1782794050,
|
||||
"narHash": "sha256-+qKEwIhTkml3DNSmIvGFJyssW9ZXTp78KGclFdcDvX8=",
|
||||
"lastModified": 1782936460,
|
||||
"narHash": "sha256-Wt72AQtA6Eizi7Ubj23JBhwZ7GKYcjY4mcV6upqHOaU=",
|
||||
"owner": "NousResearch",
|
||||
"repo": "hermes-agent",
|
||||
"rev": "972b1620906a1b80772c2f67492ad50b3c83f048",
|
||||
"rev": "7c1a029553d87c43ecff8a3821336bc95872213b",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NousResearch",
|
||||
"ref": "v2026.7.1",
|
||||
"repo": "hermes-agent",
|
||||
"type": "github"
|
||||
}
|
||||
@@ -609,11 +626,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1781981105,
|
||||
"narHash": "sha256-/1nNBbA7PrSQpTc9Qazkhl4kIPg+TNl0CjxS3UQJKlw=",
|
||||
"lastModified": 1783221248,
|
||||
"narHash": "sha256-ESQnuNHEDChsB4IxoLRhscVahqkDWkTb+qdIz8euYt4=",
|
||||
"owner": "nix-community",
|
||||
"repo": "home-manager",
|
||||
"rev": "7bfff44b465909f69a442701293bc0badcf476dc",
|
||||
"rev": "af2beae5f0fae0a4310cc0e6aef2572f56090353",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -653,11 +670,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1782749631,
|
||||
"narHash": "sha256-slFTUgDy0KTPA4LBAmC/9SngDq8GCPdX+ZR0yQHHN1E=",
|
||||
"lastModified": 1783550008,
|
||||
"narHash": "sha256-qbbZUk9IG9dLNwCPwdPBs6I5clxwugRpP+1YU+GPK20=",
|
||||
"owner": "nix-community",
|
||||
"repo": "home-manager",
|
||||
"rev": "5d72a29fc36ac21adae6ae35568fe5ee6700850f",
|
||||
"rev": "f4d01c1d87c7c2ec909549165d5a8338f1bd3315",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -676,11 +693,11 @@
|
||||
"treefmt-nix": "treefmt-nix"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1783567605,
|
||||
"narHash": "sha256-LyF84yqWppXAAEOAkL325Lt+DrVmkbbeXGzDgX9zMzI=",
|
||||
"lastModified": 1783669679,
|
||||
"narHash": "sha256-WDGDMsPXxceC0k1ktqcobmwE4GCTrzzULUKlp1CzSKQ=",
|
||||
"owner": "numtide",
|
||||
"repo": "llm-agents.nix",
|
||||
"rev": "a4c847460f0e773d02d0655ce28dc6b532dd65d6",
|
||||
"rev": "6e9f6664e72433966557ab83a7c3e8c0bbf64a44",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -706,11 +723,11 @@
|
||||
"nur": "nur"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1783536074,
|
||||
"narHash": "sha256-oftWmLYtGzP81WdaCOPN0KxPp5rWdsnYcXfjh2h24AM=",
|
||||
"lastModified": 1783689486,
|
||||
"narHash": "sha256-dXHzh9nJTR3sp3le+5GCZVMsrKP4eKr4yDgJUi7+sD4=",
|
||||
"ref": "refs/heads/master",
|
||||
"rev": "32fdaddf7ca5018154bb97813c3492101ed9aa98",
|
||||
"revCount": 88,
|
||||
"rev": "c85600a3b073832d8edf83fd27b35811e36e4688",
|
||||
"revCount": 89,
|
||||
"type": "git",
|
||||
"url": "ssh://gitea@code.m3ta.dev/m3tam3re/m3ta-home"
|
||||
},
|
||||
@@ -731,11 +748,11 @@
|
||||
"openspec": "openspec"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1783016129,
|
||||
"narHash": "sha256-Uxie6uBNLUFJ8Tts7e6AYk5cOER5/EnCqKNL77Tzgc4=",
|
||||
"lastModified": 1783269665,
|
||||
"narHash": "sha256-bA4PooGV1x3vtmhs6NEprZbWErwaEJdgxoKfAWaH0do=",
|
||||
"ref": "refs/heads/master",
|
||||
"rev": "fc5092d72dda7db13f81e5910a159de5108590d5",
|
||||
"revCount": 350,
|
||||
"rev": "f279ae9c89d59f3e0deb0fd6307800f4cb993815",
|
||||
"revCount": 356,
|
||||
"type": "git",
|
||||
"url": "ssh://gitea@code.m3ta.dev/m3tam3re/nixpkgs"
|
||||
},
|
||||
@@ -753,11 +770,11 @@
|
||||
"openspec": "openspec_2"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1783269665,
|
||||
"narHash": "sha256-bA4PooGV1x3vtmhs6NEprZbWErwaEJdgxoKfAWaH0do=",
|
||||
"lastModified": 1783682547,
|
||||
"narHash": "sha256-usY1tQB2srsjjFPoKA6BRwoRTzP48ChiM2qu3vju85w=",
|
||||
"ref": "refs/heads/master",
|
||||
"rev": "f279ae9c89d59f3e0deb0fd6307800f4cb993815",
|
||||
"revCount": 356,
|
||||
"rev": "01bede8826fb0b35a0e432d788ce0a8fad2453f1",
|
||||
"revCount": 358,
|
||||
"type": "git",
|
||||
"url": "https://code.m3ta.dev/m3tam3re/nixpkgs"
|
||||
},
|
||||
@@ -857,11 +874,11 @@
|
||||
"treefmt-nix": "treefmt-nix_2"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1781681115,
|
||||
"narHash": "sha256-z8bD7qctIVWSv42xIvdPaw8IyC5hVTsoz4Nr6wAt46o=",
|
||||
"lastModified": 1782890500,
|
||||
"narHash": "sha256-UwamW3kYf/7vSsYq2VRb3VBq8A1hvOiLzXVnJeKVS38=",
|
||||
"owner": "nix-community",
|
||||
"repo": "nixos-anywhere",
|
||||
"rev": "13d4a2c51a896417289acc0664ed41936c6897e2",
|
||||
"rev": "4dfb813db065afb0aba1f61658ef77993d382db1",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -882,11 +899,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1781173814,
|
||||
"narHash": "sha256-lCoAwHfFXM0bhy7bY3QkN3XPW5HEyCwVOAgWyO3jl/w=",
|
||||
"lastModified": 1782383811,
|
||||
"narHash": "sha256-UFSWbDEltShkthOfvlAyxDq4L7dLHcsuJzHbgu79lHM=",
|
||||
"owner": "nix-community",
|
||||
"repo": "nixos-images",
|
||||
"rev": "bf315e58d33dcc5a04df17304166f64bc550983f",
|
||||
"rev": "45c188c452d274e003c9acc6b43fab911fa6cfa5",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -897,11 +914,11 @@
|
||||
},
|
||||
"nixos-stable": {
|
||||
"locked": {
|
||||
"lastModified": 1781216227,
|
||||
"narHash": "sha256-9mUW6gNwoN2SWc/l0fW4svPNOulXLl8ijqKyeSOGgJE=",
|
||||
"lastModified": 1782691344,
|
||||
"narHash": "sha256-i5nw9BYYsMDAaOC4J+JmTof6b2GhlyH076awYRNrTV8=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "a0374025a863d007d98e3297f6aa46cc3141c2f0",
|
||||
"rev": "1f01958ffb5b3545c96d9ef2f4e24c5e5e1eb846",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -1007,11 +1024,11 @@
|
||||
},
|
||||
"nixpkgs-unstable": {
|
||||
"locked": {
|
||||
"lastModified": 1781577229,
|
||||
"narHash": "sha256-lrp67w8AulE9Ks53n27I45ADSzbOCn4H+CNW1Ck8B+8=",
|
||||
"lastModified": 1783522502,
|
||||
"narHash": "sha256-iffAls3iaNTyJC2faYcUXSI+Gp02cDjYl+MygxKl2GI=",
|
||||
"owner": "nixos",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "567a49d1913ce81ac6e9582e3553dd90a955875f",
|
||||
"rev": "0bb7ec54c8483066ec9d7720e780a5caa71f8612",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -1071,11 +1088,11 @@
|
||||
},
|
||||
"nixpkgs_5": {
|
||||
"locked": {
|
||||
"lastModified": 1782723713,
|
||||
"narHash": "sha256-oPXCU/SSUokcGaJREHibG1CBX3+s/W7orDWQOZDsEeQ=",
|
||||
"lastModified": 1783224372,
|
||||
"narHash": "sha256-8i/87eeoqiGE4yOTjwSA3Eh/ziJRQEmd/unYU+K27sk=",
|
||||
"owner": "nixos",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "b5aa0fbd538984f6e3d201be0005b4463d8b09f8",
|
||||
"rev": "d407951447dcd00442e97087bf374aad70c04cea",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -1103,11 +1120,11 @@
|
||||
},
|
||||
"nixpkgs_7": {
|
||||
"locked": {
|
||||
"lastModified": 1781216227,
|
||||
"narHash": "sha256-9mUW6gNwoN2SWc/l0fW4svPNOulXLl8ijqKyeSOGgJE=",
|
||||
"lastModified": 1783389287,
|
||||
"narHash": "sha256-0xIy4dVLqq47rA+mRy0hXDfjhQd4E5PoIns/RmB7nR4=",
|
||||
"owner": "nixos",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "a0374025a863d007d98e3297f6aa46cc3141c2f0",
|
||||
"rev": "0ad6f47ea4fe188f4bc8f0380f93ae8523337c6c",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -1177,11 +1194,11 @@
|
||||
"nixpkgs": "nixpkgs_5"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1782796354,
|
||||
"narHash": "sha256-zHZEX+twYRO4n369Nrk21DBb519JZCGJLHBo/trhHO0=",
|
||||
"lastModified": 1783593872,
|
||||
"narHash": "sha256-KvFRxsi81gBoK7JOTHD2LoR1y2sWobHkrHGPhM3v/NY=",
|
||||
"owner": "nix-community",
|
||||
"repo": "NUR",
|
||||
"rev": "df4f4323fe161a8bab3aafa2d067ff9d7e223a67",
|
||||
"rev": "73078fa6f0d34686ad26808f90cc74e81039ba9f",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -1198,11 +1215,11 @@
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1782104183,
|
||||
"narHash": "sha256-BZD9AAFrbIJ+sIfUxI4wKs9B+J8e7gfOmewdcAgi/HY=",
|
||||
"lastModified": 1783688207,
|
||||
"narHash": "sha256-003aVBGJLPSn/kjlkDAMUaHTz0hBBN7QK1qjSJpCsIQ=",
|
||||
"owner": "nix-community",
|
||||
"repo": "NUR",
|
||||
"rev": "adda660080fa2677e1b3f368024956b15b066439",
|
||||
"rev": "bea54b7713e4a22b8c1c9d4a12c706d31d235462",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
@@ -1369,29 +1386,6 @@
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"quickshell": {
|
||||
"inputs": {
|
||||
"nixpkgs": [
|
||||
"m3ta-home",
|
||||
"dms",
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1776854048,
|
||||
"narHash": "sha256-lLbV66V3RMNp1l8/UelmR4YzoJ5ONtgvEtiUMJATH/o=",
|
||||
"ref": "refs/heads/master",
|
||||
"rev": "783c953987dc56ff0601abe6845ed96f1d00495a",
|
||||
"revCount": 806,
|
||||
"type": "git",
|
||||
"url": "https://git.outfoxxed.me/quickshell/quickshell"
|
||||
},
|
||||
"original": {
|
||||
"rev": "783c953987dc56ff0601abe6845ed96f1d00495a",
|
||||
"type": "git",
|
||||
"url": "https://git.outfoxxed.me/quickshell/quickshell"
|
||||
}
|
||||
},
|
||||
"root": {
|
||||
"inputs": {
|
||||
"agenix": "agenix",
|
||||
|
||||
@@ -1,20 +1,97 @@
|
||||
{config, ...}: let
|
||||
{
|
||||
config,
|
||||
pkgs,
|
||||
...
|
||||
}: let
|
||||
serviceName = "litellm";
|
||||
servicePort = config.m3ta.ports.get serviceName;
|
||||
prmNetbirdIP = "100.91.49.26";
|
||||
cldNetbirdIP = "100.91.203.184";
|
||||
|
||||
python = pkgs.python3.withPackages (ps: [ps.pyyaml]);
|
||||
litellmConfigGenerator = pkgs.writeText "litellm-config-generator.py" ''
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
base_path = Path(sys.argv[1])
|
||||
target_path = Path(sys.argv[2])
|
||||
|
||||
with base_path.open("r", encoding="utf-8") as fh:
|
||||
config = yaml.safe_load(fh) or {}
|
||||
|
||||
settings = config.get("litellm_settings")
|
||||
if not isinstance(settings, dict):
|
||||
settings = {}
|
||||
config["litellm_settings"] = settings
|
||||
|
||||
def ensure_list(value):
|
||||
if value is None:
|
||||
return []
|
||||
if isinstance(value, list):
|
||||
return value
|
||||
return [value]
|
||||
|
||||
callbacks = ensure_list(settings.get("callbacks"))
|
||||
if "prometheus" not in callbacks:
|
||||
callbacks.append("prometheus")
|
||||
settings["callbacks"] = callbacks
|
||||
|
||||
service_callbacks = ensure_list(settings.get("service_callback"))
|
||||
if "prometheus_system" not in service_callbacks:
|
||||
service_callbacks.append("prometheus_system")
|
||||
settings["service_callback"] = service_callbacks
|
||||
|
||||
# LiteLLM >= 1.85 protects /metrics by default. Keep auth enabled;
|
||||
# Prometheus scrapes with a bearer token from an agenix secret on AZ-PRM-1.
|
||||
settings["require_auth_for_metrics_endpoint"] = True
|
||||
|
||||
target_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with target_path.open("w", encoding="utf-8") as fh:
|
||||
yaml.safe_dump(config, fh, sort_keys=False)
|
||||
'';
|
||||
in {
|
||||
systemd.services."podman-${serviceName}".preStart = ''
|
||||
set -euo pipefail
|
||||
|
||||
base_config="/var/lib/${serviceName}/config.yaml"
|
||||
target_config="/run/${serviceName}/config.yaml"
|
||||
multiproc_dir="/var/lib/${serviceName}/prometheus-multiproc"
|
||||
|
||||
if [ ! -f "$base_config" ]; then
|
||||
echo "Missing LiteLLM base config: $base_config" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$(dirname "$target_config")" "$multiproc_dir"
|
||||
rm -f "$multiproc_dir"/*
|
||||
chmod 0777 "$multiproc_dir"
|
||||
|
||||
${python}/bin/python ${litellmConfigGenerator} "$base_config" "$target_config"
|
||||
chmod 0644 "$target_config"
|
||||
'';
|
||||
|
||||
virtualisation.oci-containers.containers.${serviceName} = {
|
||||
#image = "ghcr.io/berriai/litellm:v1.78.5-stable";
|
||||
image = "docker.litellm.ai/berriai/litellm:1.89.0";
|
||||
ports = ["127.0.0.1:${toString servicePort}:4000"];
|
||||
image = "docker.litellm.ai/berriai/litellm:1.92.0";
|
||||
ports = [
|
||||
"127.0.0.1:${toString servicePort}:4000"
|
||||
"${cldNetbirdIP}:${toString servicePort}:4000"
|
||||
];
|
||||
cmd = ["--config" "/app/config.yaml" "--port" "4000"];
|
||||
environmentFiles = [config.age.secrets.litellm-env.path];
|
||||
environment = {
|
||||
ANONYMIZED_TELEMETRY = "False";
|
||||
DO_NOT_TRACK = "True";
|
||||
SCARF_NO_ANALYTICS = "True";
|
||||
STORE_MODEL_IN_DB = "True";
|
||||
LITELLM_LOG = "json";
|
||||
PROMETHEUS_MULTIPROC_DIR = "/prometheus_multiproc";
|
||||
};
|
||||
volumes = ["/var/lib/litellm/config.yaml:/app/config.yaml"];
|
||||
volumes = [
|
||||
"/run/${serviceName}/config.yaml:/app/config.yaml:ro"
|
||||
"/var/lib/${serviceName}/prometheus-multiproc:/prometheus_multiproc"
|
||||
];
|
||||
extraOptions = ["--add-host=postgres:10.89.0.1" "--ip=10.89.0.30" "--network=web"];
|
||||
};
|
||||
|
||||
@@ -26,6 +103,23 @@ in {
|
||||
}
|
||||
];
|
||||
|
||||
middlewares."${serviceName}-metrics-local-only".ipAllowList.sourceRange = [
|
||||
"127.0.0.1/32"
|
||||
"::1/128"
|
||||
"${prmNetbirdIP}/32"
|
||||
];
|
||||
|
||||
routers."${serviceName}-metrics" = {
|
||||
rule = "Host(`llm.az-gruppe.com`) && PathPrefix(`/metrics`)";
|
||||
tls = {
|
||||
certResolver = "ionos";
|
||||
};
|
||||
service = serviceName;
|
||||
entrypoints = "websecure";
|
||||
middlewares = ["${serviceName}-metrics-local-only"];
|
||||
priority = 200;
|
||||
};
|
||||
|
||||
routers.${serviceName} = {
|
||||
rule = "Host(`llm.az-gruppe.com`)";
|
||||
tls = {
|
||||
@@ -33,6 +127,11 @@ in {
|
||||
};
|
||||
service = serviceName;
|
||||
entrypoints = "websecure";
|
||||
priority = 1;
|
||||
};
|
||||
};
|
||||
|
||||
networking.firewall.extraCommands = ''
|
||||
iptables -A INPUT -p tcp -s ${prmNetbirdIP} --dport ${toString servicePort} -j ACCEPT
|
||||
'';
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
{lib, ...}: let
|
||||
ntfyGrafanaWebhookSecret = ../../secrets/ntfy-grafana-webhook.age;
|
||||
litellmPrometheusBearerSecret = ../../secrets/litellm-prometheus-bearer.age;
|
||||
in {
|
||||
age = {
|
||||
secrets =
|
||||
@@ -64,6 +65,14 @@ in {
|
||||
file = ntfyGrafanaWebhookSecret;
|
||||
mode = "0400";
|
||||
};
|
||||
}
|
||||
// lib.optionalAttrs (builtins.pathExists litellmPrometheusBearerSecret) {
|
||||
litellm-prometheus-bearer = {
|
||||
file = litellmPrometheusBearerSecret;
|
||||
owner = "prometheus";
|
||||
group = "prometheus";
|
||||
mode = "0400";
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
@@ -195,6 +195,47 @@ in {
|
||||
};
|
||||
annotations.description = "A Kestra flow execution ended with FAILED, WARNING, or KILLED.";
|
||||
})
|
||||
(mkAlertRule {
|
||||
uid = "az_litellm_down";
|
||||
title = "LiteLLM metrics unreachable";
|
||||
expr = ''up{job="litellm"}'';
|
||||
for = "2m";
|
||||
evaluatorType = "lt";
|
||||
evaluatorParams = [1 0];
|
||||
labels = {
|
||||
service = "litellm";
|
||||
severity = "critical";
|
||||
};
|
||||
annotations.description = "Prometheus cannot scrape LiteLLM /metrics on AZ-CLD-1 over Netbird.";
|
||||
})
|
||||
(mkAlertRule {
|
||||
uid = "az_litellm_proxy_errors";
|
||||
title = "LiteLLM proxy errors";
|
||||
expr = ''sum(increase(litellm_proxy_failed_requests_metric_total{job="litellm"}[5m]))'';
|
||||
for = "1m";
|
||||
noDataState = "OK";
|
||||
evaluatorType = "gt";
|
||||
evaluatorParams = [0 0];
|
||||
labels = {
|
||||
service = "litellm";
|
||||
severity = "warning";
|
||||
};
|
||||
annotations.description = "LiteLLM reported one or more failed proxy responses in the last 5 minutes.";
|
||||
})
|
||||
(mkAlertRule {
|
||||
uid = "az_litellm_latency_high";
|
||||
title = "LiteLLM latency high";
|
||||
expr = ''histogram_quantile(0.95, sum(rate(litellm_request_total_latency_metric_bucket{job="litellm"}[5m])) by (le))'';
|
||||
for = "10m";
|
||||
noDataState = "OK";
|
||||
evaluatorType = "gt";
|
||||
evaluatorParams = [30 0];
|
||||
labels = {
|
||||
service = "litellm";
|
||||
severity = "warning";
|
||||
};
|
||||
annotations.description = "LiteLLM p95 total request latency has been above 30 seconds for 10 minutes.";
|
||||
})
|
||||
(mkAlertRule {
|
||||
uid = "az_http_probe_failed";
|
||||
title = "HTTP probe failed";
|
||||
|
||||
@@ -1,7 +1,23 @@
|
||||
{config, ...}: let
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
...
|
||||
}: let
|
||||
nodeExporterPort = config.m3ta.ports.get "node-exporter";
|
||||
litellmPort = config.m3ta.ports.get "litellm";
|
||||
cldNetbirdIP = "100.91.203.184";
|
||||
litellmPrometheusBearerSecretAvailable = builtins.hasAttr "litellm-prometheus-bearer" config.age.secrets;
|
||||
litellmPrometheusAuthorization = lib.optionalAttrs litellmPrometheusBearerSecretAvailable {
|
||||
authorization = {
|
||||
type = "Bearer";
|
||||
credentials_file = config.age.secrets."litellm-prometheus-bearer".path;
|
||||
};
|
||||
};
|
||||
in {
|
||||
# Prometheus' config checker validates credentials_file paths at build time,
|
||||
# but agenix secrets only exist at runtime under /run/agenix.
|
||||
services.prometheus.checkConfig = lib.mkIf litellmPrometheusBearerSecretAvailable false;
|
||||
|
||||
services.prometheus.scrapeConfigs = [
|
||||
{
|
||||
job_name = "node-cld";
|
||||
@@ -14,11 +30,26 @@ in {
|
||||
}
|
||||
];
|
||||
}
|
||||
({
|
||||
job_name = "litellm";
|
||||
metrics_path = "/metrics/";
|
||||
scrape_interval = "15s";
|
||||
static_configs = [
|
||||
{
|
||||
targets = ["${cldNetbirdIP}:${toString litellmPort}"];
|
||||
labels = {
|
||||
instance = "AZ-CLD-1";
|
||||
service = "litellm";
|
||||
};
|
||||
}
|
||||
];
|
||||
}
|
||||
// litellmPrometheusAuthorization)
|
||||
];
|
||||
|
||||
# Allow CLD to reach PRM prometheus scrapes (prometheus initiates connection TO CLD exporter)
|
||||
# Allow CLD to reach PRM prometheus scrapes (prometheus initiates connection TO CLD exporters)
|
||||
networking.firewall.extraCommands = ''
|
||||
# No PRM-side firewall change needed: PRM scrapes outbound to CLD:9100.
|
||||
# No PRM-side firewall change needed: PRM scrapes outbound to CLD:9100/4000.
|
||||
# CLD-side must allow inbound from 100.91.49.26 (PRM netbird IP) — see CLD config.
|
||||
'';
|
||||
}
|
||||
|
||||
@@ -0,0 +1,568 @@
|
||||
{
|
||||
"annotations": {
|
||||
"list": [
|
||||
{
|
||||
"builtIn": 1,
|
||||
"datasource": {
|
||||
"type": "grafana",
|
||||
"uid": "-- Grafana --"
|
||||
},
|
||||
"enable": true,
|
||||
"hide": true,
|
||||
"iconColor": "rgba(0, 211, 255, 1)",
|
||||
"name": "Annotations & Alerts",
|
||||
"target": {
|
||||
"limit": 100,
|
||||
"matchAny": false,
|
||||
"tags": [],
|
||||
"type": "dashboard"
|
||||
},
|
||||
"type": "dashboard"
|
||||
}
|
||||
]
|
||||
},
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 1,
|
||||
"links": [],
|
||||
"liveNow": false,
|
||||
"panels": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"mappings": [
|
||||
{
|
||||
"options": {
|
||||
"0": {
|
||||
"color": "red",
|
||||
"index": 1,
|
||||
"text": "down"
|
||||
},
|
||||
"1": {
|
||||
"color": "green",
|
||||
"index": 0,
|
||||
"text": "up"
|
||||
}
|
||||
},
|
||||
"type": "value"
|
||||
}
|
||||
],
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "red",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "green",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"id": 1,
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
"graphMode": "none",
|
||||
"justifyMode": "center",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"pluginVersion": "11.0.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "up{job=\"litellm\", instance=~\"$instance\"}",
|
||||
"legendFormat": "{{instance}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Scrape status",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"decimals": 3,
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 6,
|
||||
"y": 0
|
||||
},
|
||||
"id": 2,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"pluginVersion": "11.0.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(litellm_proxy_total_requests_metric_total{job=\"litellm\", instance=~\"$instance\"}[$__rate_interval]))",
|
||||
"legendFormat": "requests/s",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Proxy request rate",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"decimals": 3,
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 12,
|
||||
"y": 0
|
||||
},
|
||||
"id": 3,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"pluginVersion": "11.0.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum(rate(litellm_proxy_failed_requests_metric_total{job=\"litellm\", instance=~\"$instance\"}[$__rate_interval])) or vector(0)",
|
||||
"legendFormat": "errors/s",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Proxy error rate",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"decimals": 2,
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 4,
|
||||
"w": 6,
|
||||
"x": 18,
|
||||
"y": 0
|
||||
},
|
||||
"id": 4,
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
},
|
||||
"pluginVersion": "11.0.0",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "histogram_quantile(0.95, sum(rate(litellm_request_total_latency_metric_bucket{job=\"litellm\", instance=~\"$instance\"}[$__rate_interval])) by (le))",
|
||||
"legendFormat": "p95",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "p95 total latency",
|
||||
"type": "stat"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 2,
|
||||
"showPoints": "never",
|
||||
"spanNulls": false
|
||||
},
|
||||
"unit": "reqps"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 4
|
||||
},
|
||||
"id": 5,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (requested_model) (rate(litellm_proxy_total_requests_metric_total{job=\"litellm\", instance=~\"$instance\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{requested_model}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Requests by requested model",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 2,
|
||||
"showPoints": "never",
|
||||
"spanNulls": false
|
||||
},
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 4
|
||||
},
|
||||
"id": 6,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (model) (rate(litellm_total_tokens_metric_total{job=\"litellm\", instance=~\"$instance\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{model}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Token rate by model",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "currencyUSD"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 0,
|
||||
"y": 12
|
||||
},
|
||||
"id": 7,
|
||||
"options": {
|
||||
"displayMode": "gradient",
|
||||
"legend": {
|
||||
"calcs": [],
|
||||
"displayMode": "list",
|
||||
"placement": "bottom",
|
||||
"showLegend": false
|
||||
},
|
||||
"orientation": "horizontal",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": true
|
||||
},
|
||||
"showUnfilled": true,
|
||||
"valueMode": "color"
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "topk(10, sum by (model) (increase(litellm_spend_metric_total{job=\"litellm\", instance=~\"$instance\"}[$__range])))",
|
||||
"legendFormat": "{{model}}",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "Spend by model in selected range",
|
||||
"type": "bargauge"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"custom": {
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 10,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 2,
|
||||
"showPoints": "never",
|
||||
"spanNulls": false
|
||||
},
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
"w": 12,
|
||||
"x": 12,
|
||||
"y": 12
|
||||
},
|
||||
"id": 8,
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"mode": "single",
|
||||
"sort": "none"
|
||||
}
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "histogram_quantile(0.95, sum by (le, model) (rate(litellm_llm_api_latency_metric_bucket{job=\"litellm\", instance=~\"$instance\"}[$__rate_interval])))",
|
||||
"legendFormat": "{{model}} p95",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "LLM API latency by model",
|
||||
"type": "timeseries"
|
||||
},
|
||||
{
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 10,
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 20
|
||||
},
|
||||
"id": 9,
|
||||
"options": {
|
||||
"dedupStrategy": "none",
|
||||
"enableLogDetails": true,
|
||||
"prettifyLogMessage": false,
|
||||
"showCommonLabels": false,
|
||||
"showLabels": false,
|
||||
"showTime": true,
|
||||
"sortOrder": "Descending",
|
||||
"wrapLogMessage": true
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
"type": "loki",
|
||||
"uid": "loki"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "{host=\"AZ-CLD-1\", unit=~\"podman-litellm.service|docker-litellm.service|litellm.service\"}",
|
||||
"queryType": "range",
|
||||
"refId": "A"
|
||||
}
|
||||
],
|
||||
"title": "LiteLLM logs from Loki",
|
||||
"type": "logs"
|
||||
}
|
||||
],
|
||||
"refresh": "30s",
|
||||
"schemaVersion": 39,
|
||||
"style": "dark",
|
||||
"tags": [
|
||||
"az",
|
||||
"litellm",
|
||||
"prometheus",
|
||||
"loki",
|
||||
"provisioned"
|
||||
],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"current": {
|
||||
"selected": true,
|
||||
"text": "All",
|
||||
"value": "$__all"
|
||||
},
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "prometheus"
|
||||
},
|
||||
"definition": "label_values(up{job=\"litellm\"}, instance)",
|
||||
"hide": 0,
|
||||
"includeAll": true,
|
||||
"label": "Instance",
|
||||
"multi": true,
|
||||
"name": "instance",
|
||||
"options": [],
|
||||
"query": {
|
||||
"query": "label_values(up{job=\"litellm\"}, instance)",
|
||||
"refId": "PrometheusVariableQueryEditor-VariableQuery"
|
||||
},
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"skipUrlSync": false,
|
||||
"sort": 1,
|
||||
"type": "query"
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": {
|
||||
"from": "now-6h",
|
||||
"to": "now"
|
||||
},
|
||||
"timepicker": {},
|
||||
"timezone": "browser",
|
||||
"title": "AZ LiteLLM Observability",
|
||||
"uid": "az-litellm-observability",
|
||||
"version": 1,
|
||||
"weekStart": ""
|
||||
}
|
||||
@@ -26,6 +26,7 @@ in {
|
||||
"secrets/librechat-env-prod.age".publicKeys = systems ++ users;
|
||||
"secrets/librechat-env-dev.age".publicKeys = systems ++ users;
|
||||
"secrets/litellm-env.age".publicKeys = systems ++ users;
|
||||
"secrets/litellm-prometheus-bearer.age".publicKeys = [AZ-PRM-1] ++ users;
|
||||
"secrets/metabase-env.age".publicKeys = systems ++ users;
|
||||
"secrets/n8n-env.age".publicKeys = systems ++ users;
|
||||
"secrets/n8n-env-prm.age".publicKeys = systems ++ users;
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
-----BEGIN AGE ENCRYPTED FILE-----
|
||||
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHNzaC1lZDI1NTE5IGxHcVZaZyBJZkVq
|
||||
aWU5RTljeXR1ZTdWcmQrL21aMDB1aTQ0Q1hrVWJWVzFSUmorLzBvCmhRWUxjaWsx
|
||||
aHczT0h0Z2NhbGhuMjZqRHdMdGRGeU9ueDd1eHJwdk90Y0EKLT4gc3NoLWVkMjU1
|
||||
MTkgQ1NNeWhnIEpMbUlxZ2xLR3NQOXhlZzFlWGdrNkd4N3pKWWVJN1I1bjZ5clJZ
|
||||
WTR0amsKZW5peVlnZzZvYUxtdU5COGNqVFRoRVQrQ2ludU9XTjc2V2tQak0zL0xP
|
||||
dwotPiBiWTt8OyUlZi1ncmVhc2UgRjEyIGUqUykgNV1WVykgdyFTe0wiYwp5aGZw
|
||||
bjRPeitGTlhzNVFpbUNuSEw3T1V2YVJlT2IxUkJRCi0tLSA3amRwTWtNOVh1cm1R
|
||||
WElER3NYZVZGeXJNdFQwTno5cnFXQWVraUNDOFVVCjQsmAl0ywy880NBAG+Zv2Wz
|
||||
jAOi0Sm6BNRdDkkfUGq1i63qy4ZnFxVGlAwmsmIxseicJ2sqteUmlqN2eqLkC3jO
|
||||
-----END AGE ENCRYPTED FILE-----
|
||||
Reference in New Issue
Block a user