diff options
Diffstat (limited to 'machines')
| -rw-r--r-- | machines/kusanagi/default.nix | 27 |
1 files changed, 17 insertions, 10 deletions
diff --git a/machines/kusanagi/default.nix b/machines/kusanagi/default.nix index 867a5eb..0585a40 100644 --- a/machines/kusanagi/default.nix +++ b/machines/kusanagi/default.nix @@ -226,19 +226,17 @@ package = pkgs.ollama-rocm; host = "0.0.0.0"; environmentVariables = { - # No HIP_VISIBLE_DEVICES / HSA_OVERRIDE_GFX_VERSION: the 7900 XT is - # native gfx1100 and discovery excludes the unsupported Raphael iGPU - # (gfx1036) on its own; ollama warns that overriding visible devices can - # itself break discovery. The past "runs 100% on CPU" incidents were the - # boot-order race handled below, not a gfx-target mismatch. - # KV-cache quantization (below) is a no-op without flash attention; ollama - # falls back to f16 KV cache, inflating VRAM use and forcing CPU offload. OLLAMA_FLASH_ATTENTION = "1"; OLLAMA_CONTEXT_LENGTH = "65536"; OLLAMA_KV_CACHE_TYPE = "q8_0"; - # Keep the model resident between agent turns; the default 5-minute - # keep-alive forces a multi-second reload after any pause. - OLLAMA_KEEP_ALIVE = "1h"; + OLLAMA_KEEP_ALIVE = "15m"; + OLLAMA_MAX_LOADED_MODELS = "1"; + OLLAMA_NUM_PARALLEL = "1"; + OLLAMA_MAX_QUEUE = "1"; + # Leave VRAM for Wayland clients so ollama offloads instead of trying + # to occupy the whole display GPU. + OLLAMA_GPU_OVERHEAD = "2147483648"; # 2 GiB, in bytes + LLAMA_ARG_FIT_TARGET = "2048"; # MiB free VRAM target for llama.cpp fit }; }; @@ -255,6 +253,15 @@ "dev-kfd.device" "dev-dri-renderD128.device" ]; + serviceConfig = { + # Hard guardrails for this no-swap desktop: if a model/request ignores the + # softer Ollama limits above, fail Ollama before the rest of the session. + MemoryHigh = "20G"; + MemoryMax = "24G"; + Nice = 10; + CPUWeight = 50; + IOWeight = 50; + }; }; services.immich = { |