From 9acf89a3af3fd4c354fcd887397b0c035bb21494 Mon Sep 17 00:00:00 2001
From: "Henry J. Webster" <hwebs@hwebs.info>
Date: Mon, 3 Aug 2026 22:51:26 -0500
Subject: kusanagi/ollama: fix boot race with amdgpu, drop misdiagnosed HSA
 override
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The recurring "ollama runs 100% on CPU" regressions were a boot-order
race, not a gfx-target mismatch: ollama.service only ordered after
network.target, and at boot it started between /dev/kfd and the 7900
XT's render node appearing. ollama probes GPUs once at startup, found
none, and silently fell back to CPU until the next manual restart —
which is why the previous HSA_OVERRIDE_GFX_VERSION commit "worked": the
deploy restarted the service on a long-running system, and the restart
was the actual cure.

The card is a 7900 XT (Navi 31), natively gfx1100 — verified that
discovery with no overrides finds it fine and drops only the
unsupported Raphael iGPU (gfx1036). Remove HSA_OVERRIDE_GFX_VERSION and
HIP_VISIBLE_DEVICES (ollama itself warns overriding visible devices can
break discovery).

Real fix: systemd-tag the kfd/renderD* char devices via udev (they get
no device units otherwise) and order ollama after them, so discovery
only runs once the GPU exists.

Assisted-by: claude-code:claude-fable-5
---
 machines/kusanagi/default.nix | 32 +++++++++++++++++++++++++-------
 1 file changed, 25 insertions(+), 7 deletions(-)

(limited to 'machines/kusanagi/default.nix')

diff --git a/machines/kusanagi/default.nix b/machines/kusanagi/default.nix
index bb719d1..ff786f6 100644
--- a/machines/kusanagi/default.nix
+++ b/machines/kusanagi/default.nix
@@ -42,10 +42,14 @@
   # --- Optical disc drives / archiving ---
   boot.kernelModules = [ "sg" ]; # for disc drive
 
-  # For disc drives
+  # For disc drives; plus systemd-tag the GPU compute nodes (kfd/renderD*) so
+  # ollama.service can order after them — char devices get no systemd device
+  # unit unless tagged.
   services.udev.extraRules = ''
     KERNEL=="sr[0-9]*", GROUP="cdrom", MODE="0660"
     KERNEL=="sg[0-9]*", GROUP="sg", MODE="0660"
+    SUBSYSTEM=="kfd", TAG+="systemd"
+    SUBSYSTEM=="drm", KERNEL=="renderD*", TAG+="systemd"
   '';
 
   services.pcscd.enable = true;
@@ -216,12 +220,11 @@
     enable = true;
     package = pkgs.ollama-rocm;
     environmentVariables = {
-      HIP_VISIBLE_DEVICES = "0";
-      # RDNA3 card reports as gfx1101/gfx1102/gfx11-generic, which the ROCm
-      # build bundled with ollama-rocm doesn't have kernels for -> it finds no
-      # usable device and silently runs 100% on CPU. Force it to present as
-      # gfx1100 (7900 XT/XTX), which is fully supported and binary-compatible.
-      HSA_OVERRIDE_GFX_VERSION = "11.0.0";
+      # No HIP_VISIBLE_DEVICES / HSA_OVERRIDE_GFX_VERSION: the 7900 XT is
+      # native gfx1100 and discovery excludes the unsupported Raphael iGPU
+      # (gfx1036) on its own; ollama warns that overriding visible devices can
+      # itself break discovery. The past "runs 100% on CPU" incidents were the
+      # boot-order race handled below, not a gfx-target mismatch.
       # KV-cache quantization (below) is a no-op without flash attention; ollama
       # falls back to f16 KV cache, inflating VRAM use and forcing CPU offload.
       OLLAMA_FLASH_ATTENTION = "1";
@@ -230,6 +233,21 @@
     };
   };
 
+  # ollama probes GPUs once at startup. At boot it raced amdgpu bring-up
+  # (started between /dev/kfd and the render node appearing) and permanently
+  # fell back to CPU. Order it after the tagged compute nodes (udev rules
+  # above) so discovery only runs once the card is really there.
+  systemd.services.ollama = {
+    wants = [
+      "dev-kfd.device"
+      "dev-dri-renderD128.device"
+    ];
+    after = [
+      "dev-kfd.device"
+      "dev-dri-renderD128.device"
+    ];
+  };
+
   services.immich = {
     enable = true;
     machine-learning.enable = false;
-- 
cgit v1.3