Запускаем LLM на AMD RX580: разбор проблем ROCm, Ollama и реальный GPU inference

Мы пытались запустить LLM inference на старой AMD RX580 (8 VRAM) через ROCm в Kubernetes. GPU корректно определялся, VRAM использовалась, но inference падал с ошибками вида:

hipMemGetInfo(free, total) CUDA error: invalid argument

После серии экспериментов с ROCm userspace, Docker‑образами и Kubernetes deployment выяснилось, что проблема лежит на границе:

kernel → ROCm runtime → ggml backend

Финальное решение включало:

• переход на kernel 6.8

• стабилизацию ROCm runtime

• использование llama.cpp + ROCm

• grammar‑constrained decoding для strict sanity prompts

В итоге мы получили стабильный GPU inference:

• ~42 токен/сек

• gpu_busy_percent → до 100%

на обычной RX580.

#!/usr/bin/env python3
import glob
import os
import signal
import sys
import time
from dataclasses import dataclass


@dataclass
class Config:
    # Цель по температуре
    target_temp_c: float = 45.0

    # Температурные зоны
    idle_temp_c: float = 42.0
    warm_temp_c: float = 48.0
    hot_temp_c: float = 55.0
    very_hot_temp_c: float = 68.0
    emergency_temp_c: float = 75.0

    # PWM границы
    min_pwm: int = 88
    idle_pwm: int = 95
    base_pwm: int = 108
    max_pwm: int = 255
    emergency_pwm: int = 255

    # PID-подобные коэффициенты
    kp: float = 7.0
    ki: float = 0.05
    kd: float = 14.0

    # Упреждающая реакция на загрузку GPU
    busy_gain: float = 0.8
    busy_threshold: float = 5.0

    # Поведение
    interval_sec: float = 2.0
    hysteresis_c: float = 0.5

    # Ограничение изменения PWM за шаг
    max_pwm_step_up: int = 16
    max_pwm_step_down: int = 8

    # Чтобы не дёргать ШИМ по мелочи
    min_effective_pwm_delta: int = 2

    # Антивиндап
    integral_min: float = -250.0
    integral_max: float = 350.0

    # Если очень холодно и GPU почти не занят
    very_cool_temp_c: float = 38.0
    very_cool_busy_max: float = 10.0

    # Усиление реакции в горячих зонах
    hot_zone_boost_pwm: int = 12
    very_hot_zone_boost_pwm: int = 28


class AmdGpuFanController:
    def __init__(self, cfg: Config):
        self.cfg = cfg
        self.hwmon_path = self._find_hwmon()
        self.card_path = "/sys/class/drm/card0/device"

        self.temp_path = os.path.join(self.hwmon_path, "temp1_input")
        self.pwm_path = os.path.join(self.hwmon_path, "pwm1")
        self.pwm_enable_path = os.path.join(self.hwmon_path, "pwm1_enable")
        self.fan_rpm_path = os.path.join(self.hwmon_path, "fan1_input")
        self.gpu_busy_path = os.path.join(self.card_path, "gpu_busy_percent")

        self.integral = 0.0
        self.last_temp_c = None
        self.last_busy = None
        self.last_pwm = None
        self.running = True

    def _find_hwmon(self) -> str:
        # сначала старый путь (иногда используется)
        matches = glob.glob("/sys/class/drm/card0/device/hwmon/hwmon*")
        if matches:
            return matches[0]
    
        # стандартный путь через /sys/class/hwmon
        for path in glob.glob("/sys/class/hwmon/hwmon*"):
            try:
                with open(os.path.join(path, "name")) as f:
                    if f.read().strip() == "amdgpu":
                        return path
            except Exception:
                pass

        raise RuntimeError("amdgpu hwmon device not found")
    def _read_int(self, path: str, default: int = 0) -> int:
        try:
            with open(path, "r") as f:
                return int(f.read().strip())
        except Exception:
            return default

    def _write_int(self, path: str, value: int) -> None:
        with open(path, "w") as f:
            f.write(str(value))

    def read_temp_c(self) -> float:
        return self._read_int(self.temp_path) / 1000.0

    def read_pwm(self) -> int:
        return self._read_int(self.pwm_path)

    def read_rpm(self) -> int:
        return self._read_int(self.fan_rpm_path, default=-1)

    def read_gpu_busy(self) -> float:
        return float(self._read_int(self.gpu_busy_path, default=0))

    def set_manual_mode(self) -> None:
        self._write_int(self.pwm_enable_path, 1)

    def set_auto_mode(self) -> None:
        self._write_int(self.pwm_enable_path, 2)

    def set_pwm(self, pwm: int) -> None:
        pwm = max(self.cfg.min_pwm, min(self.cfg.max_pwm, int(round(pwm))))
        self._write_int(self.pwm_path, pwm)
        self.last_pwm = pwm

    @staticmethod
    def clamp(value: float, lo: float, hi: float) -> float:
        return max(lo, min(hi, value))

    def rate_limit_pwm(self, target_pwm: int) -> int:
        if self.last_pwm is None:
            return target_pwm
        if target_pwm > self.last_pwm:
            return min(target_pwm, self.last_pwm + self.cfg.max_pwm_step_up)
        return max(target_pwm, self.last_pwm - self.cfg.max_pwm_step_down)

    def compute_target_pwm(self, temp_c: float, busy: float, dtemp_dt: float) -> int:
        if temp_c >= self.cfg.emergency_temp_c:
            self.integral = 0.0
            return self.cfg.emergency_pwm

        if temp_c <= self.cfg.very_cool_temp_c and busy <= self.cfg.very_cool_busy_max:
            self.integral *= 0.85
            return self.cfg.idle_pwm

        error = temp_c - self.cfg.target_temp_c
        effective_error = 0.0 if abs(error) < self.cfg.hysteresis_c else error

        self.integral += effective_error * self.cfg.interval_sec
        self.integral = self.clamp(
            self.integral,
            self.cfg.integral_min,
            self.cfg.integral_max,
        )

        busy_term = 0.0
        if busy > self.cfg.busy_threshold:
            busy_term = (busy - self.cfg.busy_threshold) * self.cfg.busy_gain

        pwm = (
            self.cfg.base_pwm
            + self.cfg.kp * effective_error
            + self.cfg.ki * self.integral
            + self.cfg.kd * dtemp_dt
            + busy_term
        )

        if temp_c >= self.cfg.hot_temp_c:
            pwm += self.cfg.hot_zone_boost_pwm
        if temp_c >= self.cfg.very_hot_temp_c:
            pwm += self.cfg.very_hot_zone_boost_pwm

        if temp_c >= self.cfg.warm_temp_c and busy >= 40:
            pwm = max(pwm, self.cfg.base_pwm + 18)

        if temp_c <= self.cfg.idle_temp_c and busy < 20:
            pwm = min(pwm, self.cfg.idle_pwm + 8)

        return int(round(self.clamp(pwm, self.cfg.min_pwm, self.cfg.max_pwm)))

    def control_step(self) -> None:
        temp_c = self.read_temp_c()
        rpm = self.read_rpm()
        busy = self.read_gpu_busy()

        if self.last_pwm is None:
            self.last_pwm = self.read_pwm()

        if self.last_temp_c is None:
            dtemp_dt = 0.0
        else:
            dtemp_dt = (temp_c - self.last_temp_c) / self.cfg.interval_sec

        target_pwm = self.compute_target_pwm(temp_c=temp_c, busy=busy, dtemp_dt=dtemp_dt)
        limited_pwm = self.rate_limit_pwm(target_pwm)

        if self.last_pwm is None or abs(limited_pwm - self.last_pwm) >= self.cfg.min_effective_pwm_delta:
            self.set_pwm(limited_pwm)
        else:
            limited_pwm = self.last_pwm

        error = temp_c - self.cfg.target_temp_c

        print(
            f"temp={temp_c:5.1f}C "
            f"busy={busy:5.1f}% "
            f"rpm={rpm:4d} "
            f"err={error:+5.1f} "
            f"dT/dt={dtemp_dt:+5.2f}C/s "
            f"int={self.integral:+7.1f} "
            f"pwm={limited_pwm:3d}",
            flush=True,
        )

        self.last_temp_c = temp_c
        self.last_busy = busy

    def run(self) -> None:
        self.set_manual_mode()

        if self.last_pwm is None:
            try:
                self.last_pwm = self.read_pwm()
            except Exception:
                self.last_pwm = self.cfg.base_pwm
                self.set_pwm(self.last_pwm)

        print(f"Using hwmon path: {self.hwmon_path}")
        print("Manual fan control enabled.")
        print(
            f"Target={self.cfg.target_temp_c}C, "
            f"idle={self.cfg.idle_temp_c}C, "
            f"warm={self.cfg.warm_temp_c}C, "
            f"hot={self.cfg.hot_temp_c}C, "
            f"very_hot={self.cfg.very_hot_temp_c}C, "
            f"emergency={self.cfg.emergency_temp_c}C",
            flush=True,
        )

        while self.running:
            self.control_step()
            time.sleep(self.cfg.interval_sec)

    def stop(self, restore_auto: bool = True) -> None:
        self.running = False
        if restore_auto:
            try:
                self.set_auto_mode()
                print("Restored automatic fan control.", flush=True)
            except Exception as e:
                print(f"Failed to restore auto mode: {e}", file=sys.stderr, flush=True)


def main() -> int:
    cfg = Config()
    ctl = AmdGpuFanController(cfg)

    def _handle_signal(signum, frame):
        ctl.stop(restore_auto=True)
        raise SystemExit(0)

    signal.signal(signal.SIGINT, _handle_signal)
    signal.signal(signal.SIGTERM, _handle_signal)

    try:
        ctl.run()
        return 0
    except KeyboardInterrupt:
        ctl.stop(restore_auto=True)
        return 0
    except Exception as e:
        print(f"Fatal error: {e}", file=sys.stderr, flush=True)
        ctl.stop(restore_auto=True)
        return 1


if __name__ == "__main__":
    sys.exit(main())