From 30185f3fd8b1eb49be1853f705ddfb00b5440550 Mon Sep 17 00:00:00 2001 From: serversdown Date: Tue, 16 Jun 2026 05:37:22 +0000 Subject: [PATCH] feat: MI50 as a Lyra backend (OpenAI-compatible local GPU) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The MI50 box (CT202) runs an OpenAI-compatible llama.cpp server on 10.0.0.44:8080. Wire it in as a third backend: - llm.complete gains backend="mi50" (OpenAI client pointed at MI50_BASE_URL) - config: MI50_BASE_URL (default http://10.0.0.44:8080/v1) + MI50_MODEL - chat.respond labels the model per backend; web _backend_for maps "mi50" - UI backend selector adds "MI50 — local GPU" Verified end-to-end: llm.complete(backend="mi50") returns from the live server. See homelab-inference memory for the box topology. Co-Authored-By: Claude Opus 4.8 (1M context) --- .env.example | 4 ++++ lyra/chat.py | 4 +++- lyra/config.py | 4 ++++ lyra/llm.py | 8 +++++++- lyra/web/server.py | 5 ++++- lyra/web/static/index.html | 5 +++++ 6 files changed, 27 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index ff584cc..82e07e9 100644 --- a/.env.example +++ b/.env.example @@ -2,6 +2,10 @@ LOCAL_BASE_URL=http://localhost:11434 LOCAL_MODEL=qwen2.5:7b-instruct +# MI50 backend — OpenAI-compatible llama.cpp server on the home-lab GPU box. +MI50_BASE_URL=http://10.0.0.44:8080/v1 +MI50_MODEL=local-gpu + # Cloud backend (OpenAI) — higher quality, costs money. OPENAI_API_KEY= CLOUD_MODEL=gpt-4o-mini diff --git a/lyra/chat.py b/lyra/chat.py index d4da349..0211fe2 100644 --- a/lyra/chat.py +++ b/lyra/chat.py @@ -81,7 +81,9 @@ def build_messages(session_id: str, user_msg: str) -> list[Message]: def respond(session_id: str, user_msg: str, backend: Backend = "cloud") -> str: """Produce Lyra's reply to a single user message and persist the exchange.""" cfg = config.load() - model = cfg.local_model if backend == "local" else cfg.cloud_model + model = {"local": cfg.local_model, "cloud": cfg.cloud_model, "mi50": cfg.mi50_model}.get( + backend, backend + ) logbus.log( "info", "chat request", session=session_id, backend=backend, model=model, embed=cfg.embed_backend, diff --git a/lyra/config.py b/lyra/config.py index 3b9b633..8405ee3 100644 --- a/lyra/config.py +++ b/lyra/config.py @@ -14,6 +14,8 @@ load_dotenv() class Config: local_base_url: str local_model: str + mi50_base_url: str # OpenAI-compatible llama.cpp server on the MI50 box + mi50_model: str openai_api_key: str cloud_model: str embed_backend: str # "cloud" (OpenAI) or "local" (Ollama) @@ -27,6 +29,8 @@ def load() -> Config: return Config( local_base_url=os.getenv("LOCAL_BASE_URL", "http://localhost:11434"), local_model=os.getenv("LOCAL_MODEL", "qwen2.5:7b-instruct"), + mi50_base_url=os.getenv("MI50_BASE_URL", "http://10.0.0.44:8080/v1"), + mi50_model=os.getenv("MI50_MODEL", "local-gpu"), openai_api_key=os.getenv("OPENAI_API_KEY", ""), cloud_model=os.getenv("CLOUD_MODEL", "gpt-4o-mini"), embed_backend=os.getenv("EMBED_BACKEND", "cloud").lower(), diff --git a/lyra/llm.py b/lyra/llm.py index 71e3fd0..471c6f9 100644 --- a/lyra/llm.py +++ b/lyra/llm.py @@ -14,7 +14,7 @@ class Message(TypedDict): content: str -Backend = Literal["local", "cloud"] +Backend = Literal["local", "cloud", "mi50"] def complete(messages: list[Message], backend: Backend = "local") -> str: @@ -26,6 +26,12 @@ def complete(messages: list[Message], backend: Backend = "local") -> str: resp = client.chat.completions.create(model=cfg.cloud_model, messages=messages) return resp.choices[0].message.content or "" + if backend == "mi50": + # MI50 box runs an OpenAI-compatible llama.cpp server; key is unused. + client = OpenAI(api_key="not-needed", base_url=cfg.mi50_base_url) + resp = client.chat.completions.create(model=cfg.mi50_model, messages=messages) + return resp.choices[0].message.content or "" + resp = httpx.post( f"{cfg.local_base_url}/api/chat", json={"model": cfg.local_model, "messages": messages, "stream": False}, diff --git a/lyra/web/server.py b/lyra/web/server.py index 94a9d13..b684401 100644 --- a/lyra/web/server.py +++ b/lyra/web/server.py @@ -32,7 +32,10 @@ _CLOUD = {"OPENAI", "cloud", "custom"} def _backend_for(label: str | None) -> Backend: - if label and label.upper() in {"PRIMARY", "SECONDARY", "FALLBACK", "LOCAL"}: + key = (label or "").lower() + if key == "mi50": + return "mi50" + if key in {"local", "primary", "secondary", "fallback"}: return "local" return "cloud" diff --git a/lyra/web/static/index.html b/lyra/web/static/index.html index adb30a6..411c737 100644 --- a/lyra/web/static/index.html +++ b/lyra/web/static/index.html @@ -123,6 +123,11 @@ Local — Ollama Free, private, runs on your home lab (LOCAL_MODEL) +