diff --git a/.env.example b/.env.example
index ff584cc..82e07e9 100644
--- a/.env.example
+++ b/.env.example
@@ -2,6 +2,10 @@
LOCAL_BASE_URL=http://localhost:11434
LOCAL_MODEL=qwen2.5:7b-instruct
+# MI50 backend — OpenAI-compatible llama.cpp server on the home-lab GPU box.
+MI50_BASE_URL=http://10.0.0.44:8080/v1
+MI50_MODEL=local-gpu
+
# Cloud backend (OpenAI) — higher quality, costs money.
OPENAI_API_KEY=
CLOUD_MODEL=gpt-4o-mini
diff --git a/lyra/chat.py b/lyra/chat.py
index d4da349..0211fe2 100644
--- a/lyra/chat.py
+++ b/lyra/chat.py
@@ -81,7 +81,9 @@ def build_messages(session_id: str, user_msg: str) -> list[Message]:
def respond(session_id: str, user_msg: str, backend: Backend = "cloud") -> str:
"""Produce Lyra's reply to a single user message and persist the exchange."""
cfg = config.load()
- model = cfg.local_model if backend == "local" else cfg.cloud_model
+ model = {"local": cfg.local_model, "cloud": cfg.cloud_model, "mi50": cfg.mi50_model}.get(
+ backend, backend
+ )
logbus.log(
"info", "chat request", session=session_id, backend=backend,
model=model, embed=cfg.embed_backend,
diff --git a/lyra/config.py b/lyra/config.py
index 3b9b633..8405ee3 100644
--- a/lyra/config.py
+++ b/lyra/config.py
@@ -14,6 +14,8 @@ load_dotenv()
class Config:
local_base_url: str
local_model: str
+ mi50_base_url: str # OpenAI-compatible llama.cpp server on the MI50 box
+ mi50_model: str
openai_api_key: str
cloud_model: str
embed_backend: str # "cloud" (OpenAI) or "local" (Ollama)
@@ -27,6 +29,8 @@ def load() -> Config:
return Config(
local_base_url=os.getenv("LOCAL_BASE_URL", "http://localhost:11434"),
local_model=os.getenv("LOCAL_MODEL", "qwen2.5:7b-instruct"),
+ mi50_base_url=os.getenv("MI50_BASE_URL", "http://10.0.0.44:8080/v1"),
+ mi50_model=os.getenv("MI50_MODEL", "local-gpu"),
openai_api_key=os.getenv("OPENAI_API_KEY", ""),
cloud_model=os.getenv("CLOUD_MODEL", "gpt-4o-mini"),
embed_backend=os.getenv("EMBED_BACKEND", "cloud").lower(),
diff --git a/lyra/llm.py b/lyra/llm.py
index 71e3fd0..471c6f9 100644
--- a/lyra/llm.py
+++ b/lyra/llm.py
@@ -14,7 +14,7 @@ class Message(TypedDict):
content: str
-Backend = Literal["local", "cloud"]
+Backend = Literal["local", "cloud", "mi50"]
def complete(messages: list[Message], backend: Backend = "local") -> str:
@@ -26,6 +26,12 @@ def complete(messages: list[Message], backend: Backend = "local") -> str:
resp = client.chat.completions.create(model=cfg.cloud_model, messages=messages)
return resp.choices[0].message.content or ""
+ if backend == "mi50":
+ # MI50 box runs an OpenAI-compatible llama.cpp server; key is unused.
+ client = OpenAI(api_key="not-needed", base_url=cfg.mi50_base_url)
+ resp = client.chat.completions.create(model=cfg.mi50_model, messages=messages)
+ return resp.choices[0].message.content or ""
+
resp = httpx.post(
f"{cfg.local_base_url}/api/chat",
json={"model": cfg.local_model, "messages": messages, "stream": False},
diff --git a/lyra/web/server.py b/lyra/web/server.py
index 94a9d13..b684401 100644
--- a/lyra/web/server.py
+++ b/lyra/web/server.py
@@ -32,7 +32,10 @@ _CLOUD = {"OPENAI", "cloud", "custom"}
def _backend_for(label: str | None) -> Backend:
- if label and label.upper() in {"PRIMARY", "SECONDARY", "FALLBACK", "LOCAL"}:
+ key = (label or "").lower()
+ if key == "mi50":
+ return "mi50"
+ if key in {"local", "primary", "secondary", "fallback"}:
return "local"
return "cloud"
diff --git a/lyra/web/static/index.html b/lyra/web/static/index.html
index adb30a6..411c737 100644
--- a/lyra/web/static/index.html
+++ b/lyra/web/static/index.html
@@ -123,6 +123,11 @@
Local — Ollama
Free, private, runs on your home lab (LOCAL_MODEL)
+