serversdown
173fd18688
Nail down which backend each LLM path uses, and make the dream cycle resilient to a backend being down (the MI50 outage left profile/era/narrative — pinned to mi50 with no fallback — aborting every dream cycle). - Consolidation (summaries + profile/era/narrative) -> cloud via SUMMARY_BACKEND= cloud (.env, not committed). Matches the documented lesson that the MI50 is too slow/hot for bulk consolidation; nothing background touches the card now. - llm.complete_with_fallback(): try the primary backend, fall back to cloud on error (re-raise if already cloud / no key). Wired into reflect + think so the introspection voice (3090/dolphin) survives the gaming PC being powered off. - dream coherence stage is now fault-isolated: a rebuild failure logs + continues instead of sinking the whole pass (reflection still runs). - .env: removed stale INTROSPECTION_BACKEND=mi50 (live routing is the web-switchable introspection_mode DB setting = dolphin/3090; the var only fed a dead fallback). Verified: forced cycle runs consolidation on cloud, introspection on the 3090, completes with zero MI50 calls. 206 pass, ruff clean. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_015yrEb5qpPGv2FjyxrB7LLk
111 lines
4.2 KiB
Python
111 lines
4.2 KiB
Python
"""llm.complete: `max_tokens` and `timeout` are threaded into the backend call.
|
|
|
|
The OpenAI client is faked so nothing hits a network. We assert the generation
|
|
cap reaches the create() call and the fast-fail timeout reaches the client (with
|
|
max_retries=0 so summary.py owns the retry policy, not the SDK).
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import types
|
|
|
|
import pytest
|
|
|
|
from lyra import llm
|
|
|
|
|
|
@pytest.fixture
|
|
def fake_openai(monkeypatch):
|
|
recorded: dict = {}
|
|
|
|
class FakeCompletions:
|
|
def create(self, **kwargs):
|
|
recorded["create"] = kwargs
|
|
msg = types.SimpleNamespace(content="ok")
|
|
return types.SimpleNamespace(choices=[types.SimpleNamespace(message=msg)])
|
|
|
|
class FakeClient:
|
|
def __init__(self, **kwargs):
|
|
recorded["client"] = kwargs
|
|
self.chat = types.SimpleNamespace(completions=FakeCompletions())
|
|
|
|
monkeypatch.setattr(llm, "OpenAI", FakeClient)
|
|
monkeypatch.setattr(llm, "load", lambda: types.SimpleNamespace(
|
|
mi50_base_url="http://mi50/v1", mi50_model="local-gpu",
|
|
cloud_model="gpt-4o-mini", openai_api_key="sk-test", local_model="l",
|
|
))
|
|
return recorded
|
|
|
|
|
|
def test_mi50_threads_max_tokens_and_timeout(fake_openai):
|
|
out = llm.complete([{"role": "user", "content": "hi"}],
|
|
backend="mi50", max_tokens=768, timeout=150)
|
|
|
|
assert out == "ok"
|
|
assert fake_openai["create"]["max_tokens"] == 768
|
|
assert fake_openai["client"]["timeout"] == 150
|
|
assert fake_openai["client"]["max_retries"] == 0
|
|
|
|
|
|
def test_cloud_threads_max_tokens_and_timeout(fake_openai):
|
|
llm.complete([{"role": "user", "content": "hi"}],
|
|
backend="cloud", max_tokens=768, timeout=150)
|
|
|
|
assert fake_openai["create"]["max_tokens"] == 768
|
|
assert fake_openai["client"]["timeout"] == 150
|
|
assert fake_openai["client"]["max_retries"] == 0
|
|
|
|
|
|
def test_fallback_uses_primary_when_it_succeeds(monkeypatch):
|
|
seen = []
|
|
monkeypatch.setattr(llm, "complete",
|
|
lambda messages, backend="local", model=None, **k:
|
|
seen.append(backend) or f"{backend}-ok")
|
|
out = llm.complete_with_fallback([{"role": "user", "content": "x"}],
|
|
backend="local", model="dolphin3:8b")
|
|
assert out == "local-ok"
|
|
assert seen == ["local"] # no fallback when the primary works
|
|
|
|
|
|
def test_fallback_to_cloud_when_primary_errors(monkeypatch):
|
|
monkeypatch.setattr(llm, "load", lambda: types.SimpleNamespace(openai_api_key="sk"))
|
|
seen = []
|
|
|
|
def fake(messages, backend="local", model=None, **k):
|
|
seen.append(backend)
|
|
if backend == "local":
|
|
raise RuntimeError("3090 is powered off")
|
|
return "cloud-ok"
|
|
monkeypatch.setattr(llm, "complete", fake)
|
|
|
|
out = llm.complete_with_fallback([{"role": "user", "content": "x"}],
|
|
backend="local", model="dolphin3:8b")
|
|
assert out == "cloud-ok"
|
|
assert seen == ["local", "cloud"]
|
|
|
|
|
|
def test_fallback_reraises_when_primary_is_already_cloud(monkeypatch):
|
|
monkeypatch.setattr(llm, "load", lambda: types.SimpleNamespace(openai_api_key="sk"))
|
|
monkeypatch.setattr(llm, "complete",
|
|
lambda *a, **k: (_ for _ in ()).throw(RuntimeError("boom")))
|
|
with pytest.raises(RuntimeError):
|
|
llm.complete_with_fallback([{"role": "user", "content": "x"}], backend="cloud")
|
|
|
|
|
|
def test_fallback_reraises_without_openai_key(monkeypatch):
|
|
monkeypatch.setattr(llm, "load", lambda: types.SimpleNamespace(openai_api_key=""))
|
|
monkeypatch.setattr(llm, "complete",
|
|
lambda *a, **k: (_ for _ in ()).throw(RuntimeError("down")))
|
|
with pytest.raises(RuntimeError):
|
|
llm.complete_with_fallback([{"role": "user", "content": "x"}], backend="local")
|
|
|
|
|
|
def test_default_bounds_calls_even_without_explicit_timeout(fake_openai):
|
|
# No cap / timeout passed -> still bounded: 300s default + no SDK retries, so
|
|
# no call can silently inherit the SDK's 600s x2 (~30 min). No length cap
|
|
# unless asked, though.
|
|
llm.complete([{"role": "user", "content": "hi"}], backend="mi50")
|
|
|
|
assert "max_tokens" not in fake_openai["create"]
|
|
assert fake_openai["client"]["timeout"] == 300
|
|
assert fake_openai["client"]["max_retries"] == 0
|