fix: cap MI50 summary length + fast-fail cloud fallback
The dream cycle's summarize_all ran uncapped against the MI50: no max_tokens and no timeout, so the OpenAI SDK's 600s x2-retry default meant ~30 min per call. Combined with summary.py's own retry loop, one unsummarizable session pegged the GPU for hours (observed 2026-07-04: stuck since 23:02, nothing saved since 00:56, 7-8k-token runaway generations, all 4 llama.cpp slots busy). Not context overflow (0 shifts/truncations) - purely unbounded length on a slow backend timing out and retrying. - llm.complete(): add optional max_tokens (caps generation; num_predict for Ollama) and timeout (bounds the request and sets max_retries=0 so the caller owns retry policy). Both default None -> unchanged for every existing caller. - summary.py: cap gists at 768 tokens, 150s/call fast-fail, 2 MI50 attempts then one cloud fallback (when primary isn't already cloud and a key exists). Known limitation (scoped out per decision): the fallback triggers on timeouts/exceptions, not on a degraded backend returning garbage as a 200. Tests: fallback fires after 2 MI50 failures; no fallback when primary is cloud or no key; cap+timeout threaded into every complete() call; llm bounds tests. 172 pass, ruff clean. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_015yrEb5qpPGv2FjyxrB7LLk
This commit is contained in:
@@ -0,0 +1,97 @@
|
||||
"""Summary consolidation: MI50 length cap, fast-fail, and cloud fallback.
|
||||
|
||||
Everything is stubbed — no real backend is touched. These drive the behavior of
|
||||
`summary._summarize_text`: try the primary backend a bounded number of times with
|
||||
a capped generation length, and fall back to cloud if the primary keeps failing.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import types
|
||||
|
||||
import pytest
|
||||
|
||||
from lyra import summary
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def calls(monkeypatch):
|
||||
"""Capture every llm.complete call; per-test behavior via `fake.responder`."""
|
||||
recorded: list[dict] = []
|
||||
|
||||
def fake_complete(messages, backend="local", model=None,
|
||||
max_tokens=None, timeout=None):
|
||||
recorded.append({"backend": backend, "max_tokens": max_tokens, "timeout": timeout})
|
||||
return fake_complete.responder(backend)
|
||||
|
||||
fake_complete.responder = lambda backend: "gist"
|
||||
monkeypatch.setattr(summary.llm, "complete", fake_complete)
|
||||
monkeypatch.setattr(summary.time, "sleep", lambda *_: None) # instant backoff
|
||||
return types.SimpleNamespace(recorded=recorded, fake=fake_complete)
|
||||
|
||||
|
||||
def _set_key(monkeypatch, key="sk-test"):
|
||||
monkeypatch.setattr(summary.config, "load",
|
||||
lambda: types.SimpleNamespace(openai_api_key=key))
|
||||
|
||||
|
||||
def test_falls_back_to_cloud_after_mi50_attempts(calls, monkeypatch):
|
||||
_set_key(monkeypatch)
|
||||
|
||||
def responder(backend):
|
||||
if backend == "mi50":
|
||||
raise RuntimeError("Request timed out.")
|
||||
return "cloud-gist"
|
||||
calls.fake.responder = responder
|
||||
|
||||
out = summary._summarize_text("transcript", "mi50")
|
||||
|
||||
assert out == "cloud-gist"
|
||||
assert [c["backend"] for c in calls.recorded] == ["mi50", "mi50", "cloud"]
|
||||
|
||||
|
||||
def test_no_fallback_when_backend_is_cloud(calls, monkeypatch):
|
||||
_set_key(monkeypatch)
|
||||
calls.fake.responder = lambda backend: (_ for _ in ()).throw(RuntimeError("boom"))
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
summary._summarize_text("t", "cloud")
|
||||
|
||||
# Cloud is already the primary: retry it, but never a redundant fallback.
|
||||
assert [c["backend"] for c in calls.recorded] == ["cloud", "cloud"]
|
||||
|
||||
|
||||
def test_no_fallback_without_openai_key(calls, monkeypatch):
|
||||
_set_key(monkeypatch, key="")
|
||||
calls.fake.responder = lambda backend: (_ for _ in ()).throw(RuntimeError("mi50 down"))
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
summary._summarize_text("t", "mi50")
|
||||
|
||||
assert [c["backend"] for c in calls.recorded] == ["mi50", "mi50"]
|
||||
|
||||
|
||||
def test_caps_length_and_timeout_on_every_call(calls, monkeypatch):
|
||||
_set_key(monkeypatch)
|
||||
|
||||
def responder(backend):
|
||||
if backend == "mi50":
|
||||
raise RuntimeError("nope")
|
||||
return "cloud-gist"
|
||||
calls.fake.responder = responder
|
||||
|
||||
summary._summarize_text("t", "mi50")
|
||||
|
||||
assert calls.recorded
|
||||
for c in calls.recorded:
|
||||
assert c["max_tokens"] == summary.SUMMARY_MAX_TOKENS
|
||||
assert c["timeout"] == summary.SUMMARY_TIMEOUT
|
||||
|
||||
|
||||
def test_happy_path_uses_primary_only(calls, monkeypatch):
|
||||
_set_key(monkeypatch)
|
||||
calls.fake.responder = lambda backend: "mi50-gist"
|
||||
|
||||
out = summary._summarize_text("t", "mi50")
|
||||
|
||||
assert out == "mi50-gist"
|
||||
assert [c["backend"] for c in calls.recorded] == ["mi50"] # no retries, no fallback
|
||||
Reference in New Issue
Block a user