diff --git a/docs/superpowers/specs/2026-07-04-mi50-summary-cap-fallback-design.md b/docs/superpowers/specs/2026-07-04-mi50-summary-cap-fallback-design.md index fc4eed4..d7bcedb 100644 --- a/docs/superpowers/specs/2026-07-04-mi50-summary-cap-fallback-design.md +++ b/docs/superpowers/specs/2026-07-04-mi50-summary-cap-fallback-design.md @@ -72,6 +72,18 @@ Fallback is per-`_summarize_text` call (i.e. per chunk), so the long-session chunk/merge path in `_summarize_transcript` is unaffected. The old `_RETRIES = 4` loop is replaced by this structure. +### 3. Degenerate-output guard (added 2026-07-04) + +A wedged local backend — observed live when the MI50 overheated to 99°C junction — +returns a single character repeated (`"?????"`) as a *successful* 200 response, +which neither the timeout nor the exception path catches. So each `_call()` +validates its output: `_looks_degenerate(text)` flags output (≥24 non-space chars) +whose most-common non-whitespace character exceeds 50% of the text, and raises +`DegenerateOutput` — which the retry/fallback loop treats exactly like any other +failure (retry the primary, then fall back to cloud). Real gists are diverse prose +(top char well under 20%), so the threshold won't false-positive; short outputs are +exempt. If cloud *also* returns junk, it raises and stops — no infinite loop. + ## Testing Unit (pytest, `tests/test_summary_fallback.py`), monkeypatching `llm.complete`: @@ -95,6 +107,9 @@ llama.cpp slot shows bounded `n_decoded ≈ 768`. ## Out of scope (YAGNI) +- The degenerate-output guard (§3) targets the *observed* failure — one char + repeated. It does not try to detect subtler degeneration (repeated phrases, + off-topic rambling); that's fuzzy and unmotivated until seen. - No change to `chat_call`/reflect/think or `config.summary_backend`. - No change to profile/era/narrative rebuild calls (separate, and not the loop culprit); can adopt the same `max_tokens` later if they show the same rambling. diff --git a/lyra/summary.py b/lyra/summary.py index 6057226..9e8735d 100644 --- a/lyra/summary.py +++ b/lyra/summary.py @@ -12,6 +12,7 @@ from __future__ import annotations import sys import threading import time +from collections import Counter from concurrent.futures import ThreadPoolExecutor, as_completed from lyra import config, llm, logbus, memory @@ -28,6 +29,25 @@ MI50_ATTEMPTS = 2 # MI50; 150s is headroom but bails a hung call fast so fallback isn't slow. SUMMARY_TIMEOUT = 150 +# Degenerate-output guard. A wedged local model (e.g. an overheated GPU) returns +# a single character repeated ("?????") as a *successful* 200, which no timeout or +# exception catches — so validate the text and treat junk as a failure. Real gists +# are diverse prose; flag output whose most-common non-space char dominates. Short +# outputs are exempt (nothing meaningful to judge). +_DEGENERATE_MIN_CHARS = 24 +_DEGENERATE_CHAR_RATIO = 0.5 + + +class DegenerateOutput(RuntimeError): + """A backend returned junk (e.g. one char repeated) as a successful response.""" + + +def _looks_degenerate(text: str) -> bool: + stripped = "".join(text.split()) + if len(stripped) < _DEGENERATE_MIN_CHARS: + return False + return max(Counter(stripped).values()) / len(stripped) > _DEGENERATE_CHAR_RATIO + # Re-summarize a session once it has accumulated this many new raw exchanges. SUMMARIZE_AFTER = 20 # Transcript budget per LLM call; longer sessions are chunked + merged. Cloud has @@ -72,8 +92,11 @@ def _summarize_text(text: str, backend: Backend) -> str: ] def _call(be: Backend) -> str: - return llm.complete(messages, backend=be, - max_tokens=SUMMARY_MAX_TOKENS, timeout=SUMMARY_TIMEOUT) + out = llm.complete(messages, backend=be, + max_tokens=SUMMARY_MAX_TOKENS, timeout=SUMMARY_TIMEOUT) + if _looks_degenerate(out): + raise DegenerateOutput(f"{be} returned degenerate output ({len(out)} chars)") + return out # Try the primary backend a bounded number of times (each call fast-fails via # SUMMARY_TIMEOUT), with a short backoff for a transient blip / restarting GPU. diff --git a/tests/test_summary_fallback.py b/tests/test_summary_fallback.py index df4ee0d..8c7f0e4 100644 --- a/tests/test_summary_fallback.py +++ b/tests/test_summary_fallback.py @@ -95,3 +95,48 @@ def test_happy_path_uses_primary_only(calls, monkeypatch): assert out == "mi50-gist" assert [c["backend"] for c in calls.recorded] == ["mi50"] # no retries, no fallback + + +# --- degenerate ("?" garbage) output guard: a wedged local model returns junk as +# a successful 200, so treat it as a failure and fall back to cloud. --- + +def test_looks_degenerate_flags_repeated_char(): + assert summary._looks_degenerate("?" * 60) is True + assert summary._looks_degenerate("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!") is True + + +def test_looks_degenerate_passes_real_prose(): + gist = ("Brian sat down at the Meadows 1/3 in seat 6 with two straddles active; " + "he tagged a seat-3 calling station and finished the session up 240.") + assert summary._looks_degenerate(gist) is False + + +def test_looks_degenerate_ignores_short_output(): + # Too short to judge — don't false-positive a terse-but-valid reply. + assert summary._looks_degenerate("ok") is False + + +def test_degenerate_mi50_output_falls_back_to_cloud(calls, monkeypatch): + _set_key(monkeypatch) + + def responder(backend): + if backend == "mi50": + return "?" * 200 # garbage-as-200, not an exception + return "a real cloud gist of the session, diverse and coherent." + calls.fake.responder = responder + + out = summary._summarize_text("transcript", "mi50") + + assert "cloud gist" in out + assert [c["backend"] for c in calls.recorded] == ["mi50", "mi50", "cloud"] + + +def test_degenerate_cloud_output_raises_no_infinite_loop(calls, monkeypatch): + _set_key(monkeypatch) + calls.fake.responder = lambda backend: "?" * 200 # every backend returns garbage + + with pytest.raises(Exception): + summary._summarize_text("t", "mi50") + + # mi50 x2, then one cloud fallback that's also garbage -> give up, no loop. + assert [c["backend"] for c in calls.recorded] == ["mi50", "mi50", "cloud"]