From abac42c344d187c88ea99c8d5bdc7674589e7a2d Mon Sep 17 00:00:00 2001 From: serversdown Date: Sat, 27 Jun 2026 09:26:25 +0000 Subject: [PATCH] fix: serial consolidation on GPU backends (was timing out the MI50) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit summarize_all fanned out 8 concurrent workers, but the MI50 llama.cpp server runs a single slot (--parallel 1). Firing 8 at once queued them, blew the client timeout ('summary retry … Request timed out'), and thrashed/cancelled the KV cache — wasted compute and heat. Concurrency is now backend-aware: 8 for cloud, 1 for local/MI50. Co-Authored-By: Claude Opus 4.8 (1M context) --- lyra/summary.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/lyra/summary.py b/lyra/summary.py index 3ffee1a..903072b 100644 --- a/lyra/summary.py +++ b/lyra/summary.py @@ -129,16 +129,20 @@ def maybe_summarize_async(session_id: str, backend: Backend | None = None) -> No def summarize_all( - backend: Backend | None = None, limit: int | None = None, workers: int = 8 + backend: Backend | None = None, limit: int | None = None, workers: int | None = None ) -> dict: """Summarize every session that needs it. Idempotent and resumable. - LLM summarization runs concurrently across `workers` threads (great for a - cloud backend). DB reads (loading transcripts) and writes (store_summary, - which also embeds) happen on the main thread, so the single SQLite - connection is never touched from multiple threads. + Concurrency is backend-aware: the cloud API parallelizes happily, but the + local/MI50 GPU servers run a single slot (llama.cpp --parallel 1) — firing N + requests at them just queues, blows the client timeout, and thrashes the KV + cache (wasted compute + heat). So GPU backends run serially unless overridden. + DB reads/writes (store_summary embeds) stay on the main thread, so the single + SQLite connection is never touched from multiple threads. """ backend = backend or config.load().summary_backend + if workers is None: + workers = 8 if backend == "cloud" else 1 # Main thread: collect the work (transcripts) for sessions needing a summary. todo: list[tuple[str, str, int]] = []