chore: update gitignore for export data

Import the parser's {title, messages} JSON into Lyra's memory so past
conversations seed recall (and, later, the era-rollup tier).

- lyra/ingest.py: one conversation -> one session, text messages -> exchanges;
  skips non-text (image asset) messages and non user/assistant roles; embeddings
  batched; idempotent by filename-derived session id; `lyra-import <dir>` CLI
- memory.add_exchanges_bulk: batched insert of pre-embedded rows

Format has no timestamps yet, so imports are stamped at import time; a future
dated export will let era memory group by real calendar time.

Verified on the 68-file lyra dev set: 7519 exchanges, idempotent re-run, recall
returns relevant history.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

.gitignore

@@ -35,3 +35,4 @@ data/
 
 #lyra Stuff
 /core/relay/sessions/
+/chat-gpt-export/

lyra/chat.py

@@ -30,6 +30,11 @@ def _detail_note(exchanges: list[memory.Exchange]) -> Message:
     return {"role": "system", "content": body}
 
 
+def _render(messages: list[Message]) -> str:
+    """Human-readable dump of the exact prompt, for the live-log inspector."""
+    return "\n\n".join(f"[{m['role']}]\n{m['content']}" for m in messages)
+
+
 def build_messages(session_id: str, user_msg: str) -> list[Message]:
     """Assemble the full, tiered message list for one turn."""
     messages: list[Message] = [{"role": "system", "content": persona.system_prompt()}]
@@ -51,16 +56,17 @@ def build_messages(session_id: str, user_msg: str) -> list[Message]:
     if recalled:
         messages.append(_detail_note(recalled))
 
-    logbus.log(
-        "debug", "context built",
-        recent=len(recent), summaries=len(summaries), details=len(recalled),
-    )
-
     # Tier 3: current session, full fidelity.
     for ex in recent:
         messages.append({"role": ex.role, "content": ex.content})
 
     messages.append({"role": "user", "content": user_msg})
+
+    logbus.log(
+        "debug", "context built",
+        recent=len(recent), summaries=len(summaries), details=len(recalled),
+        chars=sum(len(m["content"]) for m in messages), detail=_render(messages),
+    )
     return messages
 
 

lyra/ingest.py

@@ -0,0 +1,93 @@
+"""Import parsed ChatGPT chat logs into Lyra's memory.
+
+Consumes the parser's `{"title": ..., "messages": [{"role", "content"}]}` format
+(one JSON file per conversation). Each conversation becomes a Lyra session; each
+text message becomes an exchange. Embeddings are batched. Import is idempotent —
+a conversation already present (by session id) is skipped.
+
+Timestamps: this format carries no dates, so imported exchanges are stamped with
+`created_at` (default: now). A future timestamped export will let era memory group
+by real calendar time; pass real per-message dates then.
+"""
+from __future__ import annotations
+
+import json
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+from lyra import llm, logbus, memory
+
+EMBED_BATCH = 64
+EMBED_CHAR_CAP = 6000  # cap embed input size; full content is still stored
+
+
+def _session_id(path: Path) -> str:
+    """Stable id derived from the filename, so re-imports don't duplicate."""
+    return "import-" + path.stem
+
+
+def _clean_messages(messages: list[dict]) -> list[tuple[str, str]]:
+    out: list[tuple[str, str]] = []
+    for m in messages:
+        role = m.get("role")
+        if role not in ("user", "assistant"):
+            continue
+        content = (m.get("content") or "").strip()
+        if not content or content.startswith('{"content_type"'):  # skip empty / image assets
+            continue
+        out.append((role, content))
+    return out
+
+
+def import_file(path: Path, created_at: str) -> int:
+    """Import one conversation file. Returns exchanges added (0 if skipped/empty)."""
+    data = json.loads(path.read_text(encoding="utf-8"))
+    session_id = _session_id(path)
+    if memory.history(session_id):  # already imported
+        return 0
+
+    msgs = _clean_messages(data.get("messages", []))
+    if not msgs:
+        return 0
+
+    memory.ensure_session(session_id, name=data.get("title") or path.stem)
+
+    rows: list[tuple[str, str, list[float], str]] = []
+    for i in range(0, len(msgs), EMBED_BATCH):
+        batch = msgs[i : i + EMBED_BATCH]
+        embeddings = llm.embed([content[:EMBED_CHAR_CAP] for _, content in batch])
+        for (role, content), emb in zip(batch, embeddings):
+            rows.append((role, content, emb, created_at))
+
+    return memory.add_exchanges_bulk(session_id, rows)
+
+
+def import_dir(dirpath: str | Path, created_at: str | None = None) -> dict:
+    """Import every *.json under dirpath (recursively). Returns a small report."""
+    created_at = created_at or datetime.now(timezone.utc).isoformat()
+    files = sorted(Path(dirpath).rglob("*.json"))
+    sessions, exchanges = 0, 0
+    for path in files:
+        added = import_file(path, created_at)
+        if added:
+            sessions += 1
+            exchanges += added
+    logbus.log(
+        "info", "import complete", dir=str(dirpath),
+        files=len(files), sessions=sessions, exchanges=exchanges,
+    )
+    return {"files": len(files), "sessions_imported": sessions, "exchanges": exchanges}
+
+
+def main() -> int:
+    if len(sys.argv) < 2:
+        print("usage: lyra-import <dir-of-chat-json>", file=sys.stderr)
+        return 2
+    report = import_dir(sys.argv[1])
+    print(report)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

lyra/memory.py

@@ -108,6 +108,22 @@ def remember(session_id: str, role: str, content: str) -> int:
     return int(cur.lastrowid)
 
 
+def add_exchanges_bulk(session_id: str, rows: list[tuple[str, str, list[float], str]]) -> int:
+    """Insert many pre-embedded exchanges at once.
+
+    Each row is (role, content, embedding, created_at). Used by the importer to
+    avoid one INSERT (and one embed round-trip) per message. Returns row count.
+    """
+    conn = _connection()
+    with conn:
+        conn.executemany(
+            "INSERT INTO exchanges (session_id, role, content, embedding, created_at) "
+            "VALUES (?, ?, ?, ?, ?)",
+            [(session_id, role, content, _to_blob(emb), ca) for role, content, emb, ca in rows],
+        )
+    return len(rows)
+
+
 def recent(session_id: str, n: int = 10) -> list[Exchange]:
     """Last `n` exchanges from a session, oldest first."""
     conn = _connection()

lyra/web/static/index.html

@@ -734,7 +734,10 @@
 
         const level = event.level || 'info';
         const time = new Date((event.ts || 0) * 1000).toLocaleTimeString();
-        const fields = event.fields || {};
+        const fields = Object.assign({}, event.fields || {});
+        // `detail` is rendered as an expandable block, not an inline field.
+        const detail = fields.detail;
+        delete fields.detail;
         const fieldStr = Object.keys(fields).length
           ? Object.entries(fields).map(([k, v]) => `${k}=${v}`).join('  ')
           : '';
@@ -746,6 +749,7 @@
           <span class="log-level log-level-${level}">${escapeHtml(level)}</span>
           <span class="log-msg">${escapeHtml(event.msg || '')}</span>
           ${fieldStr ? `<span class="log-fields">${escapeHtml(fieldStr)}</span>` : ''}
+          ${detail ? `<details class="log-detail"><summary>view full prompt</summary><pre>${escapeHtml(detail)}</pre></details>` : ''}
         `;
 
         thinkingContent.appendChild(eventDiv);

lyra/web/static/style.css

@@ -941,3 +941,25 @@ select:hover {
 .log-error .log-level, .log-error .log-msg { color: #fca5a5; }
 .log-system  { border-left-color: #00ff66; }
 .log-system .log-level { color: #00ff66; }
+
+.log-detail { width: 100%; margin-top: 4px; }
+.log-detail summary {
+  cursor: pointer;
+  color: var(--accent);
+  font-size: 0.72rem;
+  user-select: none;
+}
+.log-detail pre {
+  margin: 6px 0 0;
+  padding: 8px;
+  max-height: 340px;
+  overflow: auto;
+  background: rgba(0,0,0,0.25);
+  border-left: 2px solid var(--accent);
+  border-radius: 4px;
+  font-size: 0.72rem;
+  line-height: 1.4;
+  white-space: pre-wrap;
+  word-break: break-word;
+  color: var(--text);
+}

pyproject.toml

@@ -16,6 +16,7 @@ dependencies = [
 [project.scripts]
 lyra = "lyra.__main__:main"
 lyra-web = "lyra.web.server:serve"
+lyra-import = "lyra.ingest:main"
 
 [dependency-groups]
 dev = [