From edb4698bfb0bd6e28eb152d7a0cf4a81e59fb3ed Mon Sep 17 00:00:00 2001 From: Brian Harrison Date: Tue, 14 Apr 2026 02:15:33 -0400 Subject: [PATCH] feat: add waveform download and storage. --- bridges/ach_server.py | 59 +++++++++++++++++++++++++++++++++++++++- sfm/database.py | 46 ++++++++++++++++++++++++++++--- sfm/server.py | 35 ++++++++++++++++++++++++ sfm/sfm_webapp.html | 15 ++++++++++ sfm/waveform_viewer.html | 47 ++++++++++++++++++++++++++++++-- 5 files changed, 195 insertions(+), 7 deletions(-) diff --git a/bridges/ach_server.py b/bridges/ach_server.py index afafe4c..1dbfc85 100644 --- a/bridges/ach_server.py +++ b/bridges/ach_server.py @@ -445,8 +445,19 @@ class AchSession: # ── Persist to SQLite DB ───────────────────────────────────── _session_start = datetime.datetime.now() try: + # Build waveform blobs for events that have full raw_samples + _waveform_blobs = {} + for _ev in new_events: + if _ev._waveform_key and _ev.raw_samples: + _blob = _build_waveform_blob(_ev) + if _blob: + _waveform_blobs[_ev._waveform_key.hex()] = _blob + if _waveform_blobs: + log.info(" [DB] waveform blobs prepared: %d", len(_waveform_blobs)) + _ev_ins, _ev_skip = self.db.insert_events( - new_events, serial=serial or self.peer, session_id=None + new_events, serial=serial or self.peer, session_id=None, + waveform_blobs=_waveform_blobs, ) _ml_ins, _ml_skip = self.db.insert_monitor_log( new_monitor_entries, session_id=None @@ -599,6 +610,52 @@ def _event_to_dict(e: Event) -> dict: } +def _build_waveform_blob(e: Event) -> Optional[str]: + """ + Serialise a downloaded event's full waveform data as a JSON string for + storage in the DB waveform_blob column. + + Returns the same shape as GET /device/event/{index}/waveform so the + waveform viewer can consume either source without modification. + Returns None if the event has no raw_samples (e.g. metadata-only download). + """ + raw = e.raw_samples or {} + if not raw: + return None + + pv = e.peak_values + peak_values = None + if pv: + peak_values = { + "tran": pv.tran, + "vert": pv.vert, + "long": pv.long, + "micl_psi": pv.micl, + "peak_vector_sum": pv.peak_vector_sum, + } + + ts = e.timestamp + timestamp_str = ( + f"{ts.year:04d}-{ts.month:02d}-{ts.day:02d}T" + f"{ts.hour:02d}:{ts.minute:02d}:{ts.second:02d}" + if ts else None + ) + + blob = { + "index": e.index, + "record_type": e.record_type, + "timestamp": timestamp_str, + "total_samples": e.total_samples, + "pretrig_samples": e.pretrig_samples, + "rectime_seconds": e.rectime_seconds, + "samples_decoded": len(raw.get("Tran", [])), + "sample_rate": e.sample_rate, + "peak_values": peak_values, + "channels": raw, + } + return json.dumps(blob) + + def _monitor_log_entry_to_dict(e: MonitorLogEntry) -> dict: return { "key": e.key, diff --git a/sfm/database.py b/sfm/database.py index 110ba7a..01758a7 100644 --- a/sfm/database.py +++ b/sfm/database.py @@ -81,6 +81,7 @@ CREATE TABLE IF NOT EXISTS events ( sample_rate INTEGER, record_type TEXT, -- "single_shot" | "continuous" false_trigger INTEGER NOT NULL DEFAULT 0, -- 0=no, 1=yes (manual flag) + waveform_blob TEXT, -- JSON waveform response (channels + metadata) created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), UNIQUE(serial, timestamp) ); @@ -216,6 +217,17 @@ class SeismoDb: """) log.info("_migrate: monitor_log table rebuilt OK") + # Migration 3: add waveform_blob column to events (nullable TEXT). + # ALter TABLE ADD COLUMN is safe in SQLite for nullable columns — no rebuild needed. + col_names = { + row[1] + for row in conn.execute("PRAGMA table_info(events)").fetchall() + } + if "waveform_blob" not in col_names: + log.info("_migrate: adding waveform_blob column to events") + conn.execute("ALTER TABLE events ADD COLUMN waveform_blob TEXT") + log.info("_migrate: waveform_blob column added OK") + @staticmethod def _iso(dt: Optional[datetime.datetime]) -> Optional[str]: return dt.isoformat() if dt is not None else None @@ -282,12 +294,19 @@ class SeismoDb: *, serial: str, session_id: Optional[str] = None, + waveform_blobs: Optional[dict[str, str]] = None, ) -> tuple[int, int]: """ Insert triggered events. Silently skips duplicates (serial+timestamp). Returns (inserted, skipped). + + waveform_blobs: optional mapping of waveform_key (hex str) → JSON string + containing the full waveform response (channels + metadata). When provided, + the blob is stored alongside the event row and is retrievable via + GET /db/events/{id}/waveform. """ inserted = skipped = 0 + blobs = waveform_blobs or {} with self._connect() as conn: for ev in events: key = ev._waveform_key.hex() if ev._waveform_key else None @@ -305,8 +324,9 @@ class SeismoDb: except Exception: ts = str(ev.timestamp) - pv = ev.peak_values - pi = ev.project_info + pv = ev.peak_values + pi = ev.project_info + blob = blobs.get(key) try: conn.execute( @@ -315,8 +335,8 @@ class SeismoDb: (id, serial, waveform_key, session_id, timestamp, tran_ppv, vert_ppv, long_ppv, peak_vector_sum, mic_ppv, project, client, operator, sensor_location, - sample_rate, record_type) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + sample_rate, record_type, waveform_blob) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( self._new_id(), serial, key, session_id, ts, @@ -331,6 +351,7 @@ class SeismoDb: pi.sensor_location if pi else None, ev.sample_rate, ev.record_type, + blob, ), ) inserted += 1 @@ -387,6 +408,23 @@ class SeismoDb: ) return cur.rowcount > 0 + def get_event_waveform(self, event_id: str) -> tuple[bool, Optional[str]]: + """ + Return (found, waveform_blob) for a given event UUID. + + found=False means the event row doesn't exist. + found=True, blob=None means the event exists but has no stored waveform + (e.g. downloaded before waveform storage was implemented). + found=True, blob= means the full waveform JSON is available. + """ + with self._connect() as conn: + row = conn.execute( + "SELECT waveform_blob FROM events WHERE id = ?", (event_id,) + ).fetchone() + if row is None: + return False, None + return True, row["waveform_blob"] + # ── Monitor log ─────────────────────────────────────────────────────────── def insert_monitor_log( diff --git a/sfm/server.py b/sfm/server.py index 0ed47ad..32ca875 100644 --- a/sfm/server.py +++ b/sfm/server.py @@ -1006,6 +1006,41 @@ def db_set_false_trigger( return {"status": "ok", "event_id": event_id, "false_trigger": value} +@app.get("/db/events/{event_id}/waveform") +def db_event_waveform(event_id: str) -> dict: + """ + Return the stored waveform blob for a DB event. + + The response shape is identical to GET /device/event/{index}/waveform so the + waveform viewer can consume either source without modification: + - total_samples, pretrig_samples, rectime_seconds, samples_decoded + - sample_rate + - peak_values (tran, vert, long, micl_psi, peak_vector_sum) + - channels ({"Tran": [...], "Vert": [...], "Long": [...], "Mic": [...]}) + + Returns 404 if the event doesn't exist, 422 if the event exists but has no + stored waveform (downloaded before waveform storage was implemented). + """ + import json as _json + db = _get_db() + found, blob_str = db.get_event_waveform(event_id) + if not found: + raise HTTPException(status_code=404, detail=f"Event {event_id} not found") + if blob_str is None: + raise HTTPException( + status_code=422, + detail=( + f"Event {event_id} has no stored waveform. " + "Waveform storage requires ACH server v0.11+. " + "Re-download the event from the device to backfill." + ), + ) + try: + return _json.loads(blob_str) + except Exception as exc: + raise HTTPException(status_code=500, detail=f"Waveform blob corrupt: {exc}") from exc + + @app.get("/db/monitor_log") def db_monitor_log( serial: Optional[str] = Query(None, description="Filter by unit serial"), diff --git a/sfm/sfm_webapp.html b/sfm/sfm_webapp.html index d59ea41..1e83c75 100644 --- a/sfm/sfm_webapp.html +++ b/sfm/sfm_webapp.html @@ -548,6 +548,18 @@ .ft-toggle-btn:hover { border-color: var(--red); color: var(--red); } .ft-toggle-btn.flagged { border-color: var(--red); color: var(--red); background: rgba(248,81,73,0.1); } + .wf-btn { + background: none; + border: 1px solid var(--border); + border-radius: var(--radius); + color: var(--accent); + cursor: pointer; + font-size: 13px; + padding: 1px 6px; + line-height: 1; + } + .wf-btn:hover { background: rgba(56,139,253,0.15); border-color: var(--accent); } + .db-empty { color: var(--text-mute); font-size: 13px; @@ -921,6 +933,7 @@ + @@ -1744,7 +1757,9 @@ async function loadHistory() { const tr = document.createElement('tr'); const pvs = ev.peak_vector_sum; const maxPPV = Math.max(ev.tran_ppv ?? 0, ev.vert_ppv ?? 0, ev.long_ppv ?? 0); + const waveformUrl = `${api()}/waveform?db_id=${encodeURIComponent(ev.id)}&api_base=${encodeURIComponent(api())}`; tr.innerHTML = ` + diff --git a/sfm/waveform_viewer.html b/sfm/waveform_viewer.html index 8770544..651207b 100644 --- a/sfm/waveform_viewer.html +++ b/sfm/waveform_viewer.html @@ -244,6 +244,46 @@ let eventList = []; // populated from /device/events after connect let currentEventIndex = 0; + // ── DB mode: opened via ?db_id=&api_base= from History tab ──────── + const _urlParams = new URLSearchParams(window.location.search); + const _dbId = _urlParams.get('db_id'); + const _dbApiBase = (_urlParams.get('api_base') || '').replace(/\/$/, ''); + + async function _loadFromDb() { + const apiBase = _dbApiBase || document.getElementById('api-base').value.replace(/\/$/, ''); + setStatus('Loading waveform from database…', 'loading'); + document.getElementById('unit-bar').style.display = 'none'; + // Hide live-device controls — not relevant in DB mode + document.querySelector('header .conn-group').style.display = 'none'; + + const url = `${apiBase}/db/events/${encodeURIComponent(_dbId)}/waveform`; + let data; + try { + const resp = await fetch(url); + if (!resp.ok) { + const err = await resp.json().catch(() => ({ detail: resp.statusText })); + throw new Error(err.detail || resp.statusText); + } + data = await resp.json(); + } catch (e) { + setStatus(`Error: ${e.message}`, 'error'); + return; + } + lastData = data; + renderWaveform(data); + } + + // Auto-load when opened with db_id param + window.addEventListener('DOMContentLoaded', () => { + if (_dbId) { + // Pre-fill api-base if provided + if (_dbApiBase) { + document.getElementById('api-base').value = _dbApiBase; + } + _loadFromDb(); + } + }); + function setStatus(msg, cls = '') { const bar = document.getElementById('status-bar'); bar.textContent = msg; @@ -404,8 +444,11 @@ bar.innerHTML = ''; bar.className = 'ok'; const ts = data.timestamp; - if (ts) { - bar.textContent = `Event #${data.index} — ${ts.display} `; + const tsDisplay = ts + ? (typeof ts === 'string' ? ts : (ts.display ?? JSON.stringify(ts))) + : null; + if (tsDisplay) { + bar.textContent = `Event #${data.index} — ${tsDisplay} `; } else { bar.textContent = `Event #${data.index} `; }
Timestamp Serial Tran (in/s) ${_fmtTs(ev.timestamp)} ${ev.serial ?? '—'} ${_ppvFmt(ev.tran_ppv)}