feat: v0.15.0
### Added
- **Layered event storage architecture.** Each event now lands as four
files in the per-serial waveform store, each with a clear role:
- `<filename>` — the Blastware-readable binary (BW file). Untouched.
- `<filename>.a5.pkl` — the raw 5A frames (regenerative source).
- `<filename>.h5` — clean per-channel waveform arrays in physical
units (in/s for geo, psi for mic) plus event metadata (HDF5 with
gzip compression). This is the canonical format for downstream
analysis tools.
- `<filename>.sfm.json` — the modern review/metadata sidecar (peaks,
project, source provenance, review state, extensions).
SQLite (`seismo_relay.db`) is the searchable index over all four.
- **Plot-ready waveform JSON (`sfm.plot.v1`).** The `/device/event/{idx}/waveform`
and `/db/events/{id}/waveform.json` endpoints now return samples in
physical units with explicit time-axis metadata, peak markers, and
per-channel unit hints — no more guessing the ADC-to-velocity scale
client-side. The webapp waveform viewer was rewritten to consume
this shape.
- **In-app waveform viewer accuracy fix.** The standalone SFM webapp
viewer was scaling geophone amplitudes by `geoAdcScale / 32767`
(≈ 6.206 / 32767), where `geoAdcScale = 6.206053` is the device's
*in/s per V* hardware constant — not the ADC-counts-to-velocity
factor. This silently scaled every plot ~38% too low for Normal-range
geophones (the correct full-scale is 10.0 in/s, or 1.25 in/s for
Sensitive). Conversion is now done server-side using the geo_range
from compliance config; the client just plots.
- New `sfm/event_hdf5.py` module: `write_event_hdf5()`,
`read_event_hdf5()`, plus a plot-JSON helper.
- Backfill script extended to also emit `.h5` for existing events.
### Dependencies
- Added `h5py>=3.10` and `numpy>=1.24` for the HDF5 storage layer.
- Added `python-multipart>=0.0.7` (required by FastAPI for the
`/db/import/blastware_file` endpoint introduced in this release).
This commit is contained in:
+39
-3
@@ -84,6 +84,7 @@ CREATE TABLE IF NOT EXISTS events (
|
||||
blastware_filename TEXT, -- event file within waveform store; extension is per-event (AB0T encodes timestamp)
|
||||
blastware_filesize INTEGER, -- bytes; NULL if no event file saved
|
||||
a5_pickle_filename TEXT, -- "<filename>.a5.pkl" sidecar
|
||||
sidecar_filename TEXT, -- "<filename>.sfm.json" review/metadata sidecar
|
||||
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
|
||||
UNIQUE(serial, timestamp)
|
||||
);
|
||||
@@ -196,6 +197,7 @@ class SeismoDb:
|
||||
("blastware_filename", "TEXT"),
|
||||
("blastware_filesize", "INTEGER"),
|
||||
("a5_pickle_filename", "TEXT"),
|
||||
("sidecar_filename", "TEXT"),
|
||||
):
|
||||
if col not in existing_cols:
|
||||
log.info("_migrate: events ADD COLUMN %s %s", col, ddl)
|
||||
@@ -346,8 +348,9 @@ class SeismoDb:
|
||||
tran_ppv, vert_ppv, long_ppv, peak_vector_sum, mic_ppv,
|
||||
project, client, operator, sensor_location,
|
||||
sample_rate, record_type,
|
||||
blastware_filename, blastware_filesize, a5_pickle_filename)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
blastware_filename, blastware_filesize,
|
||||
a5_pickle_filename, sidecar_filename)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
self._new_id(), serial, key, session_id, ts,
|
||||
@@ -365,6 +368,7 @@ class SeismoDb:
|
||||
rec.get("filename"),
|
||||
rec.get("filesize"),
|
||||
rec.get("a5_pickle_filename"),
|
||||
rec.get("sidecar_filename"),
|
||||
),
|
||||
)
|
||||
inserted += 1
|
||||
@@ -379,13 +383,15 @@ class SeismoDb:
|
||||
UPDATE events
|
||||
SET blastware_filename = ?,
|
||||
blastware_filesize = ?,
|
||||
a5_pickle_filename = ?
|
||||
a5_pickle_filename = ?,
|
||||
sidecar_filename = ?
|
||||
WHERE serial = ? AND timestamp = ?
|
||||
""",
|
||||
(
|
||||
rec.get("filename"),
|
||||
rec.get("filesize"),
|
||||
rec.get("a5_pickle_filename"),
|
||||
rec.get("sidecar_filename"),
|
||||
serial,
|
||||
ts,
|
||||
),
|
||||
@@ -449,6 +455,36 @@ class SeismoDb:
|
||||
)
|
||||
return cur.rowcount > 0
|
||||
|
||||
def update_event_review(self, event_id: str, review: dict) -> bool:
|
||||
"""
|
||||
Sync derived index columns from a sidecar's `review` block.
|
||||
|
||||
Currently the only derived index is `events.false_trigger` — kept
|
||||
in sync so `/db/events?false_trigger=true` queries don't have to
|
||||
scan every sidecar JSON on disk. The sidecar JSON itself remains
|
||||
the source of truth for the full review state.
|
||||
|
||||
Returns True when the row exists, False otherwise. No-op fields
|
||||
(review without `false_trigger`) leave the column untouched.
|
||||
"""
|
||||
if not isinstance(review, dict):
|
||||
return False
|
||||
if "false_trigger" not in review:
|
||||
# Nothing derived to update; just confirm the row exists.
|
||||
with self._connect() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT 1 FROM events WHERE id=?", (event_id,),
|
||||
).fetchone()
|
||||
return row is not None
|
||||
|
||||
flag = 1 if review.get("false_trigger") else 0
|
||||
with self._connect() as conn:
|
||||
cur = conn.execute(
|
||||
"UPDATE events SET false_trigger=? WHERE id=?",
|
||||
(flag, event_id),
|
||||
)
|
||||
return cur.rowcount > 0
|
||||
|
||||
# ── Monitor log ───────────────────────────────────────────────────────────
|
||||
|
||||
def insert_monitor_log(
|
||||
|
||||
@@ -0,0 +1,530 @@
|
||||
"""
|
||||
sfm/event_hdf5.py — HDF5 codec for the canonical "clean waveform" file.
|
||||
|
||||
Layout written to `<filename>.h5`:
|
||||
|
||||
/
|
||||
├─ samples/
|
||||
│ ├─ Tran (float32, in/s) shape: (N,)
|
||||
│ ├─ Vert (float32, in/s) shape: (N,)
|
||||
│ ├─ Long (float32, in/s) shape: (N,)
|
||||
│ └─ MicL (float32, psi) shape: (N,)
|
||||
├─ samples_int16/ (optional)
|
||||
│ ├─ Tran (int16, raw ADC counts) shape: (N,)
|
||||
│ └─ ... per channel (only when present in the source)
|
||||
└─ root attrs (event metadata):
|
||||
schema_version int = 1
|
||||
kind str = "sfm.event.hdf5"
|
||||
serial str
|
||||
waveform_key str (8-hex)
|
||||
timestamp str (ISO-8601)
|
||||
record_type str
|
||||
sample_rate int (sps)
|
||||
pretrig_samples int
|
||||
total_samples int
|
||||
rectime_seconds float
|
||||
geo_range str "normal" | "sensitive"
|
||||
geo_full_scale_ips float (10.0 or 1.250)
|
||||
project str
|
||||
client str
|
||||
operator str
|
||||
sensor_location str
|
||||
peak_tran_ips float (from 0C; authoritative)
|
||||
peak_vert_ips float
|
||||
peak_long_ips float
|
||||
peak_pvs_ips float
|
||||
peak_mic_psi float
|
||||
tool_version str
|
||||
captured_at str (ISO-8601 UTC)
|
||||
source_kind str "sfm-live" | "sfm-ach" | "bw-import"
|
||||
|
||||
Why HDF5 and not just JSON for the canonical clean format:
|
||||
- Native float32 arrays (no base64 dance, no per-value JSON parsing).
|
||||
- Per-dataset gzip compression — sample arrays compress 3-5×.
|
||||
- Cross-language: h5py (Python), HDF5.jl (Julia), io.netcdf (R), etc.
|
||||
Analysis pipelines don't have to know anything about Blastware.
|
||||
- Self-describing via attributes; future fields don't break readers.
|
||||
|
||||
The plot-ready `sfm.plot.v1` JSON returned by the REST endpoints is
|
||||
derived from this HDF5 (or computed on-the-fly when no .h5 exists yet).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Optional, Union
|
||||
|
||||
import h5py
|
||||
import numpy as np
|
||||
|
||||
from minimateplus.event_file_io import TOOL_VERSION as _DEFAULT_TOOL_VERSION
|
||||
from minimateplus.models import Event
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
SCHEMA_VERSION = 1
|
||||
HDF5_KIND = "sfm.event.hdf5"
|
||||
|
||||
# Geophone full-scale velocity per range (in/s). Confirmed in CLAUDE.md
|
||||
# from 4-20-26 captures: Normal=0x00 → 10 in/s, Sensitive=0x01 → 1.25 in/s.
|
||||
_GEO_FS_BY_RANGE = {
|
||||
"normal": 10.000,
|
||||
"sensitive": 1.2500,
|
||||
0: 10.000,
|
||||
1: 1.2500,
|
||||
}
|
||||
_INT16_FS = 32768.0
|
||||
|
||||
# Default mic conversion: ADC count → psi. Approximate; exact factor
|
||||
# depends on firmware reference voltage and mic sensitivity, neither of
|
||||
# which is independently confirmed. We try to refine it from the device-
|
||||
# reported peak when available (peak_mic_psi / max_abs_int16).
|
||||
_MIC_DEFAULT_FS_PSI = 0.0125 # ≈ 0.5 psi at full scale (rough)
|
||||
|
||||
|
||||
def _resolve_geo_full_scale(geo_range) -> float:
|
||||
"""Map a geo_range value (string or int from compliance config) to the
|
||||
full-scale velocity in in/s. Defaults to Normal range (10.0) when the
|
||||
value is unknown — same default as Blastware itself."""
|
||||
if geo_range is None:
|
||||
return _GEO_FS_BY_RANGE["normal"]
|
||||
if isinstance(geo_range, str):
|
||||
return _GEO_FS_BY_RANGE.get(geo_range.lower(), _GEO_FS_BY_RANGE["normal"])
|
||||
return _GEO_FS_BY_RANGE.get(int(geo_range), _GEO_FS_BY_RANGE["normal"])
|
||||
|
||||
|
||||
def _normalise_range(geo_range) -> str:
|
||||
"""Return 'normal' or 'sensitive' (string) regardless of input form."""
|
||||
if isinstance(geo_range, str):
|
||||
v = geo_range.lower()
|
||||
if v in ("normal", "sensitive"):
|
||||
return v
|
||||
return "normal"
|
||||
if geo_range == 1:
|
||||
return "sensitive"
|
||||
return "normal"
|
||||
|
||||
|
||||
def _ts_iso(ts) -> str:
|
||||
if ts is None:
|
||||
return ""
|
||||
try:
|
||||
return datetime.datetime(
|
||||
ts.year, ts.month, ts.day,
|
||||
ts.hour or 0, ts.minute or 0, ts.second or 0,
|
||||
).isoformat()
|
||||
except Exception:
|
||||
return str(ts)
|
||||
|
||||
|
||||
def _samples_to_float(
|
||||
samples_int16: list[int],
|
||||
full_scale: float,
|
||||
) -> np.ndarray:
|
||||
"""Convert int16 ADC counts → float32 physical units.
|
||||
|
||||
Uses _INT16_FS=32768 (not 32767) so that a count of -32768 maps to
|
||||
exactly -full_scale and +32767 maps to ~+full_scale * 32767/32768.
|
||||
Matches the device firmware's documented mapping (see CLAUDE.md
|
||||
geo_hardware_constant rationale).
|
||||
"""
|
||||
if not samples_int16:
|
||||
return np.array([], dtype=np.float32)
|
||||
arr = np.asarray(samples_int16, dtype=np.int32) # int32 to avoid overflow during scale
|
||||
return (arr.astype(np.float32) * (full_scale / _INT16_FS)).astype(np.float32)
|
||||
|
||||
|
||||
def _mic_scale_factor(
|
||||
samples_int16: list[int],
|
||||
peak_mic_psi: Optional[float],
|
||||
) -> float:
|
||||
"""Resolve the per-count psi factor for the microphone channel.
|
||||
|
||||
When the device reports a peak mic value via the 0C record, we
|
||||
back-solve the per-count factor from `peak_psi / max(|samples|)` so
|
||||
the plotted waveform peaks land exactly at the device-reported value.
|
||||
Otherwise fall back to the rough _MIC_DEFAULT_FS_PSI estimate.
|
||||
"""
|
||||
if peak_mic_psi is not None and peak_mic_psi > 0 and samples_int16:
|
||||
max_count = max(abs(int(v)) for v in samples_int16) or 1
|
||||
return float(peak_mic_psi) / float(max_count)
|
||||
return _MIC_DEFAULT_FS_PSI / _INT16_FS
|
||||
|
||||
|
||||
def write_event_hdf5(
|
||||
path: Union[str, Path],
|
||||
event: Event,
|
||||
*,
|
||||
serial: str,
|
||||
geo_range = "normal",
|
||||
source_kind: str = "sfm-live",
|
||||
tool_version: Optional[str] = None,
|
||||
captured_at: Optional[datetime.datetime] = None,
|
||||
include_int16: bool = True,
|
||||
) -> dict:
|
||||
"""
|
||||
Persist a decoded Event as an HDF5 file with samples in physical units.
|
||||
|
||||
Returns a small summary dict suitable for logging:
|
||||
{"path": Path, "n_samples": int, "geo_full_scale_ips": float}
|
||||
"""
|
||||
path = Path(path)
|
||||
raw = event.raw_samples or {}
|
||||
pv = event.peak_values
|
||||
pi = event.project_info
|
||||
|
||||
geo_fs = _resolve_geo_full_scale(geo_range)
|
||||
geo_range_str = _normalise_range(geo_range)
|
||||
captured_at = captured_at or datetime.datetime.utcnow()
|
||||
tool_version = tool_version or _DEFAULT_TOOL_VERSION
|
||||
|
||||
# Per-channel float32 arrays in physical units.
|
||||
geo_arrays = {}
|
||||
for ch in ("Tran", "Vert", "Long"):
|
||||
geo_arrays[ch] = _samples_to_float(raw.get(ch, []), geo_fs)
|
||||
|
||||
# Mic channel — the per-count factor is resolved from the device-reported
|
||||
# peak when available so the plot peaks the BW value exactly.
|
||||
mic_int16 = raw.get("MicL", [])
|
||||
mic_factor = _mic_scale_factor(
|
||||
mic_int16,
|
||||
getattr(pv, "micl", None) if pv else None,
|
||||
)
|
||||
if mic_int16:
|
||||
mic_arr = (np.asarray(mic_int16, dtype=np.int32).astype(np.float32) * mic_factor).astype(np.float32)
|
||||
else:
|
||||
mic_arr = np.array([], dtype=np.float32)
|
||||
|
||||
n_samples = max(
|
||||
(len(geo_arrays[ch]) for ch in geo_arrays),
|
||||
default=0,
|
||||
)
|
||||
|
||||
# Atomic write: temp file + os.replace.
|
||||
tmp = path.with_suffix(path.suffix + ".tmp")
|
||||
with h5py.File(tmp, "w") as f:
|
||||
# Root attrs — event-level metadata.
|
||||
attrs = f.attrs
|
||||
attrs["schema_version"] = SCHEMA_VERSION
|
||||
attrs["kind"] = HDF5_KIND
|
||||
attrs["serial"] = serial or ""
|
||||
attrs["waveform_key"] = event._waveform_key.hex() if event._waveform_key else ""
|
||||
attrs["timestamp"] = _ts_iso(event.timestamp)
|
||||
attrs["record_type"] = event.record_type or ""
|
||||
attrs["sample_rate"] = int(event.sample_rate or 0)
|
||||
attrs["pretrig_samples"] = int(event.pretrig_samples or 0)
|
||||
attrs["total_samples"] = int(event.total_samples or n_samples)
|
||||
attrs["rectime_seconds"] = float(event.rectime_seconds or 0.0)
|
||||
attrs["geo_range"] = geo_range_str
|
||||
attrs["geo_full_scale_ips"] = float(geo_fs)
|
||||
attrs["project"] = (pi.project if pi else "") or ""
|
||||
attrs["client"] = (pi.client if pi else "") or ""
|
||||
attrs["operator"] = (pi.operator if pi else "") or ""
|
||||
attrs["sensor_location"] = (pi.sensor_location if pi else "") or ""
|
||||
attrs["peak_tran_ips"] = float(pv.tran if pv and pv.tran is not None else 0.0)
|
||||
attrs["peak_vert_ips"] = float(pv.vert if pv and pv.vert is not None else 0.0)
|
||||
attrs["peak_long_ips"] = float(pv.long if pv and pv.long is not None else 0.0)
|
||||
attrs["peak_pvs_ips"] = float(pv.peak_vector_sum if pv and pv.peak_vector_sum is not None else 0.0)
|
||||
attrs["peak_mic_psi"] = float(pv.micl if pv and pv.micl is not None else 0.0)
|
||||
attrs["tool_version"] = tool_version or ""
|
||||
attrs["captured_at"] = captured_at.isoformat() + "Z" if captured_at.tzinfo is None else captured_at.isoformat()
|
||||
attrs["source_kind"] = source_kind
|
||||
|
||||
# /samples — physical-units float32 (the primary data).
|
||||
sgrp = f.create_group("samples")
|
||||
for ch, arr in geo_arrays.items():
|
||||
sgrp.create_dataset(
|
||||
ch, data=arr, dtype="float32",
|
||||
compression="gzip", compression_opts=4, shuffle=True,
|
||||
)
|
||||
sgrp.create_dataset(
|
||||
"MicL", data=mic_arr, dtype="float32",
|
||||
compression="gzip", compression_opts=4, shuffle=True,
|
||||
)
|
||||
|
||||
# /samples_int16 — optional raw ADC counts (preserved for analysis
|
||||
# tools that want pre-conversion data). Cheap to include.
|
||||
if include_int16:
|
||||
igrp = f.create_group("samples_int16")
|
||||
for ch in ("Tran", "Vert", "Long", "MicL"):
|
||||
vals = raw.get(ch, [])
|
||||
if vals:
|
||||
igrp.create_dataset(
|
||||
ch, data=np.asarray(vals, dtype=np.int16),
|
||||
compression="gzip", compression_opts=4, shuffle=True,
|
||||
)
|
||||
igrp.attrs["mic_psi_per_count"] = float(mic_factor)
|
||||
|
||||
import os
|
||||
os.replace(tmp, path)
|
||||
|
||||
log.info(
|
||||
"write_event_hdf5: %s n_samples=%d geo_fs=%.3f filesize=%d",
|
||||
path, n_samples, geo_fs, path.stat().st_size,
|
||||
)
|
||||
return {
|
||||
"path": path,
|
||||
"n_samples": n_samples,
|
||||
"geo_full_scale_ips": geo_fs,
|
||||
}
|
||||
|
||||
|
||||
def read_event_hdf5(path: Union[str, Path]) -> dict:
|
||||
"""
|
||||
Load an event HDF5 into a plain dict (no Event reconstruction —
|
||||
callers that want an Event can use the data directly).
|
||||
|
||||
Returns:
|
||||
{
|
||||
"schema_version": int,
|
||||
"kind": str,
|
||||
"attrs": dict[str, …], # all root attributes
|
||||
"samples": { # float32 lists in physical units
|
||||
"Tran": ndarray, "Vert": ndarray, "Long": ndarray, "MicL": ndarray,
|
||||
},
|
||||
"samples_int16": {…} or None,
|
||||
"mic_psi_per_count": float | None,
|
||||
}
|
||||
|
||||
Raises FileNotFoundError if missing, ValueError on bad shape /
|
||||
unsupported schema_version.
|
||||
"""
|
||||
path = Path(path)
|
||||
with h5py.File(path, "r") as f:
|
||||
attrs = {k: _h5_attr_value(v) for k, v in f.attrs.items()}
|
||||
sv = attrs.get("schema_version", 0)
|
||||
if not isinstance(sv, int) or sv < 1 or sv > SCHEMA_VERSION:
|
||||
raise ValueError(
|
||||
f"{path}: unsupported HDF5 schema_version={sv} "
|
||||
f"(this build supports 1..{SCHEMA_VERSION})"
|
||||
)
|
||||
if attrs.get("kind") != HDF5_KIND:
|
||||
raise ValueError(f"{path}: kind != {HDF5_KIND!r} (got {attrs.get('kind')!r})")
|
||||
|
||||
samples = {}
|
||||
for ch in ("Tran", "Vert", "Long", "MicL"):
|
||||
ds = f.get(f"samples/{ch}")
|
||||
samples[ch] = np.asarray(ds[()]) if ds is not None else np.array([], dtype=np.float32)
|
||||
|
||||
samples_int16 = None
|
||||
mic_psi = None
|
||||
igrp = f.get("samples_int16")
|
||||
if igrp is not None:
|
||||
samples_int16 = {}
|
||||
for ch in ("Tran", "Vert", "Long", "MicL"):
|
||||
ds = igrp.get(ch)
|
||||
if ds is not None:
|
||||
samples_int16[ch] = np.asarray(ds[()])
|
||||
mic_attr = igrp.attrs.get("mic_psi_per_count")
|
||||
if mic_attr is not None:
|
||||
mic_psi = float(mic_attr)
|
||||
|
||||
return {
|
||||
"schema_version": sv,
|
||||
"kind": attrs.get("kind"),
|
||||
"attrs": attrs,
|
||||
"samples": samples,
|
||||
"samples_int16": samples_int16,
|
||||
"mic_psi_per_count": mic_psi,
|
||||
}
|
||||
|
||||
|
||||
def _h5_attr_value(v):
|
||||
"""Convert an h5py attribute value to a plain Python type."""
|
||||
if isinstance(v, bytes):
|
||||
return v.decode("utf-8", errors="replace")
|
||||
if isinstance(v, np.generic):
|
||||
return v.item()
|
||||
return v
|
||||
|
||||
|
||||
# ── Plot-ready JSON ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def event_to_plot_json(
|
||||
event: Event,
|
||||
*,
|
||||
serial: str,
|
||||
geo_range = "normal",
|
||||
event_id: Optional[str] = None,
|
||||
index: Optional[int] = None,
|
||||
) -> dict:
|
||||
"""
|
||||
Build a `sfm.plot.v1` JSON dict directly from an Event (skipping HDF5).
|
||||
|
||||
Used by:
|
||||
- `/device/event/{idx}/waveform` (live device path)
|
||||
- The CLI / tests for in-memory conversion sanity-checks.
|
||||
|
||||
Stored events go through `plot_json_from_hdf5()` so the wire format
|
||||
is identical regardless of whether the data came from the live device
|
||||
or the on-disk HDF5.
|
||||
"""
|
||||
raw = event.raw_samples or {}
|
||||
pv = event.peak_values
|
||||
geo_fs = _resolve_geo_full_scale(geo_range)
|
||||
geo_range_str = _normalise_range(geo_range)
|
||||
sr = int(event.sample_rate or 0) or 1024
|
||||
pretrig = int(event.pretrig_samples or 0)
|
||||
|
||||
geo_arrays = {ch: _samples_to_float(raw.get(ch, []), geo_fs).tolist()
|
||||
for ch in ("Tran", "Vert", "Long")}
|
||||
mic_int16 = raw.get("MicL", [])
|
||||
mic_factor = _mic_scale_factor(
|
||||
mic_int16,
|
||||
getattr(pv, "micl", None) if pv else None,
|
||||
)
|
||||
mic_arr = [float(v) * mic_factor for v in mic_int16] if mic_int16 else []
|
||||
|
||||
n = max(
|
||||
(len(geo_arrays[ch]) for ch in geo_arrays),
|
||||
default=len(mic_arr),
|
||||
)
|
||||
return _build_plot_dict(
|
||||
n_samples=n,
|
||||
sample_rate=sr,
|
||||
pretrig_samples=pretrig,
|
||||
total_samples=int(event.total_samples or n),
|
||||
rectime_seconds=float(event.rectime_seconds or 0.0),
|
||||
timestamp_iso=_ts_iso(event.timestamp),
|
||||
serial=serial,
|
||||
record_type=event.record_type,
|
||||
waveform_key=event._waveform_key.hex() if event._waveform_key else None,
|
||||
geo_range=geo_range_str,
|
||||
geo_fs=geo_fs,
|
||||
channels_floats={
|
||||
"Tran": geo_arrays["Tran"],
|
||||
"Vert": geo_arrays["Vert"],
|
||||
"Long": geo_arrays["Long"],
|
||||
"MicL": mic_arr,
|
||||
},
|
||||
peaks_dict={
|
||||
"tran": getattr(pv, "tran", None) if pv else None,
|
||||
"vert": getattr(pv, "vert", None) if pv else None,
|
||||
"long": getattr(pv, "long", None) if pv else None,
|
||||
"pvs": getattr(pv, "peak_vector_sum", None) if pv else None,
|
||||
"mic": getattr(pv, "micl", None) if pv else None,
|
||||
},
|
||||
event_id=event_id,
|
||||
index=index if index is not None else event.index,
|
||||
)
|
||||
|
||||
|
||||
def plot_json_from_hdf5(
|
||||
path: Union[str, Path],
|
||||
*,
|
||||
event_id: Optional[str] = None,
|
||||
index: Optional[int] = None,
|
||||
) -> dict:
|
||||
"""Build a `sfm.plot.v1` JSON dict from a stored .h5 file."""
|
||||
data = read_event_hdf5(path)
|
||||
a = data["attrs"]
|
||||
s = data["samples"]
|
||||
return _build_plot_dict(
|
||||
n_samples=len(s["Tran"]) if "Tran" in s else 0,
|
||||
sample_rate=int(a.get("sample_rate", 1024) or 1024),
|
||||
pretrig_samples=int(a.get("pretrig_samples", 0) or 0),
|
||||
total_samples=int(a.get("total_samples", 0) or 0),
|
||||
rectime_seconds=float(a.get("rectime_seconds", 0.0) or 0.0),
|
||||
timestamp_iso=a.get("timestamp", ""),
|
||||
serial=a.get("serial", ""),
|
||||
record_type=a.get("record_type", ""),
|
||||
waveform_key=a.get("waveform_key", "") or None,
|
||||
geo_range=a.get("geo_range", "normal"),
|
||||
geo_fs=float(a.get("geo_full_scale_ips", 10.0) or 10.0),
|
||||
channels_floats={
|
||||
"Tran": s.get("Tran", np.array([])).tolist(),
|
||||
"Vert": s.get("Vert", np.array([])).tolist(),
|
||||
"Long": s.get("Long", np.array([])).tolist(),
|
||||
"MicL": s.get("MicL", np.array([])).tolist(),
|
||||
},
|
||||
peaks_dict={
|
||||
"tran": float(a.get("peak_tran_ips", 0.0) or 0.0) or None,
|
||||
"vert": float(a.get("peak_vert_ips", 0.0) or 0.0) or None,
|
||||
"long": float(a.get("peak_long_ips", 0.0) or 0.0) or None,
|
||||
"pvs": float(a.get("peak_pvs_ips", 0.0) or 0.0) or None,
|
||||
"mic": float(a.get("peak_mic_psi", 0.0) or 0.0) or None,
|
||||
},
|
||||
event_id=event_id,
|
||||
index=index,
|
||||
)
|
||||
|
||||
|
||||
def _build_plot_dict(
|
||||
*,
|
||||
n_samples: int,
|
||||
sample_rate: int,
|
||||
pretrig_samples: int,
|
||||
total_samples: int,
|
||||
rectime_seconds: float,
|
||||
timestamp_iso: str,
|
||||
serial: str,
|
||||
record_type: Optional[str],
|
||||
waveform_key: Optional[str],
|
||||
geo_range: str,
|
||||
geo_fs: float,
|
||||
channels_floats: dict[str, list[float]],
|
||||
peaks_dict: dict[str, Optional[float]],
|
||||
event_id: Optional[str],
|
||||
index: Optional[int] = None,
|
||||
) -> dict:
|
||||
dt_ms = (1000.0 / sample_rate) if sample_rate > 0 else 0.0
|
||||
t0_ms = -pretrig_samples * dt_ms
|
||||
|
||||
def _ch(unit: str, values: list[float], peak: Optional[float]) -> dict:
|
||||
# Locate the peak's time within the values array (max abs).
|
||||
if values:
|
||||
mags = [abs(v) for v in values]
|
||||
i = mags.index(max(mags))
|
||||
peak_t_ms = round(t0_ms + i * dt_ms, 4)
|
||||
peak_value = peak if peak is not None else values[i]
|
||||
else:
|
||||
peak_t_ms = None
|
||||
peak_value = peak
|
||||
return {
|
||||
"unit": unit,
|
||||
"values": values,
|
||||
"peak": peak_value,
|
||||
"peak_t_ms": peak_t_ms,
|
||||
}
|
||||
|
||||
return {
|
||||
"schema": "sfm.plot.v1",
|
||||
"event_id": event_id,
|
||||
"index": index,
|
||||
"serial": serial,
|
||||
"timestamp": timestamp_iso,
|
||||
"record_type": record_type,
|
||||
"waveform_key": waveform_key,
|
||||
|
||||
"time_axis": {
|
||||
"sample_rate": sample_rate,
|
||||
"pretrig_samples": pretrig_samples,
|
||||
"total_samples": total_samples or n_samples,
|
||||
"n_samples": n_samples,
|
||||
"t0_ms": round(t0_ms, 4),
|
||||
"dt_ms": round(dt_ms, 6),
|
||||
"rectime_seconds": rectime_seconds,
|
||||
},
|
||||
|
||||
"geo_range": geo_range,
|
||||
"geo_full_scale_ips": geo_fs,
|
||||
"trigger_ms": 0.0,
|
||||
|
||||
"channels": {
|
||||
"Tran": _ch("in/s", channels_floats.get("Tran", []), peaks_dict.get("tran")),
|
||||
"Vert": _ch("in/s", channels_floats.get("Vert", []), peaks_dict.get("vert")),
|
||||
"Long": _ch("in/s", channels_floats.get("Long", []), peaks_dict.get("long")),
|
||||
"MicL": _ch("psi", channels_floats.get("MicL", []), peaks_dict.get("mic")),
|
||||
},
|
||||
|
||||
"peak_values": {
|
||||
"transverse": peaks_dict.get("tran"),
|
||||
"vertical": peaks_dict.get("vert"),
|
||||
"longitudinal": peaks_dict.get("long"),
|
||||
"vector_sum": peaks_dict.get("pvs"),
|
||||
"mic_psi": peaks_dict.get("mic"),
|
||||
},
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
"""
|
||||
sfm/import_bw.py — CLI for ingesting Blastware-format event files.
|
||||
|
||||
Walks a path (file or directory), parses each recognised event-file
|
||||
binary, copies it into the canonical waveform store, writes the
|
||||
.sfm.json sidecar, and upserts a row in seismo_relay.db.
|
||||
|
||||
Use cases:
|
||||
- Migrating a Blastware ACH inbox into SFM
|
||||
- One-off imports of files emailed in by field crews
|
||||
- Bulk-loading historical archives
|
||||
|
||||
Usage:
|
||||
python -m sfm.import_bw <path-or-dir> [--serial BE11529]
|
||||
[--db-path bridges/captures/seismo_relay.db]
|
||||
[--store-root bridges/captures/waveforms]
|
||||
[--dry-run]
|
||||
[-v]
|
||||
|
||||
Examples:
|
||||
python -m sfm.import_bw ~/Downloads/M529LKIQ.7M0W
|
||||
python -m sfm.import_bw /path/to/blastware_archive --serial BE11529
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Iterator
|
||||
|
||||
# Allow running from the repo root without installation.
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from sfm.database import SeismoDb
|
||||
from sfm.waveform_store import WaveformStore
|
||||
|
||||
log = logging.getLogger("sfm.import_bw")
|
||||
|
||||
|
||||
# Blastware event-file extensions: 4-char `AB0T` (T = W or H) for ACH
|
||||
# downloads, 3-char `AB0` for direct downloads. We discover candidates
|
||||
# by length + last-char rather than enumerating every (A, B) pair.
|
||||
def _looks_like_bw_event(path: Path) -> bool:
|
||||
"""Heuristic: 3-char or 4-char extension, ends with W/H/0, and the
|
||||
file is at least 70 bytes (header + STRT + footer minimum)."""
|
||||
if not path.is_file():
|
||||
return False
|
||||
ext = path.suffix.lstrip(".")
|
||||
if not (3 <= len(ext) <= 4):
|
||||
return False
|
||||
if not (ext[-1].upper() in {"W", "H"} or ext.endswith("0")):
|
||||
return False
|
||||
try:
|
||||
return path.stat().st_size >= 70
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _walk(path: Path) -> Iterator[Path]:
|
||||
"""Yield candidate BW event-file paths under `path` (file or dir)."""
|
||||
if path.is_file():
|
||||
if _looks_like_bw_event(path):
|
||||
yield path
|
||||
return
|
||||
if path.is_dir():
|
||||
for p in sorted(path.rglob("*")):
|
||||
if _looks_like_bw_event(p):
|
||||
yield p
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
p = argparse.ArgumentParser(
|
||||
description="Import Blastware-format event files into the SFM store + DB.",
|
||||
)
|
||||
p.add_argument("path", help="File or directory to import.")
|
||||
p.add_argument(
|
||||
"--serial", default=None, metavar="SERIAL",
|
||||
help="Override the serial-number hint (e.g. BE11529). Defaults to "
|
||||
"the value decoded from each BW filename's prefix.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--db-path",
|
||||
default=str(Path(__file__).resolve().parent.parent / "bridges" / "captures" / "seismo_relay.db"),
|
||||
help="Path to seismo_relay.db (default: bridges/captures/seismo_relay.db).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--store-root",
|
||||
default=None,
|
||||
help="Root of the waveform store (default: <db_dir>/waveforms).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--dry-run", action="store_true",
|
||||
help="Parse and report per-file outcomes; don't write anything.",
|
||||
)
|
||||
p.add_argument("-v", "--verbose", action="store_true", help="Debug logging.")
|
||||
args = p.parse_args(argv)
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.DEBUG if args.verbose else logging.INFO,
|
||||
format="%(asctime)s %(levelname)-7s %(name)s %(message)s",
|
||||
datefmt="%H:%M:%S",
|
||||
)
|
||||
|
||||
src = Path(args.path).expanduser().resolve()
|
||||
if not src.exists():
|
||||
print(f"error: {src} does not exist", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
db_path = Path(args.db_path).expanduser().resolve()
|
||||
store_root = (
|
||||
Path(args.store_root).expanduser().resolve()
|
||||
if args.store_root else db_path.parent / "waveforms"
|
||||
)
|
||||
|
||||
db = None if args.dry_run else SeismoDb(db_path)
|
||||
store = None if args.dry_run else WaveformStore(store_root)
|
||||
|
||||
candidates = list(_walk(src))
|
||||
if not candidates:
|
||||
print(f"No BW event-file candidates found under {src}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print(f"Importing {len(candidates)} file(s) from {src}...")
|
||||
if args.dry_run:
|
||||
print("(dry-run — no writes will occur)")
|
||||
|
||||
ok = err = skipped = 0
|
||||
for path in candidates:
|
||||
try:
|
||||
bw_bytes = path.read_bytes()
|
||||
except Exception as exc:
|
||||
print(f" [ERR ] {path}: read failed: {exc}")
|
||||
err += 1
|
||||
continue
|
||||
|
||||
if args.dry_run:
|
||||
# Just parse to verify integrity; don't touch DB or store.
|
||||
from minimateplus import event_file_io
|
||||
try:
|
||||
ev = event_file_io.read_blastware_file(path)
|
||||
ts = ev.timestamp and (
|
||||
f"{ev.timestamp.year}-{ev.timestamp.month:02d}-{ev.timestamp.day:02d} "
|
||||
f"{ev.timestamp.hour:02d}:{ev.timestamp.minute:02d}:{ev.timestamp.second:02d}"
|
||||
) or "?"
|
||||
pv = ev.peak_values
|
||||
pvs = pv.peak_vector_sum if pv and pv.peak_vector_sum is not None else 0.0
|
||||
print(f" [OK ] {path.name} ts={ts} PVS={pvs:.4f}")
|
||||
ok += 1
|
||||
except Exception as exc:
|
||||
print(f" [ERR ] {path}: parse failed: {exc}")
|
||||
err += 1
|
||||
continue
|
||||
|
||||
try:
|
||||
ev, rec = store.save_imported_bw(
|
||||
bw_bytes, source_path=path, serial_hint=args.serial,
|
||||
)
|
||||
# Resolve serial for the DB row. Prefer the hint, then the
|
||||
# one decoded from the filename (already done by the store).
|
||||
serial_used = args.serial or _infer_serial(path.name) or "UNKNOWN"
|
||||
ins, sk = db.insert_events(
|
||||
[ev], serial=serial_used,
|
||||
waveform_records=(
|
||||
{ev._waveform_key.hex(): rec}
|
||||
if ev._waveform_key else None
|
||||
),
|
||||
)
|
||||
tag = "OK " if ins else ("SKIP" if sk else "OK ")
|
||||
print(f" [{tag}] {path.name} → {rec['filename']} "
|
||||
f"({rec['filesize']} B, sha256={rec['sha256'][:12]}…) "
|
||||
f"serial={serial_used} ins={ins} skip={sk}")
|
||||
if ins:
|
||||
ok += 1
|
||||
else:
|
||||
skipped += 1
|
||||
except Exception as exc:
|
||||
print(f" [ERR ] {path}: import failed: {exc}")
|
||||
log.debug("traceback", exc_info=True)
|
||||
err += 1
|
||||
|
||||
print(f"\nDone. ok={ok} skipped={skipped} errors={err}")
|
||||
return 0 if err == 0 else 1
|
||||
|
||||
|
||||
def _infer_serial(filename: str):
|
||||
"""Reuse WaveformStore's filename → serial decoder for log output."""
|
||||
from sfm.waveform_store import _serial_from_bw_filename
|
||||
return _serial_from_bw_filename(filename)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+269
-66
@@ -45,7 +45,7 @@ from typing import Optional
|
||||
|
||||
# FastAPI / Pydantic
|
||||
try:
|
||||
from fastapi import Body, FastAPI, HTTPException, Query
|
||||
from fastapi import Body, FastAPI, File, HTTPException, Query, UploadFile
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import FileResponse, JSONResponse, StreamingResponse
|
||||
from pydantic import BaseModel
|
||||
@@ -64,6 +64,7 @@ from minimateplus.models import CallHomeConfig, ComplianceConfig, DeviceInfo, Ev
|
||||
from minimateplus.transport import TcpTransport, DEFAULT_TCP_PORT
|
||||
from minimateplus.blastware_file import write_blastware_file, blastware_filename
|
||||
from minimateplus.client import _decode_a5_metadata_into, _decode_a5_waveform
|
||||
from sfm import event_hdf5
|
||||
from sfm.cache import SFMCache, get_cache
|
||||
from sfm.database import SeismoDb
|
||||
from sfm.live_cache import LiveCache as _LiveCache
|
||||
@@ -732,26 +733,33 @@ def device_event_waveform(
|
||||
detail=f"Event index {index} not found on device",
|
||||
)
|
||||
|
||||
raw = getattr(ev, "raw_samples", None) or {}
|
||||
samples_decoded = len(raw.get("Tran", []))
|
||||
# Backfill from compliance_config: sample_rate, record_time, and
|
||||
# derived total_samples. These are user-set authoritative values; the
|
||||
# corresponding STRT-derived guesses in `_decode_a5_waveform` can be
|
||||
# off (e.g. rectime used to read the 0x46 record-type marker = 70s).
|
||||
cc = info.compliance_config
|
||||
if cc:
|
||||
if ev.sample_rate is None and cc.sample_rate:
|
||||
ev.sample_rate = cc.sample_rate
|
||||
if cc.record_time:
|
||||
ev.rectime_seconds = cc.record_time
|
||||
if ev.sample_rate and ev.rectime_seconds:
|
||||
derived = int(round(ev.sample_rate * ev.rectime_seconds))
|
||||
if (ev.total_samples is None
|
||||
or ev.total_samples > derived * 2
|
||||
or ev.total_samples < derived // 4):
|
||||
ev.total_samples = derived
|
||||
geo_range = getattr(cc, "geo_range", None) if cc else None
|
||||
|
||||
# Resolve sample_rate from compliance config if not on the event itself
|
||||
sample_rate = ev.sample_rate
|
||||
if sample_rate is None and info.compliance_config:
|
||||
sample_rate = info.compliance_config.sample_rate
|
||||
|
||||
result = {
|
||||
"index": ev.index,
|
||||
"record_type": ev.record_type,
|
||||
"timestamp": _serialise_timestamp(ev.timestamp),
|
||||
"total_samples": ev.total_samples,
|
||||
"pretrig_samples": ev.pretrig_samples,
|
||||
"rectime_seconds": ev.rectime_seconds,
|
||||
"samples_decoded": samples_decoded,
|
||||
"sample_rate": sample_rate,
|
||||
"peak_values": _serialise_peak_values(ev.peak_values),
|
||||
"channels": raw,
|
||||
}
|
||||
# Build the plot.v1 JSON: samples in physical units (in/s for geo, psi
|
||||
# for mic), explicit time axis, peak markers — the shape clients should
|
||||
# consume directly without doing any ADC scaling.
|
||||
serial = getattr(info, "serial", None) or ""
|
||||
result = event_hdf5.event_to_plot_json(
|
||||
ev, serial=serial,
|
||||
geo_range=geo_range or "normal",
|
||||
index=index,
|
||||
)
|
||||
cache.set_waveform(conn_key, index, result)
|
||||
return result
|
||||
|
||||
@@ -781,8 +789,9 @@ def device_event_blastware_file(
|
||||
triggered events; histogram requires recording_mode
|
||||
to be populated from compliance config)
|
||||
|
||||
Performs: POLL startup → get_events(full_waveform=False, extra_chunks=1,
|
||||
stop_after_index=index) → write_blastware_file() → FileResponse.
|
||||
Performs: POLL startup → get_events(full_waveform=True,
|
||||
stop_after_index=index) → write_blastware_file() → FileResponse +
|
||||
persistent store + DB upsert.
|
||||
"""
|
||||
log.info(
|
||||
"GET /device/event/%d/blastware_file port=%s host=%s force=%s",
|
||||
@@ -790,19 +799,19 @@ def device_event_blastware_file(
|
||||
)
|
||||
# `force` always re-downloads from the device. This endpoint already
|
||||
# never short-circuits via cache, so `force` is reserved for parity with
|
||||
# the other live endpoints and to suppress the post-download persist
|
||||
# (see end of handler) when the caller wants a fetch-only escape hatch.
|
||||
# the other live endpoints.
|
||||
|
||||
try:
|
||||
def _do():
|
||||
with _build_client(port, baud, host, tcp_port, timeout=120.0) as client:
|
||||
info = client.connect()
|
||||
# Under v0.14.0 BW-exact 5A walk, the chunk loop is bounded by
|
||||
# the event end_offset extracted from STRT. No more
|
||||
# stop_after_metadata / extra_chunks gymnastics — these
|
||||
# kwargs are now no-ops.
|
||||
# full_waveform=True pulls the complete 5A stream so the
|
||||
# client populates STRT-derived fields (total_samples,
|
||||
# pretrig_samples, rectime_seconds) AND raw_samples on the
|
||||
# Event. Required for the .h5 + .sfm.json sidecar to be
|
||||
# filled in correctly — without it, those land as nulls.
|
||||
events = client.get_events(
|
||||
full_waveform=False,
|
||||
full_waveform=True,
|
||||
stop_after_index=index,
|
||||
)
|
||||
matching = [ev for ev in events if ev.index == index]
|
||||
@@ -861,7 +870,34 @@ def device_event_blastware_file(
|
||||
# queryable via /db/events afterwards (matches the ACH ingestion path).
|
||||
if serial != "UNKNOWN" and ev._waveform_key is not None:
|
||||
try:
|
||||
rec = _get_store().save(ev, serial=serial, a5_frames=a5_frames)
|
||||
cc = info.compliance_config
|
||||
# Backfill authoritative compliance-config values onto the
|
||||
# Event before persisting. These supersede whatever
|
||||
# _decode_a5_waveform read from the STRT bytes (some of which
|
||||
# have ambiguous semantics — e.g. STRT[20] is rectime but
|
||||
# STRT[8:10] / STRT[16:18] are device-specific scratch fields
|
||||
# that aren't reliable sample/pretrig counts).
|
||||
if cc:
|
||||
if ev.sample_rate is None and cc.sample_rate:
|
||||
ev.sample_rate = cc.sample_rate
|
||||
if cc.record_time:
|
||||
# record_time from compliance is authoritative — the
|
||||
# user-set value the device followed when recording.
|
||||
ev.rectime_seconds = cc.record_time
|
||||
# Derive total_samples from sample_rate × rectime when
|
||||
# we can; the STRT-derived value can land at a buffer-
|
||||
# offset rather than a sample count.
|
||||
if ev.sample_rate and ev.rectime_seconds:
|
||||
derived = int(round(ev.sample_rate * ev.rectime_seconds))
|
||||
if (ev.total_samples is None
|
||||
or ev.total_samples > derived * 2
|
||||
or ev.total_samples < derived // 4):
|
||||
ev.total_samples = derived
|
||||
geo_range = getattr(cc, "geo_range", None) if cc else None
|
||||
rec = _get_store().save(
|
||||
ev, serial=serial, a5_frames=a5_frames,
|
||||
geo_range=geo_range if geo_range is not None else "normal",
|
||||
)
|
||||
_get_db().insert_events(
|
||||
[ev],
|
||||
serial=serial,
|
||||
@@ -1412,34 +1448,50 @@ def db_event_blastware_file(event_id: str) -> FileResponse:
|
||||
@app.get("/db/events/{event_id}/waveform.json")
|
||||
def db_event_waveform_json(event_id: str) -> dict:
|
||||
"""
|
||||
Return the decoded raw_samples + metadata for a stored event in the
|
||||
same JSON shape as `/device/event/{index}/waveform`.
|
||||
Return the plot-ready JSON (`sfm.plot.v1`) for a stored event.
|
||||
|
||||
Reads the `.a5.pkl` sidecar from the store, rebuilds an Event in
|
||||
memory, runs the standard A5 decoders, and serialises the result.
|
||||
Resolution order (cheapest first):
|
||||
1. If `<filename>.h5` exists, serve it via `plot_json_from_hdf5`.
|
||||
Samples are already in physical units; no decode work needed.
|
||||
2. Else if `<filename>.a5.pkl` exists, replay the A5 decoders to
|
||||
rebuild an Event and serialise via `event_to_plot_json`.
|
||||
3. Else 404 — the event has no waveform data on disk.
|
||||
|
||||
The shape is identical regardless of source, so clients (the SFM
|
||||
webapp, Terra-View, etc.) consume the same `sfm.plot.v1` payload.
|
||||
"""
|
||||
row = _get_db().get_event(event_id)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail=f"Event {event_id} not found")
|
||||
serial = row.get("serial")
|
||||
filename = row.get("blastware_filename")
|
||||
a5_name = row.get("a5_pickle_filename")
|
||||
if not serial or not filename or not a5_name:
|
||||
if not serial or not filename:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"Event {event_id} has no event file in the store",
|
||||
)
|
||||
store = _get_store()
|
||||
|
||||
# Path 1: HDF5 (canonical clean format).
|
||||
h5_path = store.hdf5_path_for(serial, filename)
|
||||
if h5_path.exists():
|
||||
try:
|
||||
return event_hdf5.plot_json_from_hdf5(h5_path, event_id=event_id)
|
||||
except Exception as exc:
|
||||
log.warning("HDF5 read failed (%s); falling back to A5 path", exc)
|
||||
|
||||
# Path 2: A5 pickle replay.
|
||||
a5_frames = store.load_a5(serial, filename)
|
||||
if not a5_frames:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=(
|
||||
f"Event {event_id} has no A5 sidecar in the store. "
|
||||
"Re-download via the live endpoint to populate."
|
||||
f"Event {event_id} has no waveform data on disk "
|
||||
"(no .h5 and no .a5.pkl). Run the backfill script or "
|
||||
"re-download via the live endpoint to populate."
|
||||
),
|
||||
)
|
||||
a5_frames = _get_store().load_a5(serial, filename)
|
||||
if not a5_frames:
|
||||
raise HTTPException(
|
||||
status_code=410,
|
||||
detail=f"A5 sidecar missing or unreadable: {a5_name}",
|
||||
)
|
||||
|
||||
# Rebuild a minimal Event from DB fields, then decode A5 onto it.
|
||||
ev = Event(index=-1)
|
||||
try:
|
||||
_decode_a5_metadata_into(a5_frames, ev)
|
||||
@@ -1451,27 +1503,178 @@ def db_event_waveform_json(event_id: str) -> dict:
|
||||
log.error("db_event_waveform_json: waveform decode failed: %s", exc, exc_info=True)
|
||||
raise HTTPException(status_code=500, detail=f"Waveform decode failed: {exc}") from exc
|
||||
|
||||
raw = getattr(ev, "raw_samples", None) or {}
|
||||
samples_decoded = len(raw.get("Tran", []))
|
||||
return {
|
||||
"event_id": event_id,
|
||||
"serial": serial,
|
||||
"record_type": ev.record_type or row.get("record_type"),
|
||||
"timestamp": _serialise_timestamp(ev.timestamp) or row.get("timestamp"),
|
||||
"total_samples": ev.total_samples,
|
||||
"pretrig_samples": ev.pretrig_samples,
|
||||
"rectime_seconds": ev.rectime_seconds,
|
||||
"samples_decoded": samples_decoded,
|
||||
"sample_rate": ev.sample_rate or row.get("sample_rate"),
|
||||
"peak_values": _serialise_peak_values(ev.peak_values) or {
|
||||
"transverse": row.get("tran_ppv"),
|
||||
"vertical": row.get("vert_ppv"),
|
||||
"longitudinal": row.get("long_ppv"),
|
||||
"vector_sum": row.get("peak_vector_sum"),
|
||||
"mic": row.get("mic_ppv"),
|
||||
},
|
||||
"channels": raw,
|
||||
}
|
||||
# Carry over fields from the DB row when the A5 replay didn't fill them.
|
||||
if ev.sample_rate is None and row.get("sample_rate"):
|
||||
ev.sample_rate = row.get("sample_rate")
|
||||
|
||||
return event_hdf5.event_to_plot_json(
|
||||
ev, serial=serial, geo_range="normal", event_id=event_id,
|
||||
)
|
||||
|
||||
|
||||
# ── /db/events/{id}/sidecar — modern .sfm.json review/metadata accessors ──────
|
||||
|
||||
|
||||
class SidecarPatchBody(BaseModel):
|
||||
"""Body for PATCH /db/events/{id}/sidecar.
|
||||
|
||||
JSON-merge-patch semantics: only the keys you include get updated.
|
||||
`review` is the editable block for monthly-summary workflows
|
||||
(false_trigger flag, reviewer notes, etc.); `extensions` is the
|
||||
forward-compat namespace for vendor / future fields.
|
||||
"""
|
||||
review: Optional[dict] = None
|
||||
extensions: Optional[dict] = None
|
||||
|
||||
|
||||
@app.get("/db/events/{event_id}/sidecar")
|
||||
def db_event_sidecar(event_id: str) -> dict:
|
||||
"""
|
||||
Return the .sfm.json sidecar for a stored event. 404 if the event
|
||||
is unknown or has no sidecar in the store (events ingested before
|
||||
the sidecar feature landed will show this until backfilled).
|
||||
"""
|
||||
row = _get_db().get_event(event_id)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail=f"Event {event_id} not found")
|
||||
serial = row.get("serial")
|
||||
filename = row.get("blastware_filename")
|
||||
if not serial or not filename:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"Event {event_id} has no event file in the store",
|
||||
)
|
||||
sidecar = _get_store().load_sidecar(serial, filename)
|
||||
if sidecar is None:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=(
|
||||
f"No .sfm.json sidecar on disk for {filename}. "
|
||||
"Run scripts/backfill_sidecars.py to generate one."
|
||||
),
|
||||
)
|
||||
return sidecar
|
||||
|
||||
|
||||
@app.patch("/db/events/{event_id}/sidecar")
|
||||
def db_event_sidecar_patch(event_id: str, body: SidecarPatchBody) -> dict:
|
||||
"""
|
||||
JSON-merge-patch the sidecar's `review` and/or `extensions` blocks.
|
||||
|
||||
The sidecar JSON is the source of truth for review state. When
|
||||
`review.false_trigger` is updated, the SQL `events.false_trigger`
|
||||
column is kept in sync as a derived index for fast filtering.
|
||||
|
||||
Returns the new full sidecar. 404 if the event or sidecar is missing.
|
||||
"""
|
||||
row = _get_db().get_event(event_id)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail=f"Event {event_id} not found")
|
||||
serial = row.get("serial")
|
||||
filename = row.get("blastware_filename")
|
||||
if not serial or not filename:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"Event {event_id} has no event file in the store",
|
||||
)
|
||||
|
||||
if not (body.review or body.extensions):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="PATCH body must include `review` and/or `extensions`",
|
||||
)
|
||||
|
||||
new_sidecar = _get_store().patch_sidecar(
|
||||
serial, filename,
|
||||
review=body.review,
|
||||
extensions=body.extensions,
|
||||
)
|
||||
if new_sidecar is None:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"No .sfm.json sidecar on disk for {filename}",
|
||||
)
|
||||
|
||||
# Mirror false_trigger from review block into the SQL index column.
|
||||
if body.review is not None:
|
||||
_get_db().update_event_review(event_id, new_sidecar.get("review", {}))
|
||||
|
||||
return new_sidecar
|
||||
|
||||
|
||||
# ── /db/import/blastware_file — ingest BW-only event files ────────────────────
|
||||
|
||||
|
||||
@app.post("/db/import/blastware_file")
|
||||
async def db_import_blastware_file(
|
||||
files: list[UploadFile] = File(...),
|
||||
serial: Optional[str] = Query(None, description="Optional serial-number hint (e.g. BE11529); falls back to the BW filename's encoded prefix when omitted"),
|
||||
) -> dict:
|
||||
"""
|
||||
Multipart upload of one or more Blastware event file binaries
|
||||
(typically produced by Blastware's own ACH). For each file:
|
||||
|
||||
1. Parse the bytes via WaveformStore.save_imported_bw — produces
|
||||
a parsed Event + copies the file into the persistent store +
|
||||
writes a .sfm.json sidecar with source.kind = "bw-import".
|
||||
2. Upsert a row into `events` (dedup'd on serial+timestamp).
|
||||
|
||||
Response includes per-file outcomes so the caller can see which
|
||||
landed cleanly and which failed (e.g. malformed file, unknown
|
||||
serial, etc.).
|
||||
"""
|
||||
store = _get_store()
|
||||
db = _get_db()
|
||||
results: list[dict] = []
|
||||
|
||||
for upload in files:
|
||||
try:
|
||||
content = await upload.read()
|
||||
except Exception as exc:
|
||||
results.append({
|
||||
"filename": upload.filename, "status": "error",
|
||||
"detail": f"read failed: {exc}",
|
||||
})
|
||||
continue
|
||||
|
||||
try:
|
||||
ev, rec = store.save_imported_bw(
|
||||
content,
|
||||
source_path=Path(upload.filename or "imported.bw"),
|
||||
serial_hint=serial,
|
||||
)
|
||||
inserted, skipped = db.insert_events(
|
||||
[ev],
|
||||
serial=(serial or _serial_from_event(ev) or "UNKNOWN"),
|
||||
waveform_records={
|
||||
ev._waveform_key.hex(): rec
|
||||
if ev._waveform_key else None
|
||||
} if ev._waveform_key else None,
|
||||
)
|
||||
results.append({
|
||||
"filename": upload.filename,
|
||||
"status": "ok",
|
||||
"stored_filename": rec["filename"],
|
||||
"filesize": rec["filesize"],
|
||||
"sha256": rec["sha256"],
|
||||
"inserted": inserted,
|
||||
"skipped": skipped,
|
||||
})
|
||||
except Exception as exc:
|
||||
log.error("import failed for %s: %s", upload.filename, exc, exc_info=True)
|
||||
results.append({
|
||||
"filename": upload.filename, "status": "error",
|
||||
"detail": str(exc),
|
||||
})
|
||||
|
||||
return {"count": len(results), "results": results}
|
||||
|
||||
|
||||
def _serial_from_event(ev) -> Optional[str]:
|
||||
"""Fallback serial resolver — currently relies on the BW filename
|
||||
decoder via WaveformStore.save_imported_bw, so this is just a
|
||||
placeholder for future enhancement (e.g. inferring from project_info)."""
|
||||
return None
|
||||
|
||||
|
||||
@app.get("/db/units/{serial}/waveforms.zip")
|
||||
|
||||
+565
-60
@@ -639,6 +639,117 @@
|
||||
}
|
||||
.force-toggle.active .ft-dot { background: #f85149; box-shadow: 0 0 6px #f85149; }
|
||||
|
||||
/* ── Sidecar review modal ── */
|
||||
.sc-overlay {
|
||||
position: fixed; inset: 0;
|
||||
background: rgba(0,0,0,0.55);
|
||||
display: none;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
z-index: 100;
|
||||
}
|
||||
.sc-overlay.visible { display: flex; }
|
||||
.sc-modal {
|
||||
background: var(--surface2);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 8px;
|
||||
width: min(720px, 92vw);
|
||||
max-height: 88vh;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
box-shadow: 0 8px 32px rgba(0,0,0,0.5);
|
||||
}
|
||||
.sc-header {
|
||||
display: flex; align-items: center; justify-content: space-between;
|
||||
padding: 14px 18px;
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
.sc-header h3 {
|
||||
margin: 0; font-size: 14px; font-weight: 600;
|
||||
color: var(--text); font-family: monospace;
|
||||
}
|
||||
.sc-close {
|
||||
background: none; border: none; cursor: pointer;
|
||||
color: var(--text-mute); font-size: 18px; line-height: 1;
|
||||
padding: 4px 8px; border-radius: 4px;
|
||||
}
|
||||
.sc-close:hover { background: var(--surface); color: var(--text); }
|
||||
.sc-body {
|
||||
flex: 1; overflow-y: auto;
|
||||
padding: 16px 18px;
|
||||
display: flex; flex-direction: column; gap: 14px;
|
||||
}
|
||||
.sc-section {
|
||||
display: flex; flex-direction: column; gap: 6px;
|
||||
}
|
||||
.sc-section h4 {
|
||||
margin: 0 0 4px;
|
||||
font-size: 11px; font-weight: 600;
|
||||
color: var(--text-mute); text-transform: uppercase;
|
||||
letter-spacing: 0.6px;
|
||||
}
|
||||
.sc-grid {
|
||||
display: grid;
|
||||
grid-template-columns: 130px 1fr;
|
||||
gap: 4px 12px;
|
||||
font-size: 12px;
|
||||
}
|
||||
.sc-grid dt { color: var(--text-mute); }
|
||||
.sc-grid dd { margin: 0; color: var(--text); font-family: monospace; word-break: break-all; }
|
||||
.sc-row { display: flex; align-items: center; gap: 8px; font-size: 13px; }
|
||||
.sc-row label { color: var(--text-dim); }
|
||||
.sc-row input[type="checkbox"] { cursor: pointer; }
|
||||
.sc-row input[type="text"], .sc-body textarea {
|
||||
flex: 1;
|
||||
background: var(--bg);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 5px;
|
||||
padding: 6px 9px;
|
||||
font-size: 12px;
|
||||
color: var(--text);
|
||||
font-family: monospace;
|
||||
}
|
||||
.sc-body textarea {
|
||||
width: 100%;
|
||||
min-height: 80px;
|
||||
resize: vertical;
|
||||
font-family: inherit;
|
||||
}
|
||||
.sc-raw {
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 5px;
|
||||
background: var(--bg);
|
||||
}
|
||||
.sc-raw summary {
|
||||
padding: 6px 10px;
|
||||
cursor: pointer;
|
||||
font-size: 11px;
|
||||
color: var(--text-dim);
|
||||
user-select: none;
|
||||
}
|
||||
.sc-raw pre {
|
||||
margin: 0;
|
||||
padding: 8px 12px;
|
||||
max-height: 240px;
|
||||
overflow: auto;
|
||||
font-size: 11px;
|
||||
color: var(--text);
|
||||
border-top: 1px solid var(--border);
|
||||
}
|
||||
.sc-footer {
|
||||
display: flex; justify-content: flex-end; gap: 8px;
|
||||
padding: 12px 18px;
|
||||
border-top: 1px solid var(--border);
|
||||
}
|
||||
.sc-status {
|
||||
flex: 1; align-self: center;
|
||||
font-size: 11px; color: var(--text-mute);
|
||||
}
|
||||
.sc-status.error { color: #f85149; }
|
||||
.sc-status.ok { color: #56d364; }
|
||||
table.db-table tbody tr.clickable { cursor: pointer; }
|
||||
table.db-table tbody tr.clickable:hover { background: var(--surface2); }
|
||||
|
||||
/* ── Section containers ── */
|
||||
#section-live, #section-db {
|
||||
display: flex;
|
||||
@@ -806,6 +917,14 @@
|
||||
|
||||
<div class="event-toolbar">
|
||||
<button class="btn btn-ghost" id="load-btn" onclick="loadWaveform()" disabled>Load Waveform</button>
|
||||
<button class="btn btn-ghost" id="save-btn" onclick="saveEventToDb()" disabled
|
||||
title="Download the full waveform from the device and save it to the SFM database + waveform store. Honors the Force refresh toggle.">
|
||||
💾 Save to DB
|
||||
</button>
|
||||
<button class="btn btn-ghost" id="download-btn" onclick="downloadEventFile()" disabled
|
||||
title="Download the Blastware-format event file to your computer (also saves it to the server's database + store).">
|
||||
⬇ Download
|
||||
</button>
|
||||
<button class="btn btn-ghost" id="prev-btn" onclick="stepEvent(-1)" disabled>◀</button>
|
||||
<button class="btn btn-ghost" id="next-btn" onclick="stepEvent(+1)" disabled>▶</button>
|
||||
<div class="event-chips" id="event-chips"></div>
|
||||
@@ -1224,7 +1343,7 @@ let currentEvent = 0;
|
||||
let charts = {};
|
||||
let geoAdcScale = 6.206;
|
||||
const DBL_REF = 2.9e-9; // 20 µPa in psi — reference pressure for dBL
|
||||
const CHANNEL_COLORS = { Tran:'#58a6ff', Vert:'#3fb950', Long:'#d29922', Mic:'#bc8cff' };
|
||||
const CHANNEL_COLORS = { Tran:'#58a6ff', Vert:'#3fb950', Long:'#d29922', MicL:'#bc8cff' };
|
||||
|
||||
// ── Helpers ────────────────────────────────────────────────────────────────────
|
||||
function api() { return document.getElementById('api-base').value.replace(/\/$/, ''); }
|
||||
@@ -1355,9 +1474,11 @@ async function connectUnit() {
|
||||
|
||||
document.getElementById('device-bar').style.display = 'flex';
|
||||
document.getElementById('monitor-panel').style.display = 'flex';
|
||||
document.getElementById('load-btn').disabled = eventList.length === 0;
|
||||
document.getElementById('prev-btn').disabled = true;
|
||||
document.getElementById('next-btn').disabled = eventList.length <= 1;
|
||||
document.getElementById('load-btn').disabled = eventList.length === 0;
|
||||
document.getElementById('save-btn').disabled = eventList.length === 0;
|
||||
document.getElementById('download-btn').disabled = eventList.length === 0;
|
||||
document.getElementById('prev-btn').disabled = true;
|
||||
document.getElementById('next-btn').disabled = eventList.length <= 1;
|
||||
document.getElementById('cfg-read-btn').disabled = false;
|
||||
document.getElementById('cfg-write-btn').disabled = false;
|
||||
document.getElementById('ch-read-btn').disabled = false;
|
||||
@@ -1857,11 +1978,104 @@ async function loadWaveform() {
|
||||
document.getElementById('load-btn').disabled = false;
|
||||
}
|
||||
|
||||
// ── Persist current event to the SFM database + waveform store ──────────────
|
||||
//
|
||||
// Calls /device/event/{idx}/blastware_file, which on the server side:
|
||||
// 1. Downloads the full waveform from the device (5A bulk stream)
|
||||
// 2. Writes the Blastware-format event file into <db_dir>/waveforms/<serial>/
|
||||
// 3. Writes the .a5.pkl sidecar next to it (so the file can be regenerated)
|
||||
// 4. Upserts a row into seismo_relay.db `events` table (dedup'd on serial+timestamp)
|
||||
//
|
||||
// We discard the response body — the side effects are what we want. The
|
||||
// filename comes back in the Content-Disposition header for confirmation.
|
||||
async function saveEventToDb() {
|
||||
if (!devHost()) { setStatus('Enter device host first.', 'error'); return; }
|
||||
const idx = currentEvent;
|
||||
const btn = document.getElementById('save-btn');
|
||||
btn.disabled = true;
|
||||
const orig = btn.textContent;
|
||||
btn.textContent = '⏳ Saving…';
|
||||
setStatus(`Downloading event #${idx} and saving to DB…`, 'loading');
|
||||
|
||||
try {
|
||||
const r = await fetch(`${api()}/device/event/${idx}/blastware_file?${deviceParams()}`);
|
||||
if (!r.ok) {
|
||||
const e = await r.json().catch(() => ({}));
|
||||
throw new Error(e.detail || r.statusText);
|
||||
}
|
||||
// Pull the body to completion so the connection releases promptly,
|
||||
// then drop it on the floor — we just want the server-side persist.
|
||||
await r.blob();
|
||||
const filename = parseFilenameFromContentDisposition(r.headers.get('Content-Disposition'))
|
||||
|| `event ${idx}`;
|
||||
setStatus(`Saved ${filename} to database + waveform store`, 'ok');
|
||||
} catch (e) {
|
||||
setStatus(`Save error: ${e.message}`, 'error');
|
||||
} finally {
|
||||
btn.disabled = false;
|
||||
btn.textContent = orig;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Download the event file to the user's computer ──────────────────────────
|
||||
//
|
||||
// Uses a transient anchor + click trick so the browser surfaces its native
|
||||
// "Save As" / Downloads behaviour. Same backend endpoint as Save to DB —
|
||||
// the file is also persisted to the server store as a side effect.
|
||||
function downloadEventFile() {
|
||||
if (!devHost()) { setStatus('Enter device host first.', 'error'); return; }
|
||||
const idx = currentEvent;
|
||||
const url = `${api()}/device/event/${idx}/blastware_file?${deviceParams()}`;
|
||||
setStatus(`Downloading event #${idx}…`, 'loading');
|
||||
// Hidden iframe avoids navigating away from the SPA. FastAPI's FileResponse
|
||||
// sets Content-Disposition: attachment so the browser saves rather than displays.
|
||||
const a = document.createElement('a');
|
||||
a.href = url;
|
||||
a.style.display = 'none';
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
document.body.removeChild(a);
|
||||
// We can't reliably detect when the browser finishes downloading; show a
|
||||
// soft confirmation immediately. Errors will surface as a download failure
|
||||
// dialog from the browser itself.
|
||||
setTimeout(() => setStatus(`Download started for event #${idx} (also saved server-side)`, 'ok'), 250);
|
||||
}
|
||||
|
||||
function parseFilenameFromContentDisposition(header) {
|
||||
if (!header) return null;
|
||||
// RFC 6266: `attachment; filename="M529LKIQ.7M0W"` (or filename*=UTF-8''…)
|
||||
const m = /filename\*?=(?:UTF-8'')?["']?([^"';]+)["']?/i.exec(header);
|
||||
return m ? decodeURIComponent(m[1]) : null;
|
||||
}
|
||||
|
||||
// renderWaveform consumes the `sfm.plot.v1` JSON shape:
|
||||
// {
|
||||
// schema: "sfm.plot.v1",
|
||||
// time_axis: { sample_rate, pretrig_samples, t0_ms, dt_ms, n_samples, ... },
|
||||
// channels: { Tran|Vert|Long|MicL: { unit, values, peak, peak_t_ms } },
|
||||
// geo_range, geo_full_scale_ips, trigger_ms, peak_values, ...
|
||||
// }
|
||||
//
|
||||
// All sample arrays are already in PHYSICAL UNITS (in/s for geo, psi for
|
||||
// mic) — the server applied the right scaling for the unit's geo_range.
|
||||
// The viewer used to multiply ADC ints by `geoAdcScale / 32767` here,
|
||||
// which silently scaled every plot ~38% too low because `geoAdcScale` is
|
||||
// the in/s-per-V hardware constant, not the ADC-counts-to-velocity
|
||||
// factor. No scaling happens client-side now.
|
||||
function renderWaveform(data) {
|
||||
const sr = data.sample_rate || 1024;
|
||||
const pretrig = data.pretrig_samples || 0;
|
||||
const decoded = data.samples_decoded || 0;
|
||||
const total = data.total_samples || decoded;
|
||||
// Backward-compat shim: if we ever get the legacy shape from a stale
|
||||
// cache, normalise it on the client so the viewer still works.
|
||||
if (!data.schema && data.channels && Array.isArray(data.channels.Tran)) {
|
||||
data = _legacyWaveformToPlotV1(data);
|
||||
}
|
||||
|
||||
const t = data.time_axis || {};
|
||||
const sr = t.sample_rate || 1024;
|
||||
const pretrig = t.pretrig_samples || 0;
|
||||
const total = t.total_samples || t.n_samples || 0;
|
||||
const decoded = t.n_samples || 0;
|
||||
const t0 = t.t0_ms ?? -(pretrig / sr * 1000);
|
||||
const dt = t.dt_ms ?? (1000 / sr);
|
||||
const channels = data.channels || {};
|
||||
|
||||
// Status bar
|
||||
@@ -1869,70 +2083,83 @@ function renderWaveform(data) {
|
||||
bar.innerHTML = '';
|
||||
bar.className = 'ok';
|
||||
const ts = data.timestamp;
|
||||
bar.textContent = ts ? `Event #${data.index} — ${ts.display} ` : `Event #${data.index} `;
|
||||
// Title prefers `index` (live device, 0-based slot on the unit) and
|
||||
// falls back to event_id (DB lookup) when index is absent.
|
||||
const eventLabel = (data.index != null) ? `#${data.index}` : (data.event_id || '');
|
||||
bar.textContent = ts ? `Event ${eventLabel} — ${ts} ` : `Event ${eventLabel} `;
|
||||
addPill(`${data.record_type || '?'}`);
|
||||
addPill(`${sr} sps`);
|
||||
addPill(`${decoded.toLocaleString()} / ${total.toLocaleString()} samples`);
|
||||
addPill(`pretrig ${pretrig}`);
|
||||
addPill(`${data.rectime_seconds ?? '?'} s`);
|
||||
addPill(`${t.rectime_seconds ?? '?'} s`);
|
||||
if (data.geo_range) addPill(`geo: ${data.geo_range} (${data.geo_full_scale_ips} in/s FS)`);
|
||||
|
||||
// Any record_type starting with "Waveform" is a viewable triggered
|
||||
// event (the timestamp-header byte layout varies across firmware but
|
||||
// doesn't change the sample stream). Only block when there's actually
|
||||
// no waveform payload to plot.
|
||||
const isWaveformLike = !!(data.record_type || '').match(/^Waveform/i);
|
||||
if (decoded === 0) {
|
||||
document.getElementById('empty-state').style.display = 'flex';
|
||||
document.getElementById('empty-state').querySelector('p').textContent =
|
||||
data.record_type === 'Waveform'
|
||||
isWaveformLike
|
||||
? 'No samples decoded — check server logs'
|
||||
: `Record type "${data.record_type}" — waveform not supported yet`;
|
||||
: `Record type "${data.record_type}" — not a waveform event`;
|
||||
document.getElementById('charts').style.display = 'none';
|
||||
Object.values(charts).forEach(c => c.destroy()); charts = {};
|
||||
return;
|
||||
}
|
||||
|
||||
const times = Array.from({length: decoded}, (_, i) => ((i - pretrig) / sr * 1000).toFixed(2));
|
||||
// Time axis: explicit ms values from t0_ms + i*dt_ms. More precise
|
||||
// than the old (i - pretrig) / sr * 1000 since dt_ms came from the
|
||||
// server with full float precision.
|
||||
const times = Array.from({length: decoded}, (_, i) => (t0 + i * dt).toFixed(2));
|
||||
document.getElementById('empty-state').style.display = 'none';
|
||||
const chartsDiv = document.getElementById('charts');
|
||||
chartsDiv.style.display = 'flex';
|
||||
chartsDiv.innerHTML = '';
|
||||
Object.values(charts).forEach(c => c.destroy()); charts = {};
|
||||
|
||||
const micPeakPsi = data.peak_values?.micl_psi ?? null;
|
||||
|
||||
for (const [ch, color] of Object.entries(CHANNEL_COLORS)) {
|
||||
const samples = channels[ch];
|
||||
if (!samples || samples.length === 0) continue;
|
||||
const chData = channels[ch];
|
||||
if (!chData || !chData.values || chData.values.length === 0) continue;
|
||||
|
||||
const isGeo = ch !== 'Mic';
|
||||
let plotData, peakLabel, yUnit, ttFmt, tickFmt;
|
||||
const plotData = chData.values;
|
||||
const unit = chData.unit || (ch === 'MicL' ? 'psi' : 'in/s');
|
||||
const peak = chData.peak;
|
||||
const peakTms = chData.peak_t_ms;
|
||||
|
||||
if (isGeo) {
|
||||
const scale = geoAdcScale / 32767;
|
||||
plotData = samples.map(s => s * scale);
|
||||
// Use the device-recorded peak from the 0C waveform record — authoritative
|
||||
// and matches Blastware. Computing from raw samples can catch rogue
|
||||
// near-full-scale values from decoding artifacts.
|
||||
const peakKey = { Tran:'tran_in_s', Vert:'vert_in_s', Long:'long_in_s' }[ch];
|
||||
const devicePeak = data.peak_values?.[peakKey] ?? null;
|
||||
peakLabel = devicePeak != null ? `${devicePeak.toFixed(5)} in/s` : `${Math.max(...plotData.map(Math.abs)).toFixed(5)} in/s`;
|
||||
yUnit = 'in/s';
|
||||
ttFmt = v => `${ch}: ${v.toFixed(5)} in/s`;
|
||||
tickFmt = v => v.toFixed(4);
|
||||
let peakLabel, ttFmt, tickFmt;
|
||||
if (unit === 'psi') {
|
||||
const peakDbl = (peak != null && peak > 0)
|
||||
? 20 * Math.log10(peak / DBL_REF) : -Infinity;
|
||||
peakLabel = `${peakDbl.toFixed(1)} dBL (${peak != null ? peak.toExponential(2) : '—'} psi)`;
|
||||
ttFmt = v => `${v.toExponential(3)} psi`;
|
||||
tickFmt = v => v.toExponential(1);
|
||||
} else {
|
||||
const peakCounts = Math.max(...samples.map(Math.abs));
|
||||
const micScale = (micPeakPsi !== null && peakCounts > 0) ? Math.abs(micPeakPsi) / peakCounts : 1.0;
|
||||
plotData = samples.map(s => s * micScale);
|
||||
const peakPsi = Math.max(...plotData.map(Math.abs));
|
||||
const peakDbl = peakPsi > 0 ? 20 * Math.log10(peakPsi / DBL_REF) : -Infinity;
|
||||
peakLabel = `${peakDbl.toFixed(1)} dBL`;
|
||||
yUnit = 'psi';
|
||||
ttFmt = v => `${v.toExponential(3)} psi`;
|
||||
tickFmt = v => v.toExponential(1);
|
||||
peakLabel = peak != null ? `${peak.toFixed(5)} in/s` : '—';
|
||||
ttFmt = v => `${ch}: ${v.toFixed(5)} in/s`;
|
||||
tickFmt = v => v.toFixed(4);
|
||||
}
|
||||
|
||||
// Downsample for display when the chart would otherwise have to
|
||||
// rasterise tens of thousands of points. Uses every-Nth — fine for
|
||||
// monthly-summary glance work; analysis tools should use the .h5 file.
|
||||
const MAX_PTS = 4000;
|
||||
let rTimes = times, rData = plotData;
|
||||
let rTimes = times, rData = plotData, peakPlotIdx = -1;
|
||||
if (plotData.length > MAX_PTS) {
|
||||
const step = Math.ceil(plotData.length / MAX_PTS);
|
||||
rTimes = times.filter((_, i) => i % step === 0);
|
||||
rData = plotData.filter((_, i) => i % step === 0);
|
||||
// Try to keep the peak sample from being downsampled away.
|
||||
if (peakTms != null) {
|
||||
const exactIdx = Math.round((peakTms - t0) / dt);
|
||||
if (exactIdx >= 0 && exactIdx < plotData.length) {
|
||||
peakPlotIdx = Math.floor(exactIdx / step);
|
||||
}
|
||||
}
|
||||
} else if (peakTms != null) {
|
||||
peakPlotIdx = Math.round((peakTms - t0) / dt);
|
||||
}
|
||||
|
||||
const wrap = document.createElement('div');
|
||||
@@ -1960,27 +2187,94 @@ function renderWaveform(data) {
|
||||
},
|
||||
scales: {
|
||||
x: { type: 'category', ticks: { color:'#484f58', maxTicksLimit:10, maxRotation:0, callback:(v,i) => rTimes[i]+' ms' }, grid: { color:'#21262d' } },
|
||||
y: { ticks: { color:'#484f58', maxTicksLimit:5, callback: v => tickFmt(v) }, grid: { color:'#21262d' }, title: { display:true, text:yUnit, color:'#484f58', font:{size:10} } },
|
||||
y: { ticks: { color:'#484f58', maxTicksLimit:5, callback: v => tickFmt(v) }, grid: { color:'#21262d' }, title: { display:true, text:unit, color:'#484f58', font:{size:10} } },
|
||||
},
|
||||
},
|
||||
plugins: [{
|
||||
id: 'triggerLine',
|
||||
id: 'triggerAndPeakMarkers',
|
||||
afterDraw(chart) {
|
||||
const zeroIdx = rTimes.findIndex(t => parseFloat(t) >= 0);
|
||||
if (zeroIdx < 0) return;
|
||||
const { ctx, scales: {x, y} } = chart;
|
||||
const px = x.getPixelForValue(zeroIdx);
|
||||
ctx.save();
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(px, y.top); ctx.lineTo(px, y.bottom);
|
||||
ctx.strokeStyle = 'rgba(248,81,73,0.7)'; ctx.lineWidth = 1.5;
|
||||
ctx.setLineDash([4, 3]); ctx.stroke(); ctx.restore();
|
||||
// Trigger line at t = trigger_ms (typically 0).
|
||||
const triggerMs = data.trigger_ms ?? 0;
|
||||
const zeroIdx = rTimes.findIndex(s => parseFloat(s) >= triggerMs);
|
||||
if (zeroIdx >= 0) {
|
||||
const px = x.getPixelForValue(zeroIdx);
|
||||
ctx.save();
|
||||
ctx.beginPath();
|
||||
ctx.moveTo(px, y.top); ctx.lineTo(px, y.bottom);
|
||||
ctx.strokeStyle = 'rgba(248,81,73,0.7)'; ctx.lineWidth = 1.5;
|
||||
ctx.setLineDash([4, 3]); ctx.stroke(); ctx.restore();
|
||||
}
|
||||
// Peak marker (dot at the channel's peak sample).
|
||||
if (peakPlotIdx >= 0 && peakPlotIdx < rData.length) {
|
||||
const px = x.getPixelForValue(peakPlotIdx);
|
||||
const py = y.getPixelForValue(rData[peakPlotIdx]);
|
||||
ctx.save();
|
||||
ctx.beginPath();
|
||||
ctx.arc(px, py, 3.2, 0, Math.PI * 2);
|
||||
ctx.fillStyle = color;
|
||||
ctx.strokeStyle = '#0d1117';
|
||||
ctx.lineWidth = 1.5;
|
||||
ctx.fill(); ctx.stroke();
|
||||
ctx.restore();
|
||||
}
|
||||
},
|
||||
}],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// One-time normaliser for the legacy /device/event/{idx}/waveform shape
|
||||
// (samples as int16 ADC counts in `channels.{ch}: [...]`). Bridges the
|
||||
// gap if a stale cache or non-upgraded server returns the old format.
|
||||
function _legacyWaveformToPlotV1(data) {
|
||||
const sr = data.sample_rate || 1024;
|
||||
const pretrig = data.pretrig_samples || 0;
|
||||
const decoded = data.samples_decoded || 0;
|
||||
const total = data.total_samples || decoded;
|
||||
const dt = 1000 / sr;
|
||||
const t0 = -pretrig * dt;
|
||||
|
||||
// Apply the CORRECT scale: 10 in/s full-scale for Normal range.
|
||||
const geoFs = 10.0;
|
||||
const geoScale = geoFs / 32768;
|
||||
const ch = data.channels || {};
|
||||
const micPeak = data.peak_values?.micl_psi ?? null;
|
||||
const micPeakCounts = (ch.MicL || ch.Mic || []).reduce((m, v) => Math.max(m, Math.abs(v)), 0);
|
||||
const micScale = (micPeak != null && micPeakCounts > 0) ? micPeak / micPeakCounts : 1.0;
|
||||
|
||||
const mkGeo = (counts) => {
|
||||
if (!counts || !counts.length) return [];
|
||||
return counts.map(c => c * geoScale);
|
||||
};
|
||||
const mkMic = (counts) => {
|
||||
if (!counts || !counts.length) return [];
|
||||
return counts.map(c => c * micScale);
|
||||
};
|
||||
|
||||
return {
|
||||
schema: 'sfm.plot.v1',
|
||||
event_id: data.event_id || null,
|
||||
serial: data.serial || '',
|
||||
timestamp: data.timestamp?.display || data.timestamp || '',
|
||||
record_type: data.record_type,
|
||||
waveform_key: null,
|
||||
time_axis: {
|
||||
sample_rate: sr, pretrig_samples: pretrig, total_samples: total,
|
||||
n_samples: decoded, t0_ms: t0, dt_ms: dt,
|
||||
rectime_seconds: data.rectime_seconds || 0,
|
||||
},
|
||||
geo_range: 'normal', geo_full_scale_ips: geoFs, trigger_ms: 0,
|
||||
channels: {
|
||||
Tran: { unit:'in/s', values: mkGeo(ch.Tran), peak: data.peak_values?.tran_in_s ?? null, peak_t_ms: null },
|
||||
Vert: { unit:'in/s', values: mkGeo(ch.Vert), peak: data.peak_values?.vert_in_s ?? null, peak_t_ms: null },
|
||||
Long: { unit:'in/s', values: mkGeo(ch.Long), peak: data.peak_values?.long_in_s ?? null, peak_t_ms: null },
|
||||
MicL: { unit:'psi', values: mkMic(ch.MicL || ch.Mic), peak: micPeak, peak_t_ms: null },
|
||||
},
|
||||
peak_values: data.peak_values || {},
|
||||
};
|
||||
}
|
||||
|
||||
// ── DB tabs ────────────────────────────────────────────────────────────────────
|
||||
let histLoaded = false;
|
||||
let unitsLoaded = false;
|
||||
@@ -2082,7 +2376,9 @@ async function loadHistory() {
|
||||
for (const ev of events) {
|
||||
const tr = document.createElement('tr');
|
||||
const pvs = ev.peak_vector_sum;
|
||||
const maxPPV = Math.max(ev.tran_ppv ?? 0, ev.vert_ppv ?? 0, ev.long_ppv ?? 0);
|
||||
tr.classList.add('clickable');
|
||||
tr.title = 'Click to review (open sidecar editor)';
|
||||
tr.dataset.eventId = ev.id;
|
||||
tr.innerHTML = `
|
||||
<td>${_fmtTs(ev.timestamp)}</td>
|
||||
<td class="td-key">${ev.serial ?? '—'}</td>
|
||||
@@ -2095,24 +2391,157 @@ async function loadHistory() {
|
||||
<td class="td-text">${ev.client ?? '—'}</td>
|
||||
<td class="td-dim">${ev.record_type ?? '—'}</td>
|
||||
<td class="td-dim" style="font-size:10px">${ev.waveform_key ?? '—'}</td>
|
||||
<td>${ev.false_trigger ? '<span class="ft-badge">FALSE</span>' : `<button class="ft-toggle-btn" onclick="toggleFalseTrigger(${ev.id}, this)" title="Flag as false trigger">Flag</button>`}</td>
|
||||
<td>${ev.false_trigger ? '<span class="ft-badge">FALSE</span>' : ''}</td>
|
||||
`;
|
||||
tr.addEventListener('click', () => openSidecarModal(ev.id));
|
||||
tbody.appendChild(tr);
|
||||
}
|
||||
}
|
||||
|
||||
async function toggleFalseTrigger(id, btn) {
|
||||
btn.disabled = true;
|
||||
// ── Sidecar review modal ───────────────────────────────────────────────────────
|
||||
//
|
||||
// Opens on row click in the History table. Loads the .sfm.json sidecar
|
||||
// for the event via GET /db/events/{id}/sidecar, lets the user toggle
|
||||
// false_trigger / edit notes / set reviewer, and saves via PATCH on the
|
||||
// same URL. This mirrors the workflow used by the monthly vibration
|
||||
// summary process — most of the rich review UX lives in Terra-View;
|
||||
// this is the SFM-standalone equivalent for testing / direct edits.
|
||||
|
||||
let _scCurrentEventId = null;
|
||||
let _scCurrentSidecar = null;
|
||||
|
||||
async function openSidecarModal(eventId) {
|
||||
_scCurrentEventId = eventId;
|
||||
_scCurrentSidecar = null;
|
||||
document.getElementById('sc-status').textContent = 'Loading sidecar…';
|
||||
document.getElementById('sc-status').className = 'sc-status';
|
||||
document.getElementById('sc-overlay').classList.add('visible');
|
||||
// Reset edit fields
|
||||
document.getElementById('sc-edit-ft').checked = false;
|
||||
document.getElementById('sc-edit-reviewer').value = '';
|
||||
document.getElementById('sc-edit-notes').value = '';
|
||||
|
||||
try {
|
||||
const r = await fetch(`${api()}/db/events/${id}/false_trigger?value=true`, { method: 'PATCH' });
|
||||
if (!r.ok) throw new Error(r.statusText);
|
||||
btn.outerHTML = '<span class="ft-badge">FALSE</span>';
|
||||
const r = await fetch(`${api()}/db/events/${eventId}/sidecar`);
|
||||
if (!r.ok) {
|
||||
const e = await r.json().catch(() => ({}));
|
||||
throw new Error(e.detail || r.statusText);
|
||||
}
|
||||
const data = await r.json();
|
||||
_scCurrentSidecar = data;
|
||||
_renderSidecar(data);
|
||||
document.getElementById('sc-status').textContent = '';
|
||||
} catch (e) {
|
||||
btn.disabled = false;
|
||||
alert(`Failed to flag: ${e.message}`);
|
||||
document.getElementById('sc-status').className = 'sc-status error';
|
||||
document.getElementById('sc-status').textContent = `Load failed: ${e.message}`;
|
||||
}
|
||||
}
|
||||
|
||||
function _renderSidecar(data) {
|
||||
const ev = data.event || {};
|
||||
const pv = data.peak_values || {};
|
||||
const pi = data.project_info || {};
|
||||
const bw = data.blastware || {};
|
||||
const src = data.source || {};
|
||||
const rev = data.review || {};
|
||||
|
||||
document.getElementById('sc-title').textContent = `Event — ${bw.filename || ev.waveform_key || 'unknown'}`;
|
||||
|
||||
const fmtPpv = v => (v == null ? '—' : Number(v).toFixed(5) + ' in/s');
|
||||
const fmtMic = v => {
|
||||
if (v == null || v <= 0) return '—';
|
||||
const dbl = 20 * Math.log10(v / DBL_REF);
|
||||
return `${dbl.toFixed(1)} dBL (${v.toExponential(2)} psi)`;
|
||||
};
|
||||
|
||||
document.getElementById('sc-f-serial').textContent = ev.serial || '—';
|
||||
document.getElementById('sc-f-ts').textContent = ev.timestamp || '—';
|
||||
document.getElementById('sc-f-rt').textContent = ev.record_type || '—';
|
||||
document.getElementById('sc-f-sr').textContent = (ev.sample_rate ?? '—') + (ev.sample_rate ? ' sps' : '');
|
||||
document.getElementById('sc-f-key').textContent = ev.waveform_key || '—';
|
||||
|
||||
document.getElementById('sc-f-tran').textContent = fmtPpv(pv.transverse);
|
||||
document.getElementById('sc-f-vert').textContent = fmtPpv(pv.vertical);
|
||||
document.getElementById('sc-f-long').textContent = fmtPpv(pv.longitudinal);
|
||||
document.getElementById('sc-f-pvs').textContent = fmtPpv(pv.vector_sum);
|
||||
document.getElementById('sc-f-mic').textContent = fmtMic(pv.mic_psi);
|
||||
|
||||
document.getElementById('sc-f-project').textContent = pi.project || '—';
|
||||
document.getElementById('sc-f-client').textContent = pi.client || '—';
|
||||
document.getElementById('sc-f-operator').textContent = pi.operator || '—';
|
||||
document.getElementById('sc-f-loc').textContent = pi.sensor_location || '—';
|
||||
|
||||
document.getElementById('sc-f-bw').textContent = bw.filename || '—';
|
||||
document.getElementById('sc-f-bwsize').textContent = bw.filesize != null ? `${bw.filesize} bytes` : '—';
|
||||
document.getElementById('sc-f-sha').textContent = bw.sha256 || '—';
|
||||
document.getElementById('sc-f-src').textContent = src.kind || '—';
|
||||
document.getElementById('sc-f-cap').textContent = src.captured_at || '—';
|
||||
|
||||
document.getElementById('sc-edit-ft').checked = !!rev.false_trigger;
|
||||
document.getElementById('sc-edit-reviewer').value = rev.reviewer || '';
|
||||
document.getElementById('sc-edit-notes').value = rev.notes || '';
|
||||
|
||||
document.getElementById('sc-raw-json').textContent = JSON.stringify(data, null, 2);
|
||||
}
|
||||
|
||||
function closeSidecarModal() {
|
||||
document.getElementById('sc-overlay').classList.remove('visible');
|
||||
_scCurrentEventId = null;
|
||||
_scCurrentSidecar = null;
|
||||
}
|
||||
|
||||
function onSidecarOverlayClick(e) {
|
||||
// Click on the dimmed backdrop (but NOT on the modal itself) closes.
|
||||
if (e.target.id === 'sc-overlay') closeSidecarModal();
|
||||
}
|
||||
|
||||
async function saveSidecarReview() {
|
||||
if (!_scCurrentEventId) return;
|
||||
const btn = document.getElementById('sc-save-btn');
|
||||
const status = document.getElementById('sc-status');
|
||||
btn.disabled = true;
|
||||
status.className = 'sc-status';
|
||||
status.textContent = 'Saving…';
|
||||
|
||||
const review = {
|
||||
false_trigger: document.getElementById('sc-edit-ft').checked,
|
||||
reviewer: document.getElementById('sc-edit-reviewer').value.trim() || null,
|
||||
notes: document.getElementById('sc-edit-notes').value,
|
||||
};
|
||||
|
||||
try {
|
||||
const r = await fetch(`${api()}/db/events/${_scCurrentEventId}/sidecar`, {
|
||||
method: 'PATCH',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ review }),
|
||||
});
|
||||
if (!r.ok) {
|
||||
const e = await r.json().catch(() => ({}));
|
||||
throw new Error(e.detail || r.statusText);
|
||||
}
|
||||
const updated = await r.json();
|
||||
_scCurrentSidecar = updated;
|
||||
_renderSidecar(updated);
|
||||
status.className = 'sc-status ok';
|
||||
status.textContent = 'Saved.';
|
||||
// Refresh the History table so the false_trigger badge reflects the change.
|
||||
if (typeof loadHistory === 'function') loadHistory();
|
||||
setTimeout(closeSidecarModal, 600);
|
||||
} catch (e) {
|
||||
status.className = 'sc-status error';
|
||||
status.textContent = `Save failed: ${e.message}`;
|
||||
} finally {
|
||||
btn.disabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
// Esc closes the modal.
|
||||
document.addEventListener('keydown', (e) => {
|
||||
if (e.key === 'Escape' && document.getElementById('sc-overlay').classList.contains('visible')) {
|
||||
closeSidecarModal();
|
||||
}
|
||||
});
|
||||
|
||||
// ── Units tab ──────────────────────────────────────────────────────────────────
|
||||
async function loadUnits() {
|
||||
unitsLoaded = true;
|
||||
@@ -2274,5 +2703,81 @@ document.getElementById('api-base').value = window.location.origin;
|
||||
document.getElementById(id)?.addEventListener('keydown', e => { if (e.key === 'Enter') connectUnit(); });
|
||||
});
|
||||
</script>
|
||||
|
||||
<!-- ════════════════════════════════════════════════════════════════
|
||||
Sidecar review modal (Database events table → row click)
|
||||
═══════════════════════════════════════════════════════════════════ -->
|
||||
<div class="sc-overlay" id="sc-overlay" onclick="onSidecarOverlayClick(event)">
|
||||
<div class="sc-modal" id="sc-modal">
|
||||
<div class="sc-header">
|
||||
<h3 id="sc-title">Event</h3>
|
||||
<button class="sc-close" onclick="closeSidecarModal()">×</button>
|
||||
</div>
|
||||
<div class="sc-body">
|
||||
<div class="sc-section">
|
||||
<h4>Event</h4>
|
||||
<dl class="sc-grid">
|
||||
<dt>Serial</dt> <dd id="sc-f-serial">—</dd>
|
||||
<dt>Timestamp</dt> <dd id="sc-f-ts">—</dd>
|
||||
<dt>Record type</dt> <dd id="sc-f-rt">—</dd>
|
||||
<dt>Sample rate</dt> <dd id="sc-f-sr">—</dd>
|
||||
<dt>Waveform key</dt> <dd id="sc-f-key">—</dd>
|
||||
</dl>
|
||||
</div>
|
||||
<div class="sc-section">
|
||||
<h4>Peaks</h4>
|
||||
<dl class="sc-grid">
|
||||
<dt>Tran</dt> <dd id="sc-f-tran">—</dd>
|
||||
<dt>Vert</dt> <dd id="sc-f-vert">—</dd>
|
||||
<dt>Long</dt> <dd id="sc-f-long">—</dd>
|
||||
<dt>PVS</dt> <dd id="sc-f-pvs">—</dd>
|
||||
<dt>Mic</dt> <dd id="sc-f-mic">—</dd>
|
||||
</dl>
|
||||
</div>
|
||||
<div class="sc-section">
|
||||
<h4>Project</h4>
|
||||
<dl class="sc-grid">
|
||||
<dt>Project</dt> <dd id="sc-f-project">—</dd>
|
||||
<dt>Client</dt> <dd id="sc-f-client">—</dd>
|
||||
<dt>Operator</dt> <dd id="sc-f-operator">—</dd>
|
||||
<dt>Location</dt> <dd id="sc-f-loc">—</dd>
|
||||
</dl>
|
||||
</div>
|
||||
<div class="sc-section">
|
||||
<h4>Source / files</h4>
|
||||
<dl class="sc-grid">
|
||||
<dt>BW filename</dt> <dd id="sc-f-bw">—</dd>
|
||||
<dt>BW filesize</dt> <dd id="sc-f-bwsize">—</dd>
|
||||
<dt>BW sha256</dt> <dd id="sc-f-sha">—</dd>
|
||||
<dt>Source kind</dt> <dd id="sc-f-src">—</dd>
|
||||
<dt>Captured at</dt> <dd id="sc-f-cap">—</dd>
|
||||
</dl>
|
||||
</div>
|
||||
<div class="sc-section">
|
||||
<h4>Review (editable)</h4>
|
||||
<div class="sc-row">
|
||||
<input type="checkbox" id="sc-edit-ft" />
|
||||
<label for="sc-edit-ft">False trigger</label>
|
||||
</div>
|
||||
<div class="sc-row">
|
||||
<label for="sc-edit-reviewer" style="min-width:60px">Reviewer</label>
|
||||
<input type="text" id="sc-edit-reviewer" placeholder="e.g. brian" />
|
||||
</div>
|
||||
<label for="sc-edit-notes" style="font-size:11px;color:var(--text-mute)">Notes</label>
|
||||
<textarea id="sc-edit-notes" placeholder="e.g. truck thump near sensor 14:23 — false trigger"></textarea>
|
||||
</div>
|
||||
<details class="sc-raw">
|
||||
<summary>Raw sidecar JSON (read-only peek)</summary>
|
||||
<pre id="sc-raw-json"></pre>
|
||||
</details>
|
||||
</div>
|
||||
<div class="sc-footer">
|
||||
<span class="sc-status" id="sc-status"></span>
|
||||
<button class="btn btn-ghost" onclick="closeSidecarModal()">Cancel</button>
|
||||
<button class="btn" id="sc-save-btn" onclick="saveSidecarReview()">Save</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
+297
-22
@@ -1,34 +1,46 @@
|
||||
"""
|
||||
sfm/waveform_store.py — On-disk store for Blastware-format event files.
|
||||
|
||||
Layout (flat per-serial):
|
||||
Layout (flat per-serial, four files per event):
|
||||
|
||||
<root>/<serial>/<filename> ← event file (Blastware-readable binary)
|
||||
<root>/<serial>/<filename> ← event file (BW-readable binary)
|
||||
<root>/<serial>/<filename>.a5.pkl ← pickled list of A5 S3Frame dicts
|
||||
<root>/<serial>/<filename>.h5 ← clean waveform arrays (HDF5)
|
||||
<root>/<serial>/<filename>.sfm.json ← modern sidecar (peaks, project,
|
||||
review state, extensions)
|
||||
|
||||
`<filename>` is whatever `minimateplus.blastware_file.blastware_filename`
|
||||
produces for the event. The extension is NOT a fixed type tag — it encodes
|
||||
the event timestamp (`AB0T` format: 2-char base-36 of `total_seconds %
|
||||
1296`, literal `0`, then `W`=Full Waveform / `H`=Full Histogram for ACH
|
||||
downloads, or 3-char `AB0` for direct/manual downloads). Every event's
|
||||
filename therefore contains its own timestamp + record-type fingerprint and
|
||||
collisions across the same physical event don't occur.
|
||||
produces for the event. The extension is NOT a fixed type tag — it
|
||||
encodes the event timestamp (`AB0T` format).
|
||||
|
||||
The `.a5.pkl` sidecar lets the event file be regenerated later if the
|
||||
encoder changes — captures the raw 5A frame stream as serializable dicts so
|
||||
the schema isn't tied to the `S3Frame` dataclass layout.
|
||||
Roles:
|
||||
- BW binary: what Blastware reads. Untouched. The user-facing review
|
||||
waveform viewer.
|
||||
- .a5.pkl: regenerative source. Lets the BW binary be rebuilt
|
||||
byte-for-byte if the encoder changes. Never delete.
|
||||
- .h5: clean per-channel waveform arrays in physical units (in/s for
|
||||
geo, psi for mic) plus event metadata. Canonical format for
|
||||
downstream analysis tools and the `/device/event/{idx}/waveform`
|
||||
endpoint's plot-JSON output.
|
||||
- .sfm.json: small, queryable metadata + review state. SQL
|
||||
`events.false_trigger` is a derived index kept in sync via
|
||||
`patch_sidecar()`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
import logging
|
||||
import pickle
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from minimateplus import event_file_io
|
||||
from minimateplus.blastware_file import blastware_filename, write_blastware_file
|
||||
from minimateplus.framing import S3Frame
|
||||
from minimateplus.models import Event
|
||||
from sfm import event_hdf5
|
||||
|
||||
log = logging.getLogger("sfm.waveform_store")
|
||||
|
||||
@@ -80,10 +92,22 @@ class WaveformStore:
|
||||
return d
|
||||
|
||||
def paths_for(self, serial: str, filename: str) -> tuple[Path, Path]:
|
||||
"""Return (blastware_path, a5_pickle_path) for a given serial+filename."""
|
||||
"""Return (blastware_path, a5_pickle_path) for a given serial+filename.
|
||||
|
||||
For the sidecar path use `sidecar_path_for()` — kept separate so
|
||||
existing callers don't need to unpack a 3-tuple.
|
||||
"""
|
||||
d = self._serial_dir(serial)
|
||||
return d / filename, d / f"{filename}.a5.pkl"
|
||||
|
||||
def sidecar_path_for(self, serial: str, filename: str) -> Path:
|
||||
"""Return absolute path to the .sfm.json sidecar for a given event."""
|
||||
return self._serial_dir(serial) / f"{filename}.sfm.json"
|
||||
|
||||
def hdf5_path_for(self, serial: str, filename: str) -> Path:
|
||||
"""Return absolute path to the .h5 clean-waveform file for a given event."""
|
||||
return self._serial_dir(serial) / f"{filename}.h5"
|
||||
|
||||
def open_blastware(self, serial: str, filename: str) -> Optional[Path]:
|
||||
"""Return absolute path to an existing event file or None."""
|
||||
bw_path, _ = self.paths_for(serial, filename)
|
||||
@@ -96,23 +120,43 @@ class WaveformStore:
|
||||
ev: Event,
|
||||
serial: str,
|
||||
a5_frames: list[S3Frame],
|
||||
*,
|
||||
source_kind: str = "sfm-live",
|
||||
geo_range = "normal",
|
||||
) -> dict:
|
||||
"""
|
||||
Write the event file and its .a5.pkl sidecar for one event.
|
||||
Write all four event-file artifacts for one event:
|
||||
- <filename> BW binary
|
||||
- <filename>.a5.pkl raw A5 frame pickle
|
||||
- <filename>.h5 clean waveform (HDF5)
|
||||
- <filename>.sfm.json modern sidecar (metadata + review)
|
||||
|
||||
Returns a record dict suitable for persisting alongside the DB row:
|
||||
|
||||
{
|
||||
"filename": "M529LKIQ.7M0W",
|
||||
"filesize": 8708,
|
||||
"sha256": "a1b2c3...",
|
||||
"a5_pickle_filename": "M529LKIQ.7M0W.a5.pkl",
|
||||
"hdf5_filename": "M529LKIQ.7M0W.h5",
|
||||
"sidecar_filename": "M529LKIQ.7M0W.sfm.json",
|
||||
}
|
||||
|
||||
The exact extension is timestamp-encoded per event (see
|
||||
`minimateplus.blastware_file.blastware_filename`).
|
||||
`source_kind` flows into `sidecar.source.kind` — callers should
|
||||
pass "sfm-live" (default) for the live endpoint and "sfm-ach" for
|
||||
the ACH ingestion path. BW-imported events use save_imported_bw()
|
||||
instead.
|
||||
|
||||
Idempotent: if the event file already exists, it is overwritten with
|
||||
the freshly-encoded version (same bytes for the same a5_frames).
|
||||
`geo_range` controls the ADC-counts → in/s scaling in the HDF5
|
||||
file ("normal" = 10 in/s FS, "sensitive" = 1.25 in/s FS).
|
||||
Defaults to "normal" — callers with compliance-config access
|
||||
should pass the actual unit setting so the saved samples are in
|
||||
the right units.
|
||||
|
||||
Idempotent: if the event file already exists, it is overwritten
|
||||
with the freshly-encoded version (same bytes for the same
|
||||
a5_frames) and the sidecar's review block is preserved across
|
||||
re-saves.
|
||||
"""
|
||||
if not a5_frames:
|
||||
raise ValueError("WaveformStore.save: a5_frames is empty")
|
||||
@@ -121,17 +165,18 @@ class WaveformStore:
|
||||
|
||||
filename = blastware_filename(ev, serial)
|
||||
bw_path, a5_path = self.paths_for(serial, filename)
|
||||
sidecar_path = self.sidecar_path_for(serial, filename)
|
||||
hdf5_path = self.hdf5_path_for(serial, filename)
|
||||
|
||||
# 1. encode the event file
|
||||
# Delete any stale file at this path so partial writes never leak
|
||||
# trailing bytes from a previous larger file (matches the live
|
||||
# endpoint's defensive unlink).
|
||||
# 1. encode the event file (defensive unlink prevents trailing-byte
|
||||
# leaks from a previous larger file on synced/odd filesystems).
|
||||
try:
|
||||
bw_path.unlink()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
write_blastware_file(ev, a5_frames, bw_path)
|
||||
filesize = bw_path.stat().st_size
|
||||
sha256 = event_file_io.file_sha256(bw_path)
|
||||
|
||||
# 2. write the .a5.pkl sidecar
|
||||
try:
|
||||
@@ -145,14 +190,176 @@ class WaveformStore:
|
||||
with a5_path.open("wb") as fp:
|
||||
pickle.dump(payload, fp, protocol=pickle.HIGHEST_PROTOCOL)
|
||||
|
||||
# 3. write the .h5 clean-waveform file (samples in physical units).
|
||||
# Best-effort: a write failure shouldn't sink the rest of the save
|
||||
# (the HDF5 can be regenerated later from the .a5.pkl).
|
||||
hdf5_filename: Optional[str] = None
|
||||
try:
|
||||
event_hdf5.write_event_hdf5(
|
||||
hdf5_path, ev,
|
||||
serial=serial,
|
||||
geo_range=geo_range,
|
||||
source_kind=source_kind,
|
||||
)
|
||||
hdf5_filename = hdf5_path.name
|
||||
except Exception as exc:
|
||||
log.warning(
|
||||
"save: HDF5 write failed for %s: %s — continuing without .h5",
|
||||
hdf5_path, exc,
|
||||
)
|
||||
|
||||
# 4. write the .sfm.json sidecar. Preserve any existing review
|
||||
# block + extensions across re-saves so user edits aren't lost
|
||||
# when the same event is re-downloaded (e.g. via Force refresh).
|
||||
existing_review = None
|
||||
existing_extensions = None
|
||||
if sidecar_path.exists():
|
||||
try:
|
||||
old = event_file_io.read_sidecar(sidecar_path)
|
||||
existing_review = old.get("review")
|
||||
existing_extensions = old.get("extensions")
|
||||
except Exception as exc:
|
||||
log.warning(
|
||||
"save: existing sidecar at %s unreadable (%s); overwriting",
|
||||
sidecar_path, exc,
|
||||
)
|
||||
|
||||
sidecar = event_file_io.event_to_sidecar_dict(
|
||||
ev,
|
||||
serial=serial,
|
||||
blastware_filename=filename,
|
||||
blastware_filesize=filesize,
|
||||
blastware_sha256=sha256,
|
||||
source_kind=source_kind,
|
||||
a5_pickle_filename=a5_path.name,
|
||||
review=existing_review,
|
||||
extensions=existing_extensions,
|
||||
)
|
||||
event_file_io.write_sidecar(sidecar_path, sidecar)
|
||||
|
||||
log.info(
|
||||
"WaveformStore.save serial=%s filename=%s filesize=%d frames=%d",
|
||||
"WaveformStore.save serial=%s filename=%s filesize=%d frames=%d "
|
||||
"h5=%s sidecar=%s",
|
||||
serial, filename, filesize, len(a5_frames),
|
||||
hdf5_filename or "(skipped)", sidecar_path.name,
|
||||
)
|
||||
return {
|
||||
"filename": filename,
|
||||
"filesize": filesize,
|
||||
"sha256": sha256,
|
||||
"a5_pickle_filename": a5_path.name,
|
||||
"hdf5_filename": hdf5_filename,
|
||||
"sidecar_filename": sidecar_path.name,
|
||||
}
|
||||
|
||||
def save_imported_bw(
|
||||
self,
|
||||
bw_bytes: bytes,
|
||||
source_path: Path,
|
||||
*,
|
||||
serial_hint: Optional[str] = None,
|
||||
) -> tuple[Event, dict]:
|
||||
"""
|
||||
Ingest a Blastware event file produced by an external tool
|
||||
(Blastware's own ACH, manual download, etc.) where the source A5
|
||||
frames aren't available.
|
||||
|
||||
Workflow:
|
||||
1. Parse the bytes via event_file_io.read_blastware_file (writes
|
||||
a temp file to do that, since the parser takes a path).
|
||||
2. Resolve serial from BW filename (`<P><serial3>...`) or use
|
||||
serial_hint. Falls back to "UNKNOWN".
|
||||
3. Copy the BW bytes verbatim into <root>/<serial>/<filename>.
|
||||
4. Write the .sfm.json sidecar with source.kind = "bw-import"
|
||||
and a5_pickle_filename = None. Does NOT write a .a5.pkl
|
||||
(no A5 source available; byte-for-byte regeneration not
|
||||
possible — the on-disk BW file IS the byte-for-byte source).
|
||||
|
||||
Returns (event, record_dict) so callers can both insert into
|
||||
SeismoDb and surface the parsed Event.
|
||||
"""
|
||||
# Stash the bytes to a temp path so read_blastware_file (path-based)
|
||||
# can parse without us duplicating its logic.
|
||||
import tempfile
|
||||
with tempfile.NamedTemporaryFile(suffix=".bw", delete=False) as tmp:
|
||||
tmp.write(bw_bytes)
|
||||
tmp_path = Path(tmp.name)
|
||||
try:
|
||||
ev = event_file_io.read_blastware_file(tmp_path)
|
||||
finally:
|
||||
try:
|
||||
tmp_path.unlink()
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
# Resolve serial. blastware_filename derives a 4-char prefix from
|
||||
# the numeric serial (e.g. BE11529 → M529); we go the other way
|
||||
# via the source filename if a hint wasn't given.
|
||||
serial = serial_hint or _serial_from_bw_filename(source_path.name) or "UNKNOWN"
|
||||
|
||||
# Use the source filename verbatim — it already encodes timestamp
|
||||
# + record type per BW's AB0T scheme, and we want to preserve it
|
||||
# so the file BW knows about can be opened back in BW.
|
||||
filename = source_path.name
|
||||
bw_path = self._serial_dir(serial) / filename
|
||||
|
||||
# 1. copy bytes
|
||||
bw_path.write_bytes(bw_bytes)
|
||||
filesize = bw_path.stat().st_size
|
||||
sha256 = event_file_io.file_sha256(bw_path)
|
||||
|
||||
# 2. write the .h5 clean-waveform file from the parsed Event.
|
||||
# Note: peaks here are computed from raw samples (the BW file
|
||||
# doesn't carry the device-authoritative 0C peaks). Best-effort.
|
||||
hdf5_path = self.hdf5_path_for(serial, filename)
|
||||
hdf5_filename: Optional[str] = None
|
||||
try:
|
||||
event_hdf5.write_event_hdf5(
|
||||
hdf5_path, ev,
|
||||
serial=serial,
|
||||
geo_range="normal", # BW file doesn't carry the range; assume Normal
|
||||
source_kind="bw-import",
|
||||
)
|
||||
hdf5_filename = hdf5_path.name
|
||||
except Exception as exc:
|
||||
log.warning(
|
||||
"save_imported_bw: HDF5 write failed for %s: %s — continuing",
|
||||
hdf5_path, exc,
|
||||
)
|
||||
|
||||
# 3. write sidecar with source.kind = bw-import
|
||||
sidecar_path = self.sidecar_path_for(serial, filename)
|
||||
existing_review = None
|
||||
if sidecar_path.exists():
|
||||
try:
|
||||
existing_review = event_file_io.read_sidecar(sidecar_path).get("review")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
sidecar = event_file_io.event_to_sidecar_dict(
|
||||
ev,
|
||||
serial=serial,
|
||||
blastware_filename=filename,
|
||||
blastware_filesize=filesize,
|
||||
blastware_sha256=sha256,
|
||||
source_kind="bw-import",
|
||||
a5_pickle_filename=None,
|
||||
review=existing_review,
|
||||
)
|
||||
event_file_io.write_sidecar(sidecar_path, sidecar)
|
||||
|
||||
log.info(
|
||||
"WaveformStore.save_imported_bw serial=%s filename=%s filesize=%d "
|
||||
"h5=%s (no .a5.pkl — A5 source unavailable for BW-imported files)",
|
||||
serial, filename, filesize, hdf5_filename or "(skipped)",
|
||||
)
|
||||
return ev, {
|
||||
"filename": filename,
|
||||
"filesize": filesize,
|
||||
"sha256": sha256,
|
||||
"a5_pickle_filename": None,
|
||||
"hdf5_filename": hdf5_filename,
|
||||
"sidecar_filename": sidecar_path.name,
|
||||
}
|
||||
|
||||
def load_a5(self, serial: str, filename: str) -> Optional[list[S3Frame]]:
|
||||
@@ -169,3 +376,71 @@ class WaveformStore:
|
||||
log.warning("WaveformStore.load_a5: malformed sidecar at %s", a5_path)
|
||||
return None
|
||||
return [_dict_to_frame(d) for d in payload["frames"]]
|
||||
|
||||
# ── modern .sfm.json sidecar accessors ──────────────────────────────────────
|
||||
|
||||
def load_sidecar(self, serial: str, filename: str) -> Optional[dict]:
|
||||
"""Return the parsed .sfm.json sidecar dict, or None if missing."""
|
||||
path = self.sidecar_path_for(serial, filename)
|
||||
if not path.exists():
|
||||
return None
|
||||
try:
|
||||
return event_file_io.read_sidecar(path)
|
||||
except Exception as exc:
|
||||
log.warning("load_sidecar: failed to read %s: %s", path, exc)
|
||||
return None
|
||||
|
||||
def patch_sidecar(
|
||||
self,
|
||||
serial: str,
|
||||
filename: str,
|
||||
*,
|
||||
review: Optional[dict] = None,
|
||||
extensions: Optional[dict] = None,
|
||||
reviewer_now: bool = True,
|
||||
) -> Optional[dict]:
|
||||
"""
|
||||
JSON-merge-patch the .sfm.json sidecar's review/extensions blocks.
|
||||
Returns the new full dict, or None if the sidecar doesn't exist.
|
||||
"""
|
||||
path = self.sidecar_path_for(serial, filename)
|
||||
if not path.exists():
|
||||
return None
|
||||
return event_file_io.patch_sidecar(
|
||||
path,
|
||||
review=review,
|
||||
extensions=extensions,
|
||||
reviewer_now=reviewer_now,
|
||||
)
|
||||
|
||||
|
||||
# ── helpers ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
def _serial_from_bw_filename(name: str) -> Optional[str]:
|
||||
"""
|
||||
Reverse of `blastware_filename`'s serial-prefix encoding.
|
||||
|
||||
BW filename format (V10.72): `<P><serial3><stem4>.<ext>`
|
||||
where P = chr(ord('B') + floor(serial // 1000))
|
||||
and serial3 = f"{serial % 1000:03d}".
|
||||
|
||||
Examples (from CLAUDE.md verification archive):
|
||||
P036... → BE14036 H907... → BE6907
|
||||
M529... → BE11529 T003... → BE18003
|
||||
|
||||
Returns the inferred BE-prefix serial (e.g. "BE11529") or None when
|
||||
the filename doesn't match the expected pattern.
|
||||
"""
|
||||
if not name:
|
||||
return None
|
||||
# First letter encodes the thousands group; next 3 chars encode the
|
||||
# last 3 digits of the serial.
|
||||
base = name.split(".", 1)[0]
|
||||
if len(base) < 4 or not base[0].isalpha() or not base[1:4].isdigit():
|
||||
return None
|
||||
prefix_letter = base[0].upper()
|
||||
if prefix_letter < "B":
|
||||
return None
|
||||
thousands = ord(prefix_letter) - ord("B")
|
||||
serial_num = thousands * 1000 + int(base[1:4])
|
||||
return f"BE{serial_num}"
|
||||
|
||||
Reference in New Issue
Block a user