From 8d3cdba1b5ceb3408919ca7fd0d2fc9b13d73f0d Mon Sep 17 00:00:00 2001 From: serversdown Date: Wed, 16 Sep 2026 06:49:55 +0000 Subject: [PATCH] feat(h5): standardize sensor-check into the .h5 (schema v2); SFM reads it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make the sensor self-check a first-class part of the standardized decoded event so SFM stops decoding it at report time — device-agnostic, per the store's decoder→standardized-.h5→SFM model. * Event gains a `sensor_check` field; both decoders attach the traces where they set raw_samples — series-3 in event_file_io.read_blastware_file (minimateplus.sensor_check), series-4 in waveform_store's IDF path (micromate.sensor_check). Covers ingest and backfill (both re-decode). * event_hdf5 bumps schema_version 1→2 and writes an optional /sensor_check group (raw counts, int32, per channel present). read_event_hdf5 returns it; plot_json_from_hdf5 carries it as a top-level key. Old v1 files still read cleanly (no group → None), so nothing breaks before the backfill. * gather_report_data reads sensor_check_waveforms from the .h5 and drops the report-time series-3 decode — the report no longer reaches into a decoder, and a series-4 event now lights up the same strip automatically. Stored as raw counts (a shape diagnostic, rendered fit-to-box): the per-series count scale differs and a physical mic unit is ill-defined, so conversion would add complexity for no display benefit — easy to add later if a numeric use appears. Tests: .h5 roundtrip + backward-compat + plot_json + real series-3 decode attaches to the Event. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01YDXjZCr4RqT2U3QvMDhgzf --- minimateplus/event_file_io.py | 5 ++ minimateplus/models.py | 9 ++++ sfm/event_hdf5.py | 48 ++++++++++++++++-- sfm/report_pdf.py | 27 +++++----- sfm/waveform_store.py | 5 ++ tests/test_event_hdf5_sensor_check.py | 71 +++++++++++++++++++++++++++ 6 files changed, 145 insertions(+), 20 deletions(-) create mode 100644 tests/test_event_hdf5_sensor_check.py diff --git a/minimateplus/event_file_io.py b/minimateplus/event_file_io.py index 519cf4b..ccba632 100644 --- a/minimateplus/event_file_io.py +++ b/minimateplus/event_file_io.py @@ -960,6 +960,11 @@ def read_blastware_file(path: Union[str, Path]) -> Event: project=project, client=client, operator=user, sensor_location=seisloc, ) ev.raw_samples = samples + # Sensor self-check traces from the binary's trailing block (waveform + # events only; returns {} for histograms / when absent). Carried on the + # Event so the .h5 writer persists them device-agnostically. + from minimateplus.sensor_check import decode_sensor_check + ev.sensor_check = decode_sensor_check(raw) or None # Only compute peaks from samples when we actually have samples. # For events the codec couldn't decode (histogram-mode bodies, until # the §7.6.2 histogram codec is wired in), samples is an empty dict diff --git a/minimateplus/models.py b/minimateplus/models.py index 48fd326..9e7865d 100644 --- a/minimateplus/models.py +++ b/minimateplus/models.py @@ -544,6 +544,15 @@ class Event: pretrig_samples: Optional[int] = None # from STRT record: pre-trigger sample count rectime_seconds: Optional[int] = None # from STRT record: record duration (seconds) + # Sensor self-check traces keyed by channel label — the short diagnostic + # waveforms the unit records when it pulses each sensor before monitoring + # (geophone ring-downs + a mic pulse train). Decoded from the binary by + # the per-series decoder (minimateplus.sensor_check / micromate.sensor_check) + # and carried here so the .h5 writer can persist them device-agnostically. + # Raw ADC counts; the source series' scale differs but the trace is a + # shape diagnostic (rendered fit-to-box). None when absent. + sensor_check: Optional[dict] = None # {"Tran": [...], ..., "MicL": [...]} + # ── Debug / introspection ───────────────────────────────────────────────── # Raw 210-byte waveform record bytes, set when debug mode is active. # Exposed by the SFM server via ?debug=true so field layouts can be verified. diff --git a/sfm/event_hdf5.py b/sfm/event_hdf5.py index a25b34d..c63d993 100644 --- a/sfm/event_hdf5.py +++ b/sfm/event_hdf5.py @@ -12,8 +12,11 @@ Layout written to `.h5`: ├─ samples_int16/ (optional) │ ├─ Tran (int16, raw ADC counts) shape: (N,) │ └─ ... per channel (only when present in the source) + ├─ sensor_check/ (optional, schema v2+) + │ ├─ Tran (int32, raw counts) shape: (M,) M ≪ N + │ └─ ... per channel present in the source (MicL absent on 3-channel units) └─ root attrs (event metadata): - schema_version int = 1 + schema_version int = 2 kind str = "sfm.event.hdf5" serial str waveform_key str (8-hex) @@ -64,7 +67,7 @@ from minimateplus.models import Event log = logging.getLogger(__name__) -SCHEMA_VERSION = 1 +SCHEMA_VERSION = 2 # v2 adds the optional /sensor_check group HDF5_KIND = "sfm.event.hdf5" # Geophone full-scale velocity per range (in/s). Confirmed in CLAUDE.md @@ -270,6 +273,22 @@ def write_event_hdf5( ) igrp.attrs["mic_psi_per_count"] = float(mic_factor) + # /sensor_check — optional short diagnostic self-check traces (schema + # v2+). Raw ADC counts (a shape diagnostic; the per-series count scale + # differs, and the renderer fits each trace to its box). Only channels + # the decoder found are written — 3-channel units carry no MicL. + sc = event.sensor_check or {} + if sc: + scgrp = f.create_group("sensor_check") + for ch in ("Tran", "Vert", "Long", "MicL"): + vals = sc.get(ch) + if vals: + scgrp.create_dataset( + ch, data=np.asarray(vals, dtype=np.int32), + compression="gzip", compression_opts=4, shuffle=True, + ) + scgrp.attrs["units"] = "raw_counts" + import os os.replace(tmp, path) @@ -334,6 +353,16 @@ def read_event_hdf5(path: Union[str, Path]) -> dict: if mic_attr is not None: mic_psi = float(mic_attr) + # /sensor_check — optional (schema v2+); absent on older files. + sensor_check = None + scgrp = f.get("sensor_check") + if scgrp is not None: + sensor_check = {} + for ch in ("Tran", "Vert", "Long", "MicL"): + ds = scgrp.get(ch) + if ds is not None: + sensor_check[ch] = np.asarray(ds[()]) + return { "schema_version": sv, "kind": attrs.get("kind"), @@ -341,6 +370,7 @@ def read_event_hdf5(path: Union[str, Path]) -> dict: "samples": samples, "samples_int16": samples_int16, "mic_psi_per_count": mic_psi, + "sensor_check": sensor_check, } @@ -431,11 +461,16 @@ def plot_json_from_hdf5( event_id: Optional[str] = None, index: Optional[int] = None, ) -> dict: - """Build a `sfm.plot.v1` JSON dict from a stored .h5 file.""" + """Build a `sfm.plot.v1` JSON dict from a stored .h5 file. + + The dict also carries a top-level ``sensor_check`` key (the raw self-check + traces as ``{ch: [int]}``, or None) beyond the plot schema, so report + generation can read the traces from the same single .h5 load. + """ data = read_event_hdf5(path) a = data["attrs"] s = data["samples"] - return _build_plot_dict( + out = _build_plot_dict( n_samples=len(s["Tran"]) if "Tran" in s else 0, sample_rate=int(a.get("sample_rate", 1024) or 1024), pretrig_samples=int(a.get("pretrig_samples", 0) or 0), @@ -463,6 +498,11 @@ def plot_json_from_hdf5( event_id=event_id, index=index, ) + scd = data.get("sensor_check") + out["sensor_check"] = ( + {ch: v.tolist() for ch, v in scd.items()} if scd else None + ) + return out def _build_plot_dict( diff --git a/sfm/report_pdf.py b/sfm/report_pdf.py index 794c836..d4d60e7 100644 --- a/sfm/report_pdf.py +++ b/sfm/report_pdf.py @@ -121,9 +121,11 @@ class ReportData: t0_ms: Optional[float] = None dt_ms: Optional[float] = None - # Sensor self-check traces — {ch: [samples]} in raw decode units, decoded - # from the binary's trailing block (see minimateplus.sensor_check). The - # little waveforms BW draws in its "Sensor Check" strip. Empty when absent. + # Sensor self-check traces — {ch: [samples]} in raw counts, read from the + # standardized .h5 (/sensor_check group, schema v2+) where the per-series + # decoder stored them at ingest. The little diagnostic waveforms BW draws + # in its "Sensor Check" strip. Empty when absent (pre-v2 .h5, histogram, + # or 3-channel unit's MicL). sensor_check_waveforms: dict = field(default_factory=dict) # Record-type discriminator @@ -294,22 +296,15 @@ def gather_report_data( rd.pretrig_samples = ta.get("pretrig_samples") rd.t0_ms = ta.get("t0_ms") rd.dt_ms = ta.get("dt_ms") + # Sensor self-check traces — read from the standardized .h5 (schema + # v2+). Device-agnostic: whichever decoder produced the event + # stored them at ingest, so SFM reads them here without knowing or + # caring about the source instrument series. Empty on pre-v2 files + # (until backfilled) and on 3-channel / histogram events. + rd.sensor_check_waveforms = wf.get("sensor_check") or {} except Exception as exc: log.warning("gather_report_data: hdf5 read failed: %s", exc) - # ── Sensor self-check traces — decoded from the retained raw binary ── - # The .h5 holds only the main waveform; the sensor-check traces live in the - # binary's trailing block, so decode them straight from the kept BW file. - # Waveform events only (histograms have no sensor-check strip). - if not rd.is_histogram: - try: - from minimateplus.sensor_check import decode_sensor_check - bw_path, _a5 = store.paths_for(serial, filename) - if bw_path.exists(): - rd.sensor_check_waveforms = decode_sensor_check(bw_path.read_bytes()) - except Exception as exc: - log.warning("gather_report_data: sensor-check decode failed: %s", exc) - # ── Histogram aggregation ── # Codec emits ~N per-block samples (typically 1/sec); BW reports # one bar per configured interval (1 min / 5 min / etc.). When diff --git a/sfm/waveform_store.py b/sfm/waveform_store.py index d38a425..8c2e26c 100644 --- a/sfm/waveform_store.py +++ b/sfm/waveform_store.py @@ -662,6 +662,11 @@ class WaveformStore: ev.raw_samples = idf_samples n_samples = max((len(idf_samples.get(ch, [])) for ch in ("Tran", "Vert", "Long", "MicL")), default=0) ev.total_samples = ev.total_samples or n_samples + # Sensor self-check traces from the IDFW fixed header (waveform + # events only; {} on histograms / when absent). Carried on the + # bridged Event so the .h5 writer persists them like series-3. + from micromate.sensor_check import decode_idf_sensor_check + ev.sensor_check = decode_idf_sensor_check(idf_bytes) or None # For IDFH histograms there are no per-sample waveform arrays — the # device stores one peak ADC count per interval per channel. Synthesise diff --git a/tests/test_event_hdf5_sensor_check.py b/tests/test_event_hdf5_sensor_check.py new file mode 100644 index 0000000..fda0dc8 --- /dev/null +++ b/tests/test_event_hdf5_sensor_check.py @@ -0,0 +1,71 @@ +"""The event .h5 carries the sensor self-check traces (schema v2). + +The sensor check is decoded by the per-series decoder and attached to the +standardized Event, so the .h5 writer persists it device-agnostically and SFM +reads it back without knowing which instrument produced it. Old v1 files (no +sensor_check group) must still read cleanly. +""" +import tempfile +from pathlib import Path + +import numpy as np + +from minimateplus.models import Event +from minimateplus.event_file_io import read_blastware_file +from sfm import event_hdf5 + +S3_FIX = Path(__file__).parent / "fixtures" / "fft-oracle-2026-09-14" / "N844LQHB.ZT0W" + + +def _write(ev, **kw): + d = Path(tempfile.mkdtemp()) + p = d / "e.h5" + event_hdf5.write_event_hdf5(p, ev, serial="BE12844", **kw) + return p + + +def test_sensor_check_roundtrips_through_hdf5(): + ev = Event(index=0) + ev.raw_samples = {"Tran": [1, 2, -3], "Vert": [0, 1], "Long": [2], "MicL": [5, -5]} + ev.sample_rate = 1024 + sc = {"Tran": [0, -990, -500, -100], "Vert": [0, -980, -480], + "Long": [0, -986, -470], "MicL": [0, -1800, 1800, -1800]} + ev.sensor_check = sc + + r = event_hdf5.read_event_hdf5(_write(ev)) + assert r["schema_version"] == 2 + assert set(r["sensor_check"]) == {"Tran", "Vert", "Long", "MicL"} + for ch, vals in sc.items(): + assert r["sensor_check"][ch].tolist() == vals + + +def test_plot_json_carries_sensor_check(): + ev = Event(index=0) + ev.raw_samples = {"Tran": [1, 2, 3]} + ev.sample_rate = 1024 + ev.sensor_check = {"Tran": [0, -990, -500], "Vert": [0, -980], + "Long": [0, -986]} # 3-channel: no MicL + pj = event_hdf5.plot_json_from_hdf5(_write(ev)) + assert pj["sensor_check"] is not None + assert "MicL" not in pj["sensor_check"] + assert pj["sensor_check"]["Tran"] == [0, -990, -500] + + +def test_event_without_sensor_check_still_reads_as_v2(): + ev = Event(index=0) + ev.raw_samples = {"Tran": [1, 2, 3]} + ev.sample_rate = 1024 + r = event_hdf5.read_event_hdf5(_write(ev)) + assert r["schema_version"] == 2 + assert r["sensor_check"] is None + assert event_hdf5.plot_json_from_hdf5(_write(ev))["sensor_check"] is None + + +def test_series3_decode_populates_event_sensor_check(): + # The real series-3 decoder attaches the traces to the Event, so the + # ingest/backfill .h5 write picks them up with no extra plumbing. + ev = read_blastware_file(S3_FIX) + assert ev.sensor_check is not None + assert set(ev.sensor_check) == {"Tran", "Vert", "Long", "MicL"} + tran = np.asarray(ev.sensor_check["Tran"], dtype=float) + assert tran.min() < -800 # the geophone ring-down deflection