From eb92b13aac6fc367db2e75b91f73927a421ad04b Mon Sep 17 00:00:00 2001 From: serversdown Date: Fri, 21 Aug 2026 21:32:59 +0000 Subject: [PATCH 01/30] =?UTF-8?q?docs(plan):=20waveform-shape=20FT=20detec?= =?UTF-8?q?tion=20=E2=80=94=20Phase=20A=20(seismo-relay)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 7-task TDD plan: shape DSP module (crest factor + points-near-peak), shape_* columns + auto-migrate, insert_events persistence, ingest population in the save paths, backfill script, /db/events exposure + v0.24.0 bump. Phase B (terra-view scoring/UI/review) gets its own plan once this feed is live. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01YDXjZCr4RqT2U3QvMDhgzf --- ...026-08-21-waveform-ft-detection-phase-a.md | 628 ++++++++++++++++++ 1 file changed, 628 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-21-waveform-ft-detection-phase-a.md diff --git a/docs/superpowers/plans/2026-08-21-waveform-ft-detection-phase-a.md b/docs/superpowers/plans/2026-08-21-waveform-ft-detection-phase-a.md new file mode 100644 index 0000000..86c0fd9 --- /dev/null +++ b/docs/superpowers/plans/2026-08-21-waveform-ft-detection-phase-a.md @@ -0,0 +1,628 @@ +# Waveform-shape FT detection — Phase A (seismo-relay) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Compute per-event waveform-shape metrics (crest factor + points-near-peak) in SFM and store them as `events` columns, populated at ingest and by a backfill script, so Terra-View can read them from `/db/events`. + +**Architecture:** A pure DSP module (`sfm/shape_metrics.py`) turns decoded `.h5` samples into shape metrics. New nullable `events.shape_*` columns are added via the existing `_migrate` ADD COLUMN loop. The three `WaveformStore.save*` paths compute shape from the just-written `.h5` and hand it to `insert_events`; a backfill script does the same over existing events. Mirrors exactly how per-channel ZC frequency was added. + +**Tech Stack:** Python 3.10, numpy, h5py, sqlite3 (raw), pytest. seismo-relay venv: `/home/serversdown/seismo-relay/.venv/bin/python3`. + +## Global Constraints + +- Metrics are read from the `.h5` `samples/{Tran,Vert,Long}` float32 arrays (physical in/s). The measured channel is the max-|peak| geophone channel. +- All new columns are nullable; histogram records and events without usable samples store NULL (Terra-View falls back to cheap signals). Legacy rows stay valid. +- Crest factor = `max(|x|) / rms(x)`; near-peak count = number of samples with `|x| ≥ 0.5·peak`. Threshold `0.5` is a module constant so calibration can tune it. +- No manual migration: columns are added in `SeismoDb._migrate`, run at `SeismoDb()` construction. +- `/db/events` needs no change — it returns all columns via `SELECT *` (verify with a test). +- Run tests with `/home/serversdown/seismo-relay/.venv/bin/python3 -m pytest`. + +--- + +### Task 1: Shape DSP module + +**Files:** +- Create: `sfm/shape_metrics.py` +- Test: `tests/test_shape_metrics.py` + +**Interfaces:** +- Produces: + - `channel_shape(x) -> dict | None` — `{"crest_factor": float, "near_peak_count": int, "sample_count": int}` or None for unusable input (size < 2, flat, all-zero). + - `shape_from_samples(chans: dict[str, ArrayLike]) -> dict | None` — picks the max-peak geophone channel; returns `{"crest_factor","near_peak_count","sample_count","axis"}` or None. + - `shape_from_h5(path) -> dict | None` — reads `samples/{Tran,Vert,Long}` and delegates to `shape_from_samples`; None on any read error. + - Constant `NEAR_PEAK_FRACTION = 0.5`. + +- [ ] **Step 1: Write the failing test** + +```python +# tests/test_shape_metrics.py +import numpy as np +from sfm.shape_metrics import channel_shape, shape_from_samples + +def test_needle_spike_high_crest_few_near_peak(): + x = np.zeros(1024); x[500] = 1.0 # one isolated spike + s = channel_shape(x) + assert s["sample_count"] == 1024 + assert s["crest_factor"] > 15 # peak towers over rms + assert s["near_peak_count"] <= 3 # almost nothing near the peak + +def test_ringing_low_crest_many_near_peak(): + t = np.arange(1024) + x = np.sin(2*np.pi*t/32) * np.exp(-t/4000) # decaying oscillation + s = channel_shape(x) + assert s["crest_factor"] < 6 + assert s["near_peak_count"] > 30 # many samples near the peak + +def test_channel_shape_none_for_unusable(): + assert channel_shape(np.zeros(1024)) is None # flat / all-zero + assert channel_shape(np.array([1.0])) is None # too short + +def test_shape_from_samples_picks_max_peak_axis(): + chans = {"Tran": np.zeros(1024), "Vert": np.zeros(1024), "Long": np.zeros(1024)} + chans["Long"][10] = 0.5 + chans["Vert"] = np.sin(np.arange(1024)/5) * 0.01 + s = shape_from_samples(chans) + assert s["axis"] == "Long" # Long has the biggest peak + assert s["near_peak_count"] <= 3 + +def test_shape_from_samples_none_when_no_geo(): + assert shape_from_samples({"MicL": np.ones(1024)}) is None +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `/home/serversdown/seismo-relay/.venv/bin/python3 -m pytest tests/test_shape_metrics.py -q` +Expected: FAIL — `ModuleNotFoundError: sfm.shape_metrics`. + +- [ ] **Step 3: Write minimal implementation** + +```python +# sfm/shape_metrics.py +"""Waveform-shape metrics for false-trigger detection. + +A false trigger is an isolated impulse (quiet → spike → quiet); a real event +rings for many cycles. Two numbers separate them: crest factor (how far the +peak stands above the typical sample) and how many samples sit near the peak. +""" +from __future__ import annotations +import numpy as np + +_GEO_CHANNELS = ("Tran", "Vert", "Long") +NEAR_PEAK_FRACTION = 0.5 # a sample "near the peak" is >= this * peak amplitude + + +def channel_shape(x) -> dict | None: + x = np.asarray(x, dtype=float) + if x.size < 2: + return None + peak = float(np.max(np.abs(x))) + if peak <= 0: + return None + rms = float(np.sqrt(np.mean(x ** 2))) + if rms <= 0: + return None + near = int(np.sum(np.abs(x) >= NEAR_PEAK_FRACTION * peak)) + return {"crest_factor": peak / rms, "near_peak_count": near, + "sample_count": int(x.size)} + + +def shape_from_samples(chans: dict) -> dict | None: + best_axis, best_peak, best_x = None, -1.0, None + for ax in _GEO_CHANNELS: + x = chans.get(ax) + if x is None: + continue + x = np.asarray(x, dtype=float) + if x.size < 2: + continue + p = float(np.max(np.abs(x))) + if p > best_peak: + best_axis, best_peak, best_x = ax, p, x + if best_axis is None: + return None + s = channel_shape(best_x) + if s is None: + return None + s["axis"] = best_axis + return s + + +def shape_from_h5(path) -> dict | None: + import h5py + try: + with h5py.File(path, "r") as f: + chans = {ax: f[f"samples/{ax}"][:] for ax in _GEO_CHANNELS + if f"samples/{ax}" in f} + except Exception: + return None + return shape_from_samples(chans) +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `/home/serversdown/seismo-relay/.venv/bin/python3 -m pytest tests/test_shape_metrics.py -q` +Expected: PASS (5 tests). + +- [ ] **Step 5: Commit** + +```bash +git add sfm/shape_metrics.py tests/test_shape_metrics.py +git commit -m "feat(shape): crest-factor + points-near-peak waveform metrics" +``` + +--- + +### Task 2: shape_from_h5 round-trips a real .h5 + +**Files:** +- Test: `tests/test_shape_metrics_h5.py` + +**Interfaces:** +- Consumes: `sfm.shape_metrics.shape_from_h5`; `h5py`. + +- [ ] **Step 1: Write the failing test** (writes a tiny .h5 the same shape SFM writes, then reads it back) + +```python +# tests/test_shape_metrics_h5.py +import numpy as np, h5py +from sfm.shape_metrics import shape_from_h5 + +def _write_h5(path, chans): + with h5py.File(path, "w") as f: + g = f.create_group("samples") + for k, v in chans.items(): + g.create_dataset(k, data=np.asarray(v, dtype="float32")) + +def test_shape_from_h5_reads_dominant_axis(tmp_path): + p = tmp_path / "ev.h5" + long = np.zeros(1024, dtype="float32"); long[100] = 0.48 + _write_h5(p, {"Tran": np.zeros(1024), "Vert": np.zeros(1024), "Long": long, + "MicL": np.ones(1024)}) + s = shape_from_h5(str(p)) + assert s["axis"] == "Long" and s["near_peak_count"] <= 3 + +def test_shape_from_h5_none_on_missing_or_degenerate(tmp_path): + assert shape_from_h5(str(tmp_path / "nope.h5")) is None + p = tmp_path / "degen.h5" + _write_h5(p, {"Tran": np.zeros(1), "Vert": np.zeros(1), "Long": np.zeros(1)}) + assert shape_from_h5(str(p)) is None +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `/home/serversdown/seismo-relay/.venv/bin/python3 -m pytest tests/test_shape_metrics_h5.py -q` +Expected: FAIL (assertion or, if Task 1 incomplete, import error). + +- [ ] **Step 3: Implementation** — none needed; `shape_from_h5` already exists from Task 1. If a test fails, fix `shape_from_h5` (not the test). + +- [ ] **Step 4: Run to verify it passes** + +Run: `/home/serversdown/seismo-relay/.venv/bin/python3 -m pytest tests/test_shape_metrics_h5.py -q` +Expected: PASS (2 tests). + +- [ ] **Step 5: Commit** + +```bash +git add tests/test_shape_metrics_h5.py +git commit -m "test(shape): shape_from_h5 round-trips a real .h5" +``` + +--- + +### Task 3: Add shape columns to the events schema + migration + +**Files:** +- Modify: `sfm/database.py` — `_SCHEMA` CREATE TABLE `events` (after `mic_zc_above_range`); `_migrate` ADD COLUMN loop (the tuple around line 205-218). +- Test: `tests/test_shape_columns.py` + +**Interfaces:** +- Produces: `events` columns `shape_crest_factor REAL`, `shape_near_peak_count INTEGER`, `shape_sample_count INTEGER`, `shape_axis TEXT`. + +- [ ] **Step 1: Write the failing test** + +```python +# tests/test_shape_columns.py +import sqlite3 +from sfm.database import SeismoDb + +_SHAPE_COLS = {"shape_crest_factor", "shape_near_peak_count", + "shape_sample_count", "shape_axis"} + +def _cols(db): + with sqlite3.connect(db.db_path) as c: + return {r[1] for r in c.execute("PRAGMA table_info(events)")} + +def test_fresh_db_has_shape_columns(tmp_path): + db = SeismoDb(tmp_path / "s.db") + assert _SHAPE_COLS <= _cols(db) + +def test_existing_db_migrates_shape_columns(tmp_path): + p = tmp_path / "s.db" + db = SeismoDb(p) + with sqlite3.connect(p) as c: # simulate an older DB missing the columns + for col in _SHAPE_COLS: + c.execute(f"ALTER TABLE events DROP COLUMN {col}") + assert not (_SHAPE_COLS <= _cols(SeismoDb(p))) # sanity: dropped + SeismoDb(p) # re-open triggers _migrate + assert _SHAPE_COLS <= _cols(SeismoDb(p)) +``` + +> Note: sqlite `DROP COLUMN` needs sqlite ≥ 3.35 (bundled py3.10 has it). If the runner's sqlite lacks it, replace the "simulate older DB" block with building a table without the columns; keep the assertion that re-open adds them. + +- [ ] **Step 2: Run to verify it fails** + +Run: `/home/serversdown/seismo-relay/.venv/bin/python3 -m pytest tests/test_shape_columns.py -q` +Expected: FAIL — columns absent. + +- [ ] **Step 3: Implementation** + +In `_SCHEMA`, after the `mic_zc_above_range INTEGER,` line in the `events` CREATE TABLE, add: + +``` + shape_crest_factor REAL, -- peak / rms of the triggering channel + shape_near_peak_count INTEGER, -- samples >= 0.5 * peak (FT: few; real: many) + shape_sample_count INTEGER, -- total samples (to normalize near_peak_count) + shape_axis TEXT, -- geophone channel measured ("Tran"/"Vert"/"Long") +``` + +In `_migrate`, extend the ADD COLUMN tuple (the `for col, ddl in (...)` list) with: + +```python + ("shape_crest_factor", "REAL"), + ("shape_near_peak_count", "INTEGER"), + ("shape_sample_count", "INTEGER"), + ("shape_axis", "TEXT"), +``` + +- [ ] **Step 4: Run to verify it passes** + +Run: `/home/serversdown/seismo-relay/.venv/bin/python3 -m pytest tests/test_shape_columns.py -q` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add sfm/database.py tests/test_shape_columns.py +git commit -m "feat(db): shape_* columns on events (+ auto-migrate)" +``` + +--- + +### Task 4: insert_events persists shape from the waveform record + +**Files:** +- Modify: `sfm/database.py` — `insert_events` INSERT (column list + placeholders + values) and the UPSERT `UPDATE` block. +- Test: `tests/test_insert_events_shape.py` + +**Interfaces:** +- Consumes: a `waveform_records` rec dict that may carry `shape_crest_factor`, `shape_near_peak_count`, `shape_sample_count`, `shape_axis`. +- Produces: those four values stored on the row; refreshed on UPSERT. + +- [ ] **Step 1: Write the failing test** + +```python +# tests/test_insert_events_shape.py +from sfm.database import SeismoDb +from tests.helpers_events import make_event # existing helper used by other insert tests + +def test_insert_stores_shape_from_record(tmp_path): + db = SeismoDb(tmp_path / "s.db") + ev = make_event(serial="BE1", key="0111abcd") + rec = {ev._waveform_key.hex(): { + "filename": "F.CE0W", "filesize": 10, + "shape_crest_factor": 34.0, "shape_near_peak_count": 3, + "shape_sample_count": 1024, "shape_axis": "Long"}} + db.insert_events([ev], serial="BE1", waveform_records=rec) + row = db.query_events(serial="BE1")[0] + assert row["shape_crest_factor"] == 34.0 + assert row["shape_near_peak_count"] == 3 + assert row["shape_axis"] == "Long" +``` + +> If `tests/helpers_events.make_event` doesn't exist, build the `Event` inline the way `tests/test_zc_freq_columns.py` does (copy its event-construction helper). Keep the assertion identical. + +- [ ] **Step 2: Run to verify it fails** + +Run: `/home/serversdown/seismo-relay/.venv/bin/python3 -m pytest tests/test_insert_events_shape.py -q` +Expected: FAIL — `KeyError`/`sqlite3.OperationalError` (columns not in INSERT) or values are NULL. + +- [ ] **Step 3: Implementation** + +In `insert_events` INSERT: add `shape_crest_factor, shape_near_peak_count, shape_sample_count, shape_axis` to the column list, add four `?` placeholders, and add these to the VALUES tuple (after the `mic_zc_above_range` value): + +```python + rec.get("shape_crest_factor"), + rec.get("shape_near_peak_count"), + rec.get("shape_sample_count"), + rec.get("shape_axis"), +``` + +In the UPSERT `UPDATE ... SET`: add + +```sql + shape_crest_factor = COALESCE(?, shape_crest_factor), + shape_near_peak_count = COALESCE(?, shape_near_peak_count), + shape_sample_count = COALESCE(?, shape_sample_count), + shape_axis = COALESCE(?, shape_axis), +``` + +and the matching params (before `serial, ts`): + +```python + rec.get("shape_crest_factor") if rec else None, + rec.get("shape_near_peak_count") if rec else None, + rec.get("shape_sample_count") if rec else None, + rec.get("shape_axis") if rec else None, +``` + +(`COALESCE` on UPSERT so a re-import that lacks samples doesn't wipe a previously-computed shape.) + +- [ ] **Step 4: Run to verify it passes** + +Run: `/home/serversdown/seismo-relay/.venv/bin/python3 -m pytest tests/test_insert_events_shape.py -q` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add sfm/database.py tests/test_insert_events_shape.py +git commit -m "feat(db): insert_events persists shape_* from waveform record" +``` + +--- + +### Task 5: Populate shape at ingest (the three save paths) + +**Files:** +- Modify: `sfm/waveform_store.py` — in `save`, `save_imported_bw`, `save_imported_idf`, after the `.h5` is written, add its shape to the returned `rec` dict. +- Test: `tests/test_save_shape.py` + +**Interfaces:** +- Consumes: `sfm.shape_metrics.shape_from_h5`. +- Produces: `save*` return dicts carry `shape_crest_factor / shape_near_peak_count / shape_sample_count / shape_axis` (present only when the `.h5` had usable samples). + +- [ ] **Step 1: Write the failing test** (drives the BW-import path, which the existing suite already exercises) + +```python +# tests/test_save_shape.py +from sfm.waveform_store import WaveformStore +from tests.helpers_bw import sample_bw_bytes, sample_serial # reuse existing import-test fixtures + +def test_save_imported_bw_attaches_shape(tmp_path): + store = WaveformStore(tmp_path / "waveforms") + ev, rec = store.save_imported_bw(sample_bw_bytes(), serial=sample_serial()) + # A real BW waveform → shape present with a geo axis. + assert rec.get("shape_axis") in ("Tran", "Vert", "Long") + assert rec["shape_crest_factor"] > 0 + assert rec["shape_sample_count"] > 200 +``` + +> Reuse whatever fixture `tests/test_save_imported_bw*.py` already uses for BW bytes; match its import. If the existing BW fixture produces a degenerate/short waveform, use the fixture from the test that asserts a full h5. + +- [ ] **Step 2: Run to verify it fails** + +Run: `/home/serversdown/seismo-relay/.venv/bin/python3 -m pytest tests/test_save_shape.py -q` +Expected: FAIL — `rec` has no `shape_*` keys. + +- [ ] **Step 3: Implementation** + +Add a helper near the top of `WaveformStore` methods (module-level import `from sfm.shape_metrics import shape_from_h5`). In each of `save`, `save_imported_bw`, `save_imported_idf`, immediately before building/returning the `rec` dict — and only when the `.h5` was written (i.e. `hdf5_filename`/`hdf5_path` is set) — compute and merge: + +```python + shape = shape_from_h5(hdf5_path) if hdf5_filename else None + # ... in the returned rec dict literal, add: + # **(shape and { + # "shape_crest_factor": shape["crest_factor"], + # "shape_near_peak_count": shape["near_peak_count"], + # "shape_sample_count": shape["sample_count"], + # "shape_axis": shape["axis"], + # } or {}), +``` + +Concretely, after each method computes `hdf5_filename`, add before its `return {...}`: + +```python + _shape = shape_from_h5(hdf5_path) if hdf5_filename else None + _shape_rec = { + "shape_crest_factor": _shape["crest_factor"], + "shape_near_peak_count": _shape["near_peak_count"], + "shape_sample_count": _shape["sample_count"], + "shape_axis": _shape["axis"], + } if _shape else {} +``` + +and spread `**_shape_rec` into the returned dict. (`save_imported_bw`/`save_imported_idf` use their own hdf5 path variable names — use whichever local holds the written `.h5` path in each method.) + +- [ ] **Step 4: Run to verify it passes** + +Run: `/home/serversdown/seismo-relay/.venv/bin/python3 -m pytest tests/test_save_shape.py -q` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add sfm/waveform_store.py tests/test_save_shape.py +git commit -m "feat(ingest): compute shape from the written .h5 in all save paths" +``` + +--- + +### Task 6: Backfill script for existing events + +**Files:** +- Create: `scripts/backfill_event_shape.py` (mirror `scripts/backfill_event_zc_freq.py`, but read the `.h5` for samples instead of the sidecar). +- Test: `tests/test_backfill_event_shape.py` + +**Interfaces:** +- Produces: `backfill_shape(db, store, *, dry_run=False) -> dict` with counts `{"updated","skipped_no_h5","skipped_no_samples"}`; `main(argv)` CLI mirroring the zc-freq script's args (`--db-path`, `--store-root`, `--dry-run`). + +- [ ] **Step 1: Write the failing test** + +```python +# tests/test_backfill_event_shape.py +import numpy as np, h5py +from sfm.database import SeismoDb +from sfm.waveform_store import WaveformStore +from scripts.backfill_event_shape import backfill_shape +from tests.helpers_events import make_event # or inline as in test_zc_freq_columns + +def _h5(path, long): + with h5py.File(path, "w") as f: + g = f.create_group("samples") + for k in ("Tran", "Vert"): g.create_dataset(k, data=np.zeros(1024, "float32")) + g.create_dataset("Long", data=np.asarray(long, "float32")) + +def test_backfill_updates_shape_and_is_idempotent(tmp_path): + db = SeismoDb(tmp_path / "s.db") + store = WaveformStore(tmp_path / "waveforms") + ev = make_event(serial="BE1", key="0111abcd") + db.insert_events([ev], serial="BE1", + waveform_records={ev._waveform_key.hex(): + {"filename": "F.CE0W", "filesize": 10}}) + # place the .h5 where store.paths_for expects it + long = np.zeros(1024); long[100] = 0.48 + _h5(store.hdf5_path_for("BE1", "F.CE0W"), long) + + c1 = backfill_shape(db, store) + assert c1["updated"] == 1 + row = db.query_events(serial="BE1")[0] + assert row["shape_axis"] == "Long" and row["shape_near_peak_count"] <= 3 + c2 = backfill_shape(db, store) # idempotent: re-run overwrites same values + assert db.query_events(serial="BE1")[0]["shape_crest_factor"] == row["shape_crest_factor"] +``` + +> Match `make_event` / column names to whatever `tests/test_zc_freq_columns.py` uses. `store.hdf5_path_for(serial, filename)` is the existing helper that returns the `.h5` path. + +- [ ] **Step 2: Run to verify it fails** + +Run: `/home/serversdown/seismo-relay/.venv/bin/python3 -m pytest tests/test_backfill_event_shape.py -q` +Expected: FAIL — `ModuleNotFoundError: scripts.backfill_event_shape`. + +- [ ] **Step 3: Implementation** (mirror the zc-freq script structure) + +```python +#!/usr/bin/env python3 +"""Backfill events.shape_* from each event's .h5 waveform samples. Idempotent.""" +from __future__ import annotations +import argparse, logging, sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from sfm.database import SeismoDb +from sfm.waveform_store import WaveformStore +from sfm.shape_metrics import shape_from_h5 + +log = logging.getLogger("backfill_event_shape") + +def backfill_shape(db: SeismoDb, store: WaveformStore, *, dry_run: bool = False) -> dict: + counts = {"updated": 0, "skipped_no_h5": 0, "skipped_no_samples": 0} + for row in db.query_events(limit=1_000_000): + serial, filename = row.get("serial"), row.get("blastware_filename") + if not serial or not filename: + counts["skipped_no_h5"] += 1; continue + h5_path = store.hdf5_path_for(serial, filename) + if not h5_path.exists(): + counts["skipped_no_h5"] += 1; continue + shape = shape_from_h5(h5_path) + if shape is None: + counts["skipped_no_samples"] += 1; continue + if not dry_run: + with db._connect() as conn: + conn.execute( + "UPDATE events SET shape_crest_factor=?, shape_near_peak_count=?, " + "shape_sample_count=?, shape_axis=? WHERE id=?", + (shape["crest_factor"], shape["near_peak_count"], + shape["sample_count"], shape["axis"], row["id"])) + counts["updated"] += 1 + log.info("backfill_shape: %s", counts) + return counts + +def main(argv=None) -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--db-path", required=True) + ap.add_argument("--store-root", required=True) + ap.add_argument("--dry-run", action="store_true") + a = ap.parse_args(argv) + logging.basicConfig(level=logging.INFO) + counts = backfill_shape(SeismoDb(a.db_path), WaveformStore(a.store_root), dry_run=a.dry_run) + print(counts) + return 0 + +if __name__ == "__main__": + raise SystemExit(main()) +``` + +> `store.hdf5_path_for` and `db._connect` are existing internals used the same way by other scripts. If `hdf5_path_for` isn't public, use `store.paths_for(...)` sibling `.h5` path exactly as `save()` derives `hdf5_path`. + +- [ ] **Step 4: Run to verify it passes** + +Run: `/home/serversdown/seismo-relay/.venv/bin/python3 -m pytest tests/test_backfill_event_shape.py -q` +Expected: PASS. + +- [ ] **Step 5: Run the full suite + commit** + +```bash +/home/serversdown/seismo-relay/.venv/bin/python3 -m pytest -q +git add scripts/backfill_event_shape.py tests/test_backfill_event_shape.py +git commit -m "feat(scripts): backfill events.shape_* from .h5 samples" +``` + +--- + +### Task 7: Confirm /db/events carries shape + version bump + +**Files:** +- Test: `tests/test_db_events_exposes_shape.py` +- Modify: `pyproject.toml` version; `sfm/server.py` version string; `CHANGELOG.md`. + +**Interfaces:** +- Consumes: the running `/db/events` route (already returns `SELECT *`). + +- [ ] **Step 1: Write the failing test** (guards that the feed dict includes the new keys) + +```python +# tests/test_db_events_exposes_shape.py +from sfm.database import SeismoDb +def test_query_events_row_includes_shape_keys(tmp_path): + db = SeismoDb(tmp_path / "s.db") + # query_events returns dict(row); a fresh insert has the keys (values may be None) + from tests.helpers_events import make_event + db.insert_events([make_event(serial="BE1", key="0111abcd")], serial="BE1") + row = db.query_events(serial="BE1")[0] + for k in ("shape_crest_factor","shape_near_peak_count","shape_sample_count","shape_axis"): + assert k in row +``` + +- [ ] **Step 2: Run to verify it fails / passes** + +Run: `/home/serversdown/seismo-relay/.venv/bin/python3 -m pytest tests/test_db_events_exposes_shape.py -q` +Expected: PASS immediately if Task 3 landed (columns present in `SELECT *`). If it fails, the columns weren't added — fix Task 3. (This task is the guard, not new behavior.) + +- [ ] **Step 3: Version bump** + +Bump `pyproject.toml` `version` 0.23.0 → 0.24.0; set `sfm/server.py` `version="0.24.0"`; add a `## v0.24.0` CHANGELOG entry ("waveform-shape metrics on events: crest factor + points-near-peak, ingest + backfill"). + +- [ ] **Step 4: Full suite** + +Run: `/home/serversdown/seismo-relay/.venv/bin/python3 -m pytest -q` +Expected: PASS (no regressions). + +- [ ] **Step 5: Commit** + +```bash +git add tests/test_db_events_exposes_shape.py pyproject.toml sfm/server.py CHANGELOG.md +git commit -m "chore(release): v0.24.0 — waveform-shape metrics on events" +``` + +--- + +## Self-Review + +**Spec coverage:** SFM shape columns (Tasks 3–4) ✓; DSP crest + near-peak (Task 1) ✓; ingest population (Task 5) ✓; backfill (Task 6) ✓; `/db/events` exposure (Task 7) ✓; NULL for histogram/no-sample events (Tasks 1/5/6 return None → NULL) ✓; 94%-coverage / series-4 fallback handled by NULL-then-Terra-View-fallback (Phase B) ✓. Terra-View scoring, 3-state review, twin flagging, export Notes column → **Phase B plan** (separate, depends on this feed). Calibration → Phase C. + +**Placeholder scan:** No TBD/TODO; every code step has real code. The two "reuse existing fixture" notes point at concrete existing tests (`test_zc_freq_columns.py`, `test_save_imported_bw*.py`) rather than leaving blanks. + +**Type consistency:** `shape_from_h5`/`shape_from_samples`/`channel_shape` return the same dict keys (`crest_factor`, `near_peak_count`, `sample_count`, `axis`) throughout; the DB columns (`shape_crest_factor`, `shape_near_peak_count`, `shape_sample_count`, `shape_axis`) and rec keys match across Tasks 4–6. + +## Deferred to Phase B (terra-view, separate plan) + +Scoring service combining shape + cheap signals; suspicion column + reason chips; `reviewed_real` mirror + 3-state review; twin-aware flag propagation; Notes-column export + Maximums "(excludes N flagged)". Written once this feed is live so column names/values are real. -- 2.54.0 From 2539f903de489da1f04a64e69967e64ce949f157 Mon Sep 17 00:00:00 2001 From: serversdown Date: Sat, 22 Aug 2026 05:42:36 +0000 Subject: [PATCH 02/30] feat(shape): crest-factor + points-near-peak waveform metrics --- sfm/shape_metrics.py | 58 +++++++++++++++++++++++++++++++++++++ tests/test_shape_metrics.py | 31 ++++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 sfm/shape_metrics.py create mode 100644 tests/test_shape_metrics.py diff --git a/sfm/shape_metrics.py b/sfm/shape_metrics.py new file mode 100644 index 0000000..05c9dd5 --- /dev/null +++ b/sfm/shape_metrics.py @@ -0,0 +1,58 @@ +"""Waveform-shape metrics for false-trigger detection. + +A false trigger is an isolated impulse (quiet → spike → quiet); a real event +rings for many cycles. Two numbers separate them: crest factor (how far the +peak stands above the typical sample) and how many samples sit near the peak. +""" +from __future__ import annotations +import numpy as np + +_GEO_CHANNELS = ("Tran", "Vert", "Long") +NEAR_PEAK_FRACTION = 0.5 # a sample "near the peak" is >= this * peak amplitude + + +def channel_shape(x) -> dict | None: + x = np.asarray(x, dtype=float) + if x.size < 2: + return None + peak = float(np.max(np.abs(x))) + if peak <= 0: + return None + rms = float(np.sqrt(np.mean(x ** 2))) + if rms <= 0: + return None + near = int(np.sum(np.abs(x) >= NEAR_PEAK_FRACTION * peak)) + return {"crest_factor": peak / rms, "near_peak_count": near, + "sample_count": int(x.size)} + + +def shape_from_samples(chans: dict) -> dict | None: + best_axis, best_peak, best_x = None, -1.0, None + for ax in _GEO_CHANNELS: + x = chans.get(ax) + if x is None: + continue + x = np.asarray(x, dtype=float) + if x.size < 2: + continue + p = float(np.max(np.abs(x))) + if p > best_peak: + best_axis, best_peak, best_x = ax, p, x + if best_axis is None: + return None + s = channel_shape(best_x) + if s is None: + return None + s["axis"] = best_axis + return s + + +def shape_from_h5(path) -> dict | None: + import h5py + try: + with h5py.File(path, "r") as f: + chans = {ax: f[f"samples/{ax}"][:] for ax in _GEO_CHANNELS + if f"samples/{ax}" in f} + except Exception: + return None + return shape_from_samples(chans) diff --git a/tests/test_shape_metrics.py b/tests/test_shape_metrics.py new file mode 100644 index 0000000..ae23475 --- /dev/null +++ b/tests/test_shape_metrics.py @@ -0,0 +1,31 @@ +import numpy as np +from sfm.shape_metrics import channel_shape, shape_from_samples + +def test_needle_spike_high_crest_few_near_peak(): + x = np.zeros(1024); x[500] = 1.0 # one isolated spike + s = channel_shape(x) + assert s["sample_count"] == 1024 + assert s["crest_factor"] > 15 # peak towers over rms + assert s["near_peak_count"] <= 3 # almost nothing near the peak + +def test_ringing_low_crest_many_near_peak(): + t = np.arange(1024) + x = np.sin(2*np.pi*t/32) * np.exp(-t/4000) # decaying oscillation + s = channel_shape(x) + assert s["crest_factor"] < 6 + assert s["near_peak_count"] > 30 # many samples near the peak + +def test_channel_shape_none_for_unusable(): + assert channel_shape(np.zeros(1024)) is None # flat / all-zero + assert channel_shape(np.array([1.0])) is None # too short + +def test_shape_from_samples_picks_max_peak_axis(): + chans = {"Tran": np.zeros(1024), "Vert": np.zeros(1024), "Long": np.zeros(1024)} + chans["Long"][10] = 0.5 + chans["Vert"] = np.sin(np.arange(1024)/5) * 0.01 + s = shape_from_samples(chans) + assert s["axis"] == "Long" # Long has the biggest peak + assert s["near_peak_count"] <= 3 + +def test_shape_from_samples_none_when_no_geo(): + assert shape_from_samples({"MicL": np.ones(1024)}) is None -- 2.54.0 From cec82038ea79ba52287a7cbe56eb6992b7f33aaa Mon Sep 17 00:00:00 2001 From: serversdown Date: Sat, 22 Aug 2026 05:46:58 +0000 Subject: [PATCH 03/30] test(shape): shape_from_h5 round-trips a real .h5 --- tests/test_shape_metrics_h5.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 tests/test_shape_metrics_h5.py diff --git a/tests/test_shape_metrics_h5.py b/tests/test_shape_metrics_h5.py new file mode 100644 index 0000000..c601675 --- /dev/null +++ b/tests/test_shape_metrics_h5.py @@ -0,0 +1,22 @@ +import numpy as np, h5py +from sfm.shape_metrics import shape_from_h5 + +def _write_h5(path, chans): + with h5py.File(path, "w") as f: + g = f.create_group("samples") + for k, v in chans.items(): + g.create_dataset(k, data=np.asarray(v, dtype="float32")) + +def test_shape_from_h5_reads_dominant_axis(tmp_path): + p = tmp_path / "ev.h5" + long = np.zeros(1024, dtype="float32"); long[100] = 0.48 + _write_h5(p, {"Tran": np.zeros(1024), "Vert": np.zeros(1024), "Long": long, + "MicL": np.ones(1024)}) + s = shape_from_h5(str(p)) + assert s["axis"] == "Long" and s["near_peak_count"] <= 3 + +def test_shape_from_h5_none_on_missing_or_degenerate(tmp_path): + assert shape_from_h5(str(tmp_path / "nope.h5")) is None + p = tmp_path / "degen.h5" + _write_h5(p, {"Tran": np.zeros(1), "Vert": np.zeros(1), "Long": np.zeros(1)}) + assert shape_from_h5(str(p)) is None -- 2.54.0 From a894b001b1c6bdcf62dcde813b64dc529be3221e Mon Sep 17 00:00:00 2001 From: serversdown Date: Sat, 22 Aug 2026 05:50:45 +0000 Subject: [PATCH 04/30] feat(db): shape_* columns on events (+ auto-migrate) --- sfm/database.py | 8 ++++++++ tests/test_shape_columns.py | 27 +++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 tests/test_shape_columns.py diff --git a/sfm/database.py b/sfm/database.py index ff042f9..8ce35d7 100644 --- a/sfm/database.py +++ b/sfm/database.py @@ -94,6 +94,10 @@ CREATE TABLE IF NOT EXISTS events ( vert_zc_above_range INTEGER, long_zc_above_range INTEGER, mic_zc_above_range INTEGER, + shape_crest_factor REAL, -- peak / rms of the triggering channel + shape_near_peak_count INTEGER, -- samples >= 0.5 * peak (FT: few; real: many) + shape_sample_count INTEGER, -- total samples (to normalize near_peak_count) + shape_axis TEXT, -- geophone channel measured ("Tran"/"Vert"/"Long") created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), UNIQUE(serial, timestamp) ); @@ -216,6 +220,10 @@ class SeismoDb: ("vert_zc_above_range", "INTEGER"), ("long_zc_above_range", "INTEGER"), ("mic_zc_above_range", "INTEGER"), + ("shape_crest_factor", "REAL"), + ("shape_near_peak_count", "INTEGER"), + ("shape_sample_count", "INTEGER"), + ("shape_axis", "TEXT"), ): if col not in existing_cols: log.info("_migrate: events ADD COLUMN %s %s", col, ddl) diff --git a/tests/test_shape_columns.py b/tests/test_shape_columns.py new file mode 100644 index 0000000..9bb92ac --- /dev/null +++ b/tests/test_shape_columns.py @@ -0,0 +1,27 @@ +import sqlite3 +from sfm.database import SeismoDb + +_SHAPE_COLS = {"shape_crest_factor", "shape_near_peak_count", + "shape_sample_count", "shape_axis"} + +def _cols(db): + with sqlite3.connect(db.db_path) as c: + return {r[1] for r in c.execute("PRAGMA table_info(events)")} + +def test_fresh_db_has_shape_columns(tmp_path): + db = SeismoDb(tmp_path / "s.db") + assert _SHAPE_COLS <= _cols(db) + +def test_existing_db_migrates_shape_columns(tmp_path): + p = tmp_path / "s.db" + db = SeismoDb(p) + with sqlite3.connect(p) as c: # simulate an older DB missing the columns + for col in _SHAPE_COLS: + c.execute(f"ALTER TABLE events DROP COLUMN {col}") + # sanity: dropped. Read via the already-constructed `db` (a raw PRAGMA + # read against db.db_path) rather than a fresh SeismoDb(p) — the latter + # would immediately re-trigger _migrate's self-healing ADD COLUMN loop + # and mask the gap we're trying to confirm. + assert not (_SHAPE_COLS <= _cols(db)) + SeismoDb(p) # re-open triggers _migrate + assert _SHAPE_COLS <= _cols(SeismoDb(p)) -- 2.54.0 From 54c4182023421e37f972780efa71c432c1c28c11 Mon Sep 17 00:00:00 2001 From: serversdown Date: Sat, 22 Aug 2026 05:55:25 +0000 Subject: [PATCH 05/30] feat(db): insert_events persists shape_* from waveform record --- sfm/database.py | 20 +++++++-- tests/test_insert_events_shape.py | 69 +++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 3 deletions(-) create mode 100644 tests/test_insert_events_shape.py diff --git a/sfm/database.py b/sfm/database.py index 8ce35d7..464c2b5 100644 --- a/sfm/database.py +++ b/sfm/database.py @@ -426,9 +426,11 @@ class SeismoDb: device_family, tran_zc_freq, vert_zc_freq, long_zc_freq, mic_zc_freq, tran_zc_above_range, vert_zc_above_range, - long_zc_above_range, mic_zc_above_range) + long_zc_above_range, mic_zc_above_range, + shape_crest_factor, shape_near_peak_count, + shape_sample_count, shape_axis) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ?, ?, ?, ?, ?, ?, ?) + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( self._new_id(), serial, key, session_id, ts, @@ -456,6 +458,10 @@ class SeismoDb: (1 if (pv and pv.vert_zc_above_range) else 0), (1 if (pv and pv.long_zc_above_range) else 0), (1 if (pv and pv.mic_zc_above_range) else 0), + rec.get("shape_crest_factor"), + rec.get("shape_near_peak_count"), + rec.get("shape_sample_count"), + rec.get("shape_axis"), ), ) inserted += 1 @@ -505,7 +511,11 @@ class SeismoDb: tran_zc_above_range = ?, vert_zc_above_range = ?, long_zc_above_range = ?, - mic_zc_above_range = ? + mic_zc_above_range = ?, + shape_crest_factor = COALESCE(?, shape_crest_factor), + shape_near_peak_count = COALESCE(?, shape_near_peak_count), + shape_sample_count = COALESCE(?, shape_sample_count), + shape_axis = COALESCE(?, shape_axis) WHERE serial = ? AND timestamp = ? """, ( @@ -533,6 +543,10 @@ class SeismoDb: (1 if (pv and pv.vert_zc_above_range) else 0), (1 if (pv and pv.long_zc_above_range) else 0), (1 if (pv and pv.mic_zc_above_range) else 0), + rec.get("shape_crest_factor") if rec else None, + rec.get("shape_near_peak_count") if rec else None, + rec.get("shape_sample_count") if rec else None, + rec.get("shape_axis") if rec else None, serial, ts, ), diff --git a/tests/test_insert_events_shape.py b/tests/test_insert_events_shape.py new file mode 100644 index 0000000..63e90a6 --- /dev/null +++ b/tests/test_insert_events_shape.py @@ -0,0 +1,69 @@ +from __future__ import annotations +from pathlib import Path + +from sfm.database import SeismoDb +from minimateplus.models import Event, Timestamp, PeakValues + + +def _event(waveform_key="0111abcd"): + ev = Event(index=0) + ev._waveform_key = bytes.fromhex(waveform_key) + ev.timestamp = Timestamp(raw=b"", flag=0x10, year=2026, unknown_byte=0, + month=6, day=25, hour=8, minute=50, second=0) + ev.record_type = "Waveform" + ev.peak_values = PeakValues(tran=0.075, vert=0.220, long=0.045, + peak_vector_sum=0.231, micl=0.01) + return ev + + +def test_insert_stores_shape_from_record(tmp_path: Path): + db = SeismoDb(tmp_path / "s.db") + ev = _event() + rec = {ev._waveform_key.hex(): { + "filename": "F.CE0W", "filesize": 10, + "shape_crest_factor": 34.0, "shape_near_peak_count": 3, + "shape_sample_count": 1024, "shape_axis": "Long"}} + db.insert_events([ev], serial="BE1", waveform_records=rec) + row = db.query_events(serial="BE1")[0] + assert row["shape_crest_factor"] == 34.0 + assert row["shape_near_peak_count"] == 3 + assert row["shape_axis"] == "Long" + + +def test_upsert_refreshes_shape_from_record(tmp_path: Path): + db = SeismoDb(tmp_path / "s.db") + ev = _event() + rec1 = {ev._waveform_key.hex(): { + "shape_crest_factor": 10.0, "shape_near_peak_count": 1, + "shape_sample_count": 512, "shape_axis": "Tran"}} + db.insert_events([ev], serial="BE1", waveform_records=rec1) + + ev2 = _event() + rec2 = {ev2._waveform_key.hex(): { + "shape_crest_factor": 22.5, "shape_near_peak_count": 7, + "shape_sample_count": 2048, "shape_axis": "Vert"}} + db.insert_events([ev2], serial="BE1", waveform_records=rec2) + + row = db.query_events(serial="BE1")[0] + assert row["shape_crest_factor"] == 22.5 + assert row["shape_near_peak_count"] == 7 + assert row["shape_sample_count"] == 2048 + assert row["shape_axis"] == "Vert" + + +def test_upsert_preserves_shape_when_record_missing(tmp_path: Path): + db = SeismoDb(tmp_path / "s.db") + ev = _event() + rec1 = {ev._waveform_key.hex(): { + "shape_crest_factor": 10.0, "shape_near_peak_count": 1, + "shape_sample_count": 512, "shape_axis": "Tran"}} + db.insert_events([ev], serial="BE1", waveform_records=rec1) + + ev2 = _event() # re-import with no waveform_records (e.g. sample missing) + db.insert_events([ev2], serial="BE1") + + row = db.query_events(serial="BE1")[0] + assert row["shape_crest_factor"] == 10.0 + assert row["shape_near_peak_count"] == 1 + assert row["shape_sample_count"] == 512 + assert row["shape_axis"] == "Tran" -- 2.54.0 From e64e3bcd3ecc4031244719386e0a4e316d18a6f0 Mon Sep 17 00:00:00 2001 From: serversdown Date: Sat, 22 Aug 2026 06:01:43 +0000 Subject: [PATCH 06/30] feat(ingest): compute shape from the written .h5 in all save paths --- sfm/waveform_store.py | 25 +++++++++++++++++++++++++ tests/test_save_shape.py | 13 +++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 tests/test_save_shape.py diff --git a/sfm/waveform_store.py b/sfm/waveform_store.py index 5144754..6b25e14 100644 --- a/sfm/waveform_store.py +++ b/sfm/waveform_store.py @@ -41,6 +41,7 @@ from minimateplus.blastware_file import blastware_filename, write_blastware_file from minimateplus.framing import S3Frame from minimateplus.models import Event from sfm import event_hdf5 +from sfm.shape_metrics import shape_from_h5 log = logging.getLogger("sfm.waveform_store") @@ -262,6 +263,13 @@ class WaveformStore: serial, filename, filesize, len(a5_frames), hdf5_filename or "(skipped)", sidecar_path.name, ) + _shape = shape_from_h5(hdf5_path) if hdf5_filename else None + _shape_rec = { + "shape_crest_factor": _shape["crest_factor"], + "shape_near_peak_count": _shape["near_peak_count"], + "shape_sample_count": _shape["sample_count"], + "shape_axis": _shape["axis"], + } if _shape else {} return { "filename": filename, "filesize": filesize, @@ -269,6 +277,7 @@ class WaveformStore: "a5_pickle_filename": a5_path.name, "hdf5_filename": hdf5_filename, "sidecar_filename": sidecar_path.name, + **_shape_rec, } def save_imported_bw( @@ -445,6 +454,13 @@ class WaveformStore: "h5=%s (no .a5.pkl — A5 source unavailable for BW-imported files)", serial, filename, filesize, hdf5_filename or "(skipped)", ) + _shape = shape_from_h5(hdf5_path) if hdf5_filename else None + _shape_rec = { + "shape_crest_factor": _shape["crest_factor"], + "shape_near_peak_count": _shape["near_peak_count"], + "shape_sample_count": _shape["sample_count"], + "shape_axis": _shape["axis"], + } if _shape else {} return ev, { "filename": filename, "filesize": filesize, @@ -453,6 +469,7 @@ class WaveformStore: "hdf5_filename": hdf5_filename, "sidecar_filename": sidecar_path.name, "serial": serial, + **_shape_rec, } def save_imported_idf( @@ -727,6 +744,13 @@ class WaveformStore: hdf5_filename or "(skipped)", len(idf_intervals) if idf_intervals else 0, ) + _shape = shape_from_h5(hdf5_path) if hdf5_filename else None + _shape_rec = { + "shape_crest_factor": _shape["crest_factor"], + "shape_near_peak_count": _shape["near_peak_count"], + "shape_sample_count": _shape["sample_count"], + "shape_axis": _shape["axis"], + } if _shape else {} return ev, { "filename": filename, "filesize": filesize, @@ -735,6 +759,7 @@ class WaveformStore: "hdf5_filename": hdf5_filename, "sidecar_filename": sidecar_path.name, "serial": serial, + **_shape_rec, } def load_a5(self, serial: str, filename: str) -> Optional[list[S3Frame]]: diff --git a/tests/test_save_shape.py b/tests/test_save_shape.py new file mode 100644 index 0000000..77ae592 --- /dev/null +++ b/tests/test_save_shape.py @@ -0,0 +1,13 @@ +from pathlib import Path + +from sfm.waveform_store import WaveformStore + +_FIX = Path(__file__).parent / "fixtures/histogram-extension-re/events-5-21-26/K558LL8B.7I0W" + + +def test_save_imported_bw_attaches_shape(tmp_path): + store = WaveformStore(tmp_path / "waveforms") + ev, rec = store.save_imported_bw(_FIX.read_bytes(), source_path=_FIX, serial_hint="BE9558") + assert rec.get("shape_axis") in ("Tran", "Vert", "Long") + assert rec["shape_crest_factor"] > 0 + assert rec["shape_sample_count"] > 200 -- 2.54.0 From c982512e170c3cdd1bf2a2ed1eece5a41dcd461e Mon Sep 17 00:00:00 2001 From: serversdown Date: Sat, 22 Aug 2026 06:06:46 +0000 Subject: [PATCH 07/30] feat(scripts): backfill events.shape_* from .h5 samples --- scripts/backfill_event_shape.py | 48 ++++++++++++++++++++++++++++++ tests/test_backfill_event_shape.py | 35 ++++++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 scripts/backfill_event_shape.py create mode 100644 tests/test_backfill_event_shape.py diff --git a/scripts/backfill_event_shape.py b/scripts/backfill_event_shape.py new file mode 100644 index 0000000..9de2f85 --- /dev/null +++ b/scripts/backfill_event_shape.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Backfill events.shape_* from each event's .h5 waveform samples. Idempotent.""" +from __future__ import annotations +import argparse, logging, sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from sfm.database import SeismoDb +from sfm.waveform_store import WaveformStore +from sfm.shape_metrics import shape_from_h5 + +log = logging.getLogger("backfill_event_shape") + +def backfill_shape(db: SeismoDb, store: WaveformStore, *, dry_run: bool = False) -> dict: + counts = {"updated": 0, "skipped_no_h5": 0, "skipped_no_samples": 0} + for row in db.query_events(limit=1_000_000): + serial, filename = row.get("serial"), row.get("blastware_filename") + if not serial or not filename: + counts["skipped_no_h5"] += 1; continue + h5_path = store.hdf5_path_for(serial, filename) + if not h5_path.exists(): + counts["skipped_no_h5"] += 1; continue + shape = shape_from_h5(h5_path) + if shape is None: + counts["skipped_no_samples"] += 1; continue + if not dry_run: + with db._connect() as conn: + conn.execute( + "UPDATE events SET shape_crest_factor=?, shape_near_peak_count=?, " + "shape_sample_count=?, shape_axis=? WHERE id=?", + (shape["crest_factor"], shape["near_peak_count"], + shape["sample_count"], shape["axis"], row["id"])) + counts["updated"] += 1 + log.info("backfill_shape: %s", counts) + return counts + +def main(argv=None) -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--db-path", required=True) + ap.add_argument("--store-root", required=True) + ap.add_argument("--dry-run", action="store_true") + a = ap.parse_args(argv) + logging.basicConfig(level=logging.INFO) + counts = backfill_shape(SeismoDb(a.db_path), WaveformStore(a.store_root), dry_run=a.dry_run) + print(counts) + return 0 + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_backfill_event_shape.py b/tests/test_backfill_event_shape.py new file mode 100644 index 0000000..3c926c3 --- /dev/null +++ b/tests/test_backfill_event_shape.py @@ -0,0 +1,35 @@ +from __future__ import annotations +import numpy as np, h5py + +from sfm.database import SeismoDb +from sfm.waveform_store import WaveformStore +from scripts.backfill_event_shape import backfill_shape +from minimateplus.models import Event + + +def _h5(path, long): + with h5py.File(path, "w") as f: + g = f.create_group("samples") + for k in ("Tran", "Vert"): + g.create_dataset(k, data=np.zeros(1024, "float32")) + g.create_dataset("Long", data=np.asarray(long, "float32")) + + +def test_backfill_updates_shape_and_is_idempotent(tmp_path): + db = SeismoDb(tmp_path / "s.db") + store = WaveformStore(tmp_path / "waveforms") + ev = Event(index=0) + ev._waveform_key = bytes.fromhex("0111abcd") + db.insert_events([ev], serial="BE1", + waveform_records={ev._waveform_key.hex(): + {"filename": "F.CE0W", "filesize": 10}}) + # place the .h5 where store.paths_for expects it + long = np.zeros(1024); long[100] = 0.48 + _h5(store.hdf5_path_for("BE1", "F.CE0W"), long) + + c1 = backfill_shape(db, store) + assert c1["updated"] == 1 + row = db.query_events(serial="BE1")[0] + assert row["shape_axis"] == "Long" and row["shape_near_peak_count"] <= 3 + c2 = backfill_shape(db, store) # idempotent: re-run overwrites same values + assert db.query_events(serial="BE1")[0]["shape_crest_factor"] == row["shape_crest_factor"] -- 2.54.0 From ac67e83bcff65b7d4495c533574f73bbab7cd035 Mon Sep 17 00:00:00 2001 From: serversdown Date: Sat, 22 Aug 2026 06:12:01 +0000 Subject: [PATCH 08/30] =?UTF-8?q?chore(release):=20v0.24.0=20=E2=80=94=20w?= =?UTF-8?q?aveform-shape=20metrics=20on=20events?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 31 +++++++++++++++++++++++++++ pyproject.toml | 2 +- sfm/server.py | 2 +- tests/test_db_events_exposes_shape.py | 29 +++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 tests/test_db_events_exposes_shape.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 914b4ad..b4fa04a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,37 @@ All notable changes to seismo-relay are documented here. --- +## v0.24.0 — 2026-08-22 + +**Waveform-shape metrics on events.** The `events` table and the `/db/events` +feed now carry per-event crest factor and points-near-peak, computed from the +decoded waveform samples at ingest — groundwork for Terra-View's +false-trigger detection (Phase B). + +### Added + +- **`events` columns** `shape_crest_factor`, `shape_near_peak_count`, + `shape_sample_count`, `shape_axis`. Added via the existing incremental + `_migrate` ADD COLUMN pass — **auto-migrates on `SeismoDb()` construction, + no manual migration**. `query_events` / `get_event` (and thus `/db/events`) + return them automatically (`SELECT *`). +- **Populated at ingest** — crest factor + near-peak-count are computed from + the decoded samples in every save path (`shape_from_h5`/ + `shape_from_samples`), and `insert_events` persists them on INSERT and + UPSERT. +- **Backfill** `scripts/backfill_event_shape.py` — fills the columns for + existing events from their on-disk `.h5` waveform samples (idempotent, + UPDATE-only). + +### Upgrade Notes + +Run the backfill once after deploying, against the events DB + waveform store: +`python3 scripts/backfill_event_shape.py --db-path --store-root ` +Events with no decodable samples (or no waveform file) stay NULL and render +"—" downstream. + +--- + ## v0.23.0 — 2026-08-04 **Per-channel ZC frequency in the events store.** The `events` table and the diff --git a/pyproject.toml b/pyproject.toml index 11f37d4..2a51e33 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "seismo-relay" -version = "0.23.0" +version = "0.24.0" description = "Python client and REST server for MiniMate Plus seismographs" requires-python = ">=3.10" dependencies = [ diff --git a/sfm/server.py b/sfm/server.py index 81041c4..ea0c92a 100644 --- a/sfm/server.py +++ b/sfm/server.py @@ -90,7 +90,7 @@ app = FastAPI( "Implements the minimateplus RS-232 protocol library.\n" "Proxied by terra-view at /api/sfm/*." ), - version="0.23.0", + version="0.24.0", ) # Allow requests from the waveform viewer opened as a local file (file://) diff --git a/tests/test_db_events_exposes_shape.py b/tests/test_db_events_exposes_shape.py new file mode 100644 index 0000000..9590ab8 --- /dev/null +++ b/tests/test_db_events_exposes_shape.py @@ -0,0 +1,29 @@ +from __future__ import annotations +import os, sys +from pathlib import Path + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from sfm.database import SeismoDb +from minimateplus.models import Event + +SHAPE_COLS = ( + "shape_crest_factor", + "shape_near_peak_count", + "shape_sample_count", + "shape_axis", +) + + +def test_query_events_row_includes_shape_keys(tmp_path: Path): + db = SeismoDb(tmp_path / "s.db") + + ev = Event(index=0) + ev._waveform_key = bytes.fromhex("0111abcd") + + db.insert_events([ev], serial="BE1") + row = db.query_events(serial="BE1")[0] + + for k in SHAPE_COLS: + assert k in row + assert row[k] is None -- 2.54.0 From 5247e7866987148147a14f5b80b9c8aca2694712 Mon Sep 17 00:00:00 2001 From: serversdown Date: Tue, 25 Aug 2026 00:25:02 +0000 Subject: [PATCH 09/30] =?UTF-8?q?docs(plan):=20B2-A=20=E2=80=94=20reviewed?= =?UTF-8?q?=5Freal=20+=203-state=20mirror=20+=20twin=20review-propagation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01YDXjZCr4RqT2U3QvMDhgzf --- ...8-25-b2a-reviewed-real-twin-propagation.md | 264 ++++++++++++++++++ 1 file changed, 264 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-25-b2a-reviewed-real-twin-propagation.md diff --git a/docs/superpowers/plans/2026-08-25-b2a-reviewed-real-twin-propagation.md b/docs/superpowers/plans/2026-08-25-b2a-reviewed-real-twin-propagation.md new file mode 100644 index 0000000..e19582f --- /dev/null +++ b/docs/superpowers/plans/2026-08-25-b2a-reviewed-real-twin-propagation.md @@ -0,0 +1,264 @@ +# Phase B2-A (seismo-relay) — reviewed_real + 3-state mirror + twin propagation + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Give SFM a persisted, queryable `reviewed_real` flag alongside `false_trigger` (mirrored from the sidecar review block, mutually exclusive), and propagate a review to an event's histogram/waveform twin so flagging one flags both. + +**Architecture:** Mirror the existing `false_trigger` mechanism. New `events.reviewed_real` column (auto-migrate). `update_event_review` mirrors BOTH flags from the sidecar review block and enforces mutual exclusivity on the columns. A `find_twins` matcher (same serial + identical peak_vector_sum + timestamp within a window) drives `propagate_review_to_twins`, which the sidecar-PATCH endpoint calls after mirroring the primary. Terra-View reads `reviewed_real` from `/db/events` (SELECT *). + +**Tech Stack:** Python 3.10, raw sqlite3, FastAPI, pytest. Runner: `/home/serversdown/seismo-relay/.venv/bin/python3`. + +## Global Constraints + +- 3 review states are **mutually exclusive**: an event is `false_trigger=1` XOR `reviewed_real=1` XOR neither. Setting one true forces the other's column to 0. +- The sidecar JSON stays the source of truth for full review state; the `false_trigger`/`reviewed_real` columns are derived indexes (like today). B2-A propagates twins at the **column** level (what the feed/peak/export read); twin sidecars are not rewritten (known limitation — noted). +- Twin match = **same serial AND identical `peak_vector_sum` (exact equality) AND `timestamp` within ± window (default 300 s), excluding the event itself.** Identical PVS is the safety anchor. +- Known pre-existing test failures (~16, missing gitignored fixtures under `tests/fixtures/`) are unrelated — confirm zero NEW failures, don't try to fix them. +- Run tests with `/home/serversdown/seismo-relay/.venv/bin/python3 -m pytest`. + +--- + +### Task 1: `reviewed_real` column on events + +**Files:** Modify `sfm/database.py` (`_SCHEMA` CREATE TABLE `events` + the Migration-1 rebuild `CREATE TABLE` + the `_migrate` ADD COLUMN loop). Test: `tests/test_reviewed_real_column.py`. + +**Interfaces:** Produces `events.reviewed_real INTEGER NOT NULL DEFAULT 0`. + +- [ ] **Step 1: Failing test** +```python +# tests/test_reviewed_real_column.py +import sqlite3 +from sfm.database import SeismoDb +def _cols(db): + with sqlite3.connect(db.db_path) as c: + return {r[1] for r in c.execute("PRAGMA table_info(events)")} +def test_fresh_db_has_reviewed_real(tmp_path): + assert "reviewed_real" in _cols(SeismoDb(tmp_path/"s.db")) +def test_existing_db_migrates_reviewed_real(tmp_path): + p = tmp_path/"s.db"; db = SeismoDb(p) + with sqlite3.connect(p) as c: + c.execute("ALTER TABLE events DROP COLUMN reviewed_real") + assert "reviewed_real" not in _cols(db) # dropped (read via existing handle/connection) + SeismoDb(p) # re-open migrates + assert "reviewed_real" in _cols(SeismoDb(p)) +``` +> If sqlite < 3.35 lacks DROP COLUMN, fall back to building a table without the column and asserting re-open adds it (same as the shape-columns test did). + +- [ ] **Step 2: Run → FAIL** (`pytest tests/test_reviewed_real_column.py -q`). +- [ ] **Step 3: Implement** + - In `_SCHEMA` `events` CREATE TABLE, after the `false_trigger ... DEFAULT 0,` line add: ` reviewed_real INTEGER NOT NULL DEFAULT 0, -- 0=no, 1=operator-confirmed real (mutually exclusive with false_trigger)` + - In the Migration-1 rebuild `CREATE TABLE events (...)` block, add the same `reviewed_real INTEGER NOT NULL DEFAULT 0,` line after `false_trigger`. + - In the `_migrate` ADD COLUMN loop tuple add: `("reviewed_real", "INTEGER NOT NULL DEFAULT 0"),` +- [ ] **Step 4: Run → PASS.** +- [ ] **Step 5: Commit** `feat(db): reviewed_real column on events (+ auto-migrate)` + +--- + +### Task 2: `update_event_review` mirrors both flags + mutual exclusivity + +**Files:** Modify `sfm/database.py` `update_event_review`. Test: `tests/test_update_event_review_reviewed_real.py`. + +**Interfaces:** Consumes a `review` dict that may carry `false_trigger` and/or `reviewed_real` (bools). Produces mutually-exclusive column state. + +- [ ] **Step 1: Failing test** +```python +# tests/test_update_event_review_reviewed_real.py +from sfm.database import SeismoDb +from minimateplus.models import Event +def _ins(db, eid_key="01110000", serial="BE1"): + ev = Event(index=0); ev._waveform_key = bytes.fromhex(eid_key) + db.insert_events([ev], serial=serial) + return db.query_events(serial=serial)[0]["id"] +def test_confirm_real_sets_and_clears_ft(tmp_path): + db = SeismoDb(tmp_path/"s.db"); eid = _ins(db) + db.update_event_review(eid, {"false_trigger": True}) + assert db.get_event(eid)["false_trigger"] == 1 + db.update_event_review(eid, {"reviewed_real": True}) # confirming real clears FT + row = db.get_event(eid) + assert row["reviewed_real"] == 1 and row["false_trigger"] == 0 +def test_flag_ft_clears_reviewed_real(tmp_path): + db = SeismoDb(tmp_path/"s.db"); eid = _ins(db) + db.update_event_review(eid, {"reviewed_real": True}) + db.update_event_review(eid, {"false_trigger": True}) + row = db.get_event(eid) + assert row["false_trigger"] == 1 and row["reviewed_real"] == 0 +``` +> Build the Event inline like `tests/test_zc_freq_columns.py` if the import differs. + +- [ ] **Step 2: Run → FAIL.** +- [ ] **Step 3: Implement** — replace the body of `update_event_review` so it handles both keys: +```python + if not isinstance(review, dict): + return False + has_ft = "false_trigger" in review + has_real = "reviewed_real" in review + if not has_ft and not has_real: + with self._connect() as conn: + row = conn.execute("SELECT 1 FROM events WHERE id=?", (event_id,)).fetchone() + return row is not None + sets = {} + if has_ft: + sets["false_trigger"] = 1 if review.get("false_trigger") else 0 + if has_real: + sets["reviewed_real"] = 1 if review.get("reviewed_real") else 0 + # mutual exclusivity: a true in one forces the other column to 0 + if sets.get("false_trigger") == 1: + sets["reviewed_real"] = 0 + if sets.get("reviewed_real") == 1: + sets["false_trigger"] = 0 + assign = ", ".join(f"{k}=?" for k in sets) + params = list(sets.values()) + [event_id] + with self._connect() as conn: + cur = conn.execute(f"UPDATE events SET {assign} WHERE id=?", params) + return cur.rowcount > 0 +``` +- [ ] **Step 4: Run → PASS** (+ run `tests/test_*false_trigger*`/existing review tests to confirm no regression). +- [ ] **Step 5: Commit** `feat(db): update_event_review mirrors reviewed_real + enforces 3-state exclusivity` + +--- + +### Task 3: `find_twins` matcher + +**Files:** Modify `sfm/database.py` (add `find_twins`). Test: `tests/test_find_twins.py`. + +**Interfaces:** Produces `find_twins(event_id, *, window_seconds=300) -> list[dict]` — same serial, identical peak_vector_sum, timestamp within ±window, excluding self. + +- [ ] **Step 1: Failing test** +```python +# tests/test_find_twins.py +import datetime +from sfm.database import SeismoDb +from minimateplus.models import Event, Timestamp +def _ins(db, key, serial, pvs, ts): + ev = Event(index=0); ev._waveform_key = bytes.fromhex(key) + ev.timestamp = ts + # peak_vector_sum comes from peak_values; simplest: insert then UPDATE pvs directly + db.insert_events([ev], serial=serial) + row = [r for r in db.query_events(serial=serial) if r["waveform_key"] == key][0] + import sqlite3 + with sqlite3.connect(db.db_path) as c: + c.execute("UPDATE events SET peak_vector_sum=? WHERE id=?", (pvs, row["id"])) + return row["id"] +def test_find_twins_matches_same_serial_pvs_near_time(tmp_path): + db = SeismoDb(tmp_path/"s.db") + base = Timestamp(raw=b"", flag=0x10, year=2026, unknown_byte=0, month=2, day=25, hour=20, minute=19, second=5) + twin = Timestamp(raw=b"", flag=0x10, year=2026, unknown_byte=0, month=2, day=25, hour=20, minute=19, second=45) + far = Timestamp(raw=b"", flag=0x10, year=2026, unknown_byte=0, month=2, day=25, hour=21, minute=0, second=0) + a = _ins(db, "01110001", "BE1", 0.4763, base) + b = _ins(db, "01110002", "BE1", 0.4763, twin) # twin: same serial+pvs, 40s apart + c = _ins(db, "01110003", "BE1", 0.4763, far) # same pvs but >window away + d = _ins(db, "01110004", "BE1", 0.9999, twin) # near time but different pvs + ids = {r["id"] for r in db.find_twins(a, window_seconds=300)} + assert ids == {b} +``` +> Adjust the Event/Timestamp construction to match how `tests/test_waveform_store.py::_make_synthetic_event` builds them if fields differ. + +- [ ] **Step 2: Run → FAIL.** +- [ ] **Step 3: Implement** +```python + def find_twins(self, event_id: str, *, window_seconds: int = 300) -> list[dict]: + row = self.get_event(event_id) + if not row: + return [] + serial = row.get("serial"); pvs = row.get("peak_vector_sum"); ts = row.get("timestamp") + if serial is None or pvs is None or not ts: + return [] + try: + t = datetime.datetime.fromisoformat(ts.replace(" ", "T")) + except ValueError: + return [] + lo = (t - datetime.timedelta(seconds=window_seconds)).isoformat() + hi = (t + datetime.timedelta(seconds=window_seconds)).isoformat() + with self._connect() as conn: + rows = conn.execute( + "SELECT * FROM events WHERE serial=? AND id!=? AND peak_vector_sum=? " + "AND timestamp BETWEEN ? AND ?", + (serial, event_id, pvs, lo, hi), + ).fetchall() + return [dict(r) for r in rows] +``` +- [ ] **Step 4: Run → PASS.** +- [ ] **Step 5: Commit** `feat(db): find_twins (serial + identical PVS + time window)` + +--- + +### Task 4: propagate a review to twins + wire into the sidecar-PATCH endpoint + +**Files:** Modify `sfm/database.py` (add `propagate_review_to_twins`); `sfm/server.py` (`db_event_sidecar_patch`). Test: `tests/test_twin_propagation.py`. + +**Interfaces:** `propagate_review_to_twins(event_id, *, window_seconds=300) -> list[str]` copies the event's `false_trigger`/`reviewed_real` columns onto each twin; returns twin ids. + +- [ ] **Step 1: Failing test** (DB-level) +```python +# tests/test_twin_propagation.py — reuse the _ins helper pattern from test_find_twins +def test_propagate_copies_flags_to_twins(tmp_path): + db = SeismoDb(tmp_path/"s.db") + # (build primary + twin via the _ins helper as in test_find_twins) + # flag the primary FT, then propagate: + db.update_event_review(primary_id, {"false_trigger": True}) + moved = db.propagate_review_to_twins(primary_id) + assert twin_id in moved + assert db.get_event(twin_id)["false_trigger"] == 1 +``` +- [ ] **Step 2: Run → FAIL.** +- [ ] **Step 3: Implement** + - `sfm/database.py`: +```python + def propagate_review_to_twins(self, event_id: str, *, window_seconds: int = 300) -> list[str]: + row = self.get_event(event_id) + if not row: + return [] + ft = 1 if row.get("false_trigger") else 0 + real = 1 if row.get("reviewed_real") else 0 + twins = self.find_twins(event_id, window_seconds=window_seconds) + moved = [] + with self._connect() as conn: + for tw in twins: + conn.execute("UPDATE events SET false_trigger=?, reviewed_real=? WHERE id=?", + (ft, real, tw["id"])) + moved.append(tw["id"]) + return moved +``` + - `sfm/server.py` `db_event_sidecar_patch`: after the existing `_get_db().update_event_review(event_id, new_sidecar.get("review", {}))`, add: +```python + # Propagate the review to the event's histogram/waveform twin(s) so + # flagging one flags both (column-level; twins share serial+PVS+near time). + try: + _get_db().propagate_review_to_twins(event_id) + except Exception as exc: + log.warning("twin review-propagation failed for %s: %s", event_id, exc) +``` + (Guard the `if body.review is not None:` block so propagation only runs when review changed.) +- [ ] **Step 4: Run → PASS.** +- [ ] **Step 5: Commit** `feat(review): propagate false_trigger/reviewed_real to twins on sidecar PATCH` + +--- + +### Task 5: expose in feed guard + version bump + +**Files:** Test `tests/test_reviewed_real_in_feed.py`; `pyproject.toml`, `sfm/server.py` version, `CHANGELOG.md`. + +- [ ] **Step 1: Guard test** — a `query_events` row dict includes `reviewed_real` (SELECT * returns it). +```python +from sfm.database import SeismoDb +from minimateplus.models import Event +def test_query_events_includes_reviewed_real(tmp_path): + db = SeismoDb(tmp_path/"s.db") + ev = Event(index=0); ev._waveform_key = bytes.fromhex("01110000") + db.insert_events([ev], serial="BE1") + assert "reviewed_real" in db.query_events(serial="BE1")[0] +``` +- [ ] **Step 2: Run → PASS** (columns already present from Task 1). +- [ ] **Step 3: Bump** `pyproject.toml` 0.24.0 → 0.25.0; `sfm/server.py` version="0.25.0"; add `## v0.25.0` CHANGELOG entry ("reviewed_real 3-state review flag + histogram/waveform twin review-propagation"). +- [ ] **Step 4: Full suite** — confirm zero NEW failures beyond the ~16 pre-existing. +- [ ] **Step 5: Commit** `chore(release): v0.25.0 — reviewed_real + twin review-propagation` + +--- + +## Self-Review + +**Spec coverage:** reviewed_real column (T1) ✓; mirror + mutual exclusivity (T2) ✓; twin match serial+identical-PVS+window (T3) ✓; twin propagation wired into the review path (T4) ✓; feed exposure + version (T5) ✓. Twin **sidecar** rewrite intentionally deferred (column-level only — documented in Global Constraints); this is the SFM half — the 3-state UI + confirm-real PATCH is the separate **B2-B (terra-view)** plan. + +**Placeholder scan:** none — every step has real code (the two "adjust Event construction to match test_waveform_store" notes point at a concrete existing helper). + +**Type consistency:** `false_trigger`/`reviewed_real` INTEGER columns used identically across T1/T2/T4; `find_twins`→`propagate_review_to_twins` both key on the same match; server calls the DB methods by the exact names T2–T4 define. -- 2.54.0 From d9cc5f178077e0630c99526dfe02bbee7e812d04 Mon Sep 17 00:00:00 2001 From: serversdown Date: Tue, 25 Aug 2026 00:26:22 +0000 Subject: [PATCH 10/30] feat(db): reviewed_real column on events (+ auto-migrate) --- sfm/database.py | 3 +++ tests/test_reviewed_real_column.py | 14 ++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 tests/test_reviewed_real_column.py diff --git a/sfm/database.py b/sfm/database.py index 464c2b5..91a0bfc 100644 --- a/sfm/database.py +++ b/sfm/database.py @@ -81,6 +81,7 @@ CREATE TABLE IF NOT EXISTS events ( sample_rate INTEGER, record_type TEXT, -- "single_shot" | "continuous" false_trigger INTEGER NOT NULL DEFAULT 0, -- 0=no, 1=yes (manual flag) + reviewed_real INTEGER NOT NULL DEFAULT 0, -- 0=no, 1=operator-confirmed real (mutually exclusive with false_trigger) blastware_filename TEXT, -- event file within waveform store; extension is per-event (AB0T encodes timestamp) blastware_filesize INTEGER, -- bytes; NULL if no event file saved a5_pickle_filename TEXT, -- ".a5.pkl" sidecar @@ -188,6 +189,7 @@ class SeismoDb: sample_rate INTEGER, record_type TEXT, false_trigger INTEGER NOT NULL DEFAULT 0, + reviewed_real INTEGER NOT NULL DEFAULT 0, created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), UNIQUE(serial, timestamp) ); @@ -224,6 +226,7 @@ class SeismoDb: ("shape_near_peak_count", "INTEGER"), ("shape_sample_count", "INTEGER"), ("shape_axis", "TEXT"), + ("reviewed_real", "INTEGER NOT NULL DEFAULT 0"), ): if col not in existing_cols: log.info("_migrate: events ADD COLUMN %s %s", col, ddl) diff --git a/tests/test_reviewed_real_column.py b/tests/test_reviewed_real_column.py new file mode 100644 index 0000000..3fe153b --- /dev/null +++ b/tests/test_reviewed_real_column.py @@ -0,0 +1,14 @@ +import sqlite3 +from sfm.database import SeismoDb +def _cols(db): + with sqlite3.connect(db.db_path) as c: + return {r[1] for r in c.execute("PRAGMA table_info(events)")} +def test_fresh_db_has_reviewed_real(tmp_path): + assert "reviewed_real" in _cols(SeismoDb(tmp_path/"s.db")) +def test_existing_db_migrates_reviewed_real(tmp_path): + p = tmp_path/"s.db"; db = SeismoDb(p) + with sqlite3.connect(p) as c: + c.execute("ALTER TABLE events DROP COLUMN reviewed_real") + assert "reviewed_real" not in _cols(db) # dropped (read via existing handle/connection) + SeismoDb(p) # re-open migrates + assert "reviewed_real" in _cols(SeismoDb(p)) -- 2.54.0 From 23e4f585a8fce50e15276337a9d5f353a8fa8acf Mon Sep 17 00:00:00 2001 From: serversdown Date: Tue, 25 Aug 2026 00:31:39 +0000 Subject: [PATCH 11/30] fix(db): keep reviewed_real out of the Migration-1 rebuild table (positional SELECT *); regression test --- ...8-25-b2a-reviewed-real-twin-propagation.md | 2 +- sfm/database.py | 1 - tests/test_reviewed_real_column.py | 44 +++++++++++++++++++ 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/docs/superpowers/plans/2026-08-25-b2a-reviewed-real-twin-propagation.md b/docs/superpowers/plans/2026-08-25-b2a-reviewed-real-twin-propagation.md index e19582f..8b7948a 100644 --- a/docs/superpowers/plans/2026-08-25-b2a-reviewed-real-twin-propagation.md +++ b/docs/superpowers/plans/2026-08-25-b2a-reviewed-real-twin-propagation.md @@ -47,8 +47,8 @@ def test_existing_db_migrates_reviewed_real(tmp_path): - [ ] **Step 2: Run → FAIL** (`pytest tests/test_reviewed_real_column.py -q`). - [ ] **Step 3: Implement** - In `_SCHEMA` `events` CREATE TABLE, after the `false_trigger ... DEFAULT 0,` line add: ` reviewed_real INTEGER NOT NULL DEFAULT 0, -- 0=no, 1=operator-confirmed real (mutually exclusive with false_trigger)` - - In the Migration-1 rebuild `CREATE TABLE events (...)` block, add the same `reviewed_real INTEGER NOT NULL DEFAULT 0,` line after `false_trigger`. - In the `_migrate` ADD COLUMN loop tuple add: `("reviewed_real", "INTEGER NOT NULL DEFAULT 0"),` + - **Do NOT** add it to the Migration-1 rebuild `CREATE TABLE events (...)` block — that block uses a positional `INSERT ... SELECT * FROM events_old` and, by convention, contains only the columns that existed when Migration 1 was written; every later column is added by the ADD COLUMN loop only. Adding it there crashes `_migrate` on genuinely legacy (pre-Migration-1) DBs. - [ ] **Step 4: Run → PASS.** - [ ] **Step 5: Commit** `feat(db): reviewed_real column on events (+ auto-migrate)` diff --git a/sfm/database.py b/sfm/database.py index 91a0bfc..846ebcf 100644 --- a/sfm/database.py +++ b/sfm/database.py @@ -189,7 +189,6 @@ class SeismoDb: sample_rate INTEGER, record_type TEXT, false_trigger INTEGER NOT NULL DEFAULT 0, - reviewed_real INTEGER NOT NULL DEFAULT 0, created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), UNIQUE(serial, timestamp) ); diff --git a/tests/test_reviewed_real_column.py b/tests/test_reviewed_real_column.py index 3fe153b..6601e4d 100644 --- a/tests/test_reviewed_real_column.py +++ b/tests/test_reviewed_real_column.py @@ -12,3 +12,47 @@ def test_existing_db_migrates_reviewed_real(tmp_path): assert "reviewed_real" not in _cols(db) # dropped (read via existing handle/connection) SeismoDb(p) # re-open migrates assert "reviewed_real" in _cols(SeismoDb(p)) + +def test_legacy_pre_migration1_db_rebuild_does_not_crash(tmp_path): + # Genuinely legacy (pre-Migration-1) events table: no UNIQUE(serial, timestamp) + # (still on the old UNIQUE(serial, waveform_key)) and columns only through + # false_trigger/created_at — i.e. none of the columns added later by the + # Migration-1b ADD COLUMN loop (blastware_filename, device_family, + # tran_zc_freq, shape_*, reviewed_real, ...). Opening this DB triggers the + # Migration-1 rebuild (`events_old` -> `events` via positional + # `INSERT OR IGNORE INTO events SELECT * FROM events_old`), which requires + # the rebuild's CREATE TABLE to have exactly the legacy column count. + p = tmp_path / "legacy.db" + with sqlite3.connect(p) as c: + c.execute(""" + CREATE TABLE events ( + id TEXT PRIMARY KEY, + serial TEXT NOT NULL, + waveform_key TEXT NOT NULL, + session_id TEXT, + timestamp TEXT, + tran_ppv REAL, + vert_ppv REAL, + long_ppv REAL, + peak_vector_sum REAL, + mic_ppv REAL, + project TEXT, + client TEXT, + operator TEXT, + sensor_location TEXT, + sample_rate INTEGER, + record_type TEXT, + false_trigger INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), + UNIQUE(serial, waveform_key) + ) + """) + c.execute( + "INSERT INTO events (id, serial, waveform_key, timestamp) VALUES (?, ?, ?, ?)", + ("evt-1", "BE11529", "01110000", "2026-01-01T00:00:00"), + ) + # Pre-fix this raised: OperationalError: table events has 19 columns but + # 18 values were supplied (reviewed_real had leaked into the rebuild's + # CREATE TABLE, but events_old — and the positional SELECT * — only had 18). + db = SeismoDb(p) + assert "reviewed_real" in _cols(db) -- 2.54.0 From f73c8eec9105e2d9ccc56a0172def29e08b09618 Mon Sep 17 00:00:00 2001 From: serversdown Date: Tue, 25 Aug 2026 00:34:25 +0000 Subject: [PATCH 12/30] feat(db): update_event_review mirrors reviewed_real + enforces 3-state exclusivity --- sfm/database.py | 36 +++++++++++++------ .../test_update_event_review_reviewed_real.py | 25 +++++++++++++ 2 files changed, 50 insertions(+), 11 deletions(-) create mode 100644 tests/test_update_event_review_reviewed_real.py diff --git a/sfm/database.py b/sfm/database.py index 846ebcf..90f54ea 100644 --- a/sfm/database.py +++ b/sfm/database.py @@ -685,17 +685,23 @@ class SeismoDb: """ Sync derived index columns from a sidecar's `review` block. - Currently the only derived index is `events.false_trigger` — kept - in sync so `/db/events?false_trigger=true` queries don't have to - scan every sidecar JSON on disk. The sidecar JSON itself remains - the source of truth for the full review state. + The derived indexes are `events.false_trigger` and + `events.reviewed_real` — kept in sync so `/db/events` queries don't + have to scan every sidecar JSON on disk. The sidecar JSON itself + remains the source of truth for the full review state. + + The two columns are mutually exclusive: setting either one true + forces the other's column to 0. Returns True when the row exists, False otherwise. No-op fields - (review without `false_trigger`) leave the column untouched. + (review without `false_trigger` or `reviewed_real`) leave both + columns untouched. """ if not isinstance(review, dict): return False - if "false_trigger" not in review: + has_ft = "false_trigger" in review + has_real = "reviewed_real" in review + if not has_ft and not has_real: # Nothing derived to update; just confirm the row exists. with self._connect() as conn: row = conn.execute( @@ -703,12 +709,20 @@ class SeismoDb: ).fetchone() return row is not None - flag = 1 if review.get("false_trigger") else 0 + sets = {} + if has_ft: + sets["false_trigger"] = 1 if review.get("false_trigger") else 0 + if has_real: + sets["reviewed_real"] = 1 if review.get("reviewed_real") else 0 + # mutual exclusivity: a true in one forces the other column to 0 + if sets.get("false_trigger") == 1: + sets["reviewed_real"] = 0 + if sets.get("reviewed_real") == 1: + sets["false_trigger"] = 0 + assign = ", ".join(f"{k}=?" for k in sets) + params = list(sets.values()) + [event_id] with self._connect() as conn: - cur = conn.execute( - "UPDATE events SET false_trigger=? WHERE id=?", - (flag, event_id), - ) + cur = conn.execute(f"UPDATE events SET {assign} WHERE id=?", params) return cur.rowcount > 0 # ── Monitor log ─────────────────────────────────────────────────────────── diff --git a/tests/test_update_event_review_reviewed_real.py b/tests/test_update_event_review_reviewed_real.py new file mode 100644 index 0000000..ea09162 --- /dev/null +++ b/tests/test_update_event_review_reviewed_real.py @@ -0,0 +1,25 @@ +from sfm.database import SeismoDb +from minimateplus.models import Event + + +def _ins(db, eid_key="01110000", serial="BE1"): + ev = Event(index=0); ev._waveform_key = bytes.fromhex(eid_key) + db.insert_events([ev], serial=serial) + return db.query_events(serial=serial)[0]["id"] + + +def test_confirm_real_sets_and_clears_ft(tmp_path): + db = SeismoDb(tmp_path/"s.db"); eid = _ins(db) + db.update_event_review(eid, {"false_trigger": True}) + assert db.get_event(eid)["false_trigger"] == 1 + db.update_event_review(eid, {"reviewed_real": True}) # confirming real clears FT + row = db.get_event(eid) + assert row["reviewed_real"] == 1 and row["false_trigger"] == 0 + + +def test_flag_ft_clears_reviewed_real(tmp_path): + db = SeismoDb(tmp_path/"s.db"); eid = _ins(db) + db.update_event_review(eid, {"reviewed_real": True}) + db.update_event_review(eid, {"false_trigger": True}) + row = db.get_event(eid) + assert row["false_trigger"] == 1 and row["reviewed_real"] == 0 -- 2.54.0 From 5ffa92ab877f66e596ad712008cb0cad30f30483 Mon Sep 17 00:00:00 2001 From: serversdown Date: Tue, 25 Aug 2026 00:38:49 +0000 Subject: [PATCH 13/30] feat(db): find_twins (serial + identical PVS + time window) --- sfm/database.py | 28 ++++++++++++++++++++++++++++ tests/test_find_twins.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 tests/test_find_twins.py diff --git a/sfm/database.py b/sfm/database.py index 90f54ea..c1f596c 100644 --- a/sfm/database.py +++ b/sfm/database.py @@ -603,6 +603,34 @@ class SeismoDb: ).fetchall() return [dict(r) for r in rows] + def find_twins(self, event_id: str, *, window_seconds: int = 300) -> list[dict]: + """ + Find this event's histogram/waveform twin(s): rows sharing the same + serial and an identical peak_vector_sum, whose timestamp falls + within ``window_seconds`` of this event's timestamp. Excludes the + event itself. Returns [] if the event or any required field + (serial / peak_vector_sum / timestamp) is missing. + """ + row = self.get_event(event_id) + if not row: + return [] + serial = row.get("serial"); pvs = row.get("peak_vector_sum"); ts = row.get("timestamp") + if serial is None or pvs is None or not ts: + return [] + try: + t = datetime.datetime.fromisoformat(ts.replace(" ", "T")) + except ValueError: + return [] + lo = (t - datetime.timedelta(seconds=window_seconds)).isoformat() + hi = (t + datetime.timedelta(seconds=window_seconds)).isoformat() + with self._connect() as conn: + rows = conn.execute( + "SELECT * FROM events WHERE serial=? AND id!=? AND peak_vector_sum=? " + "AND timestamp BETWEEN ? AND ?", + (serial, event_id, pvs, lo, hi), + ).fetchall() + return [dict(r) for r in rows] + def set_false_trigger(self, event_id: str, value: bool) -> bool: """Set or clear the false_trigger flag on an event. Returns True if found.""" with self._connect() as conn: diff --git a/tests/test_find_twins.py b/tests/test_find_twins.py new file mode 100644 index 0000000..d0b356c --- /dev/null +++ b/tests/test_find_twins.py @@ -0,0 +1,33 @@ +import datetime +from sfm.database import SeismoDb +from minimateplus.models import Event, Timestamp + + +def _ins(db, key, serial, pvs, ts): + ev = Event(index=0) + ev._waveform_key = bytes.fromhex(key) + ev.timestamp = ts + # peak_vector_sum comes from peak_values; simplest: insert then UPDATE pvs directly + db.insert_events([ev], serial=serial) + row = [r for r in db.query_events(serial=serial) if r["waveform_key"] == key][0] + import sqlite3 + with sqlite3.connect(db.db_path) as c: + c.execute("UPDATE events SET peak_vector_sum=? WHERE id=?", (pvs, row["id"])) + return row["id"] + + +def test_find_twins_matches_same_serial_pvs_near_time(tmp_path): + db = SeismoDb(tmp_path / "s.db") + base = Timestamp(raw=b"", flag=0x10, year=2026, unknown_byte=0, month=2, day=25, hour=20, minute=19, second=5) + twin = Timestamp(raw=b"", flag=0x10, year=2026, unknown_byte=0, month=2, day=25, hour=20, minute=19, second=45) + far = Timestamp(raw=b"", flag=0x10, year=2026, unknown_byte=0, month=2, day=25, hour=21, minute=0, second=0) + # d needs a timestamp distinct from `twin` (UNIQUE(serial, timestamp) would + # otherwise collide with b and UPSERT onto its row instead of inserting a + # new one) while staying near `base` in time. + near = Timestamp(raw=b"", flag=0x10, year=2026, unknown_byte=0, month=2, day=25, hour=20, minute=19, second=44) + a = _ins(db, "01110001", "BE1", 0.4763, base) + b = _ins(db, "01110002", "BE1", 0.4763, twin) # twin: same serial+pvs, 40s apart + c = _ins(db, "01110003", "BE1", 0.4763, far) # same pvs but >window away + d = _ins(db, "01110004", "BE1", 0.9999, near) # near time but different pvs + ids = {r["id"] for r in db.find_twins(a, window_seconds=300)} + assert ids == {b} -- 2.54.0 From 7aae0208f802c87be9a163acb052e062b2f225b8 Mon Sep 17 00:00:00 2001 From: serversdown Date: Tue, 25 Aug 2026 00:42:27 +0000 Subject: [PATCH 14/30] feat(review): propagate false_trigger/reviewed_real to twins on sidecar PATCH --- sfm/database.py | 20 ++++++++++++++++++++ sfm/server.py | 7 +++++++ tests/test_twin_propagation.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+) create mode 100644 tests/test_twin_propagation.py diff --git a/sfm/database.py b/sfm/database.py index c1f596c..c0e8654 100644 --- a/sfm/database.py +++ b/sfm/database.py @@ -631,6 +631,26 @@ class SeismoDb: ).fetchall() return [dict(r) for r in rows] + def propagate_review_to_twins(self, event_id: str, *, window_seconds: int = 300) -> list[str]: + """ + Copy this event's `false_trigger`/`reviewed_real` columns onto each + of its histogram/waveform twins (see `find_twins`), so flagging one + twin flags both. Returns the list of twin ids updated. + """ + row = self.get_event(event_id) + if not row: + return [] + ft = 1 if row.get("false_trigger") else 0 + real = 1 if row.get("reviewed_real") else 0 + twins = self.find_twins(event_id, window_seconds=window_seconds) + moved = [] + with self._connect() as conn: + for tw in twins: + conn.execute("UPDATE events SET false_trigger=?, reviewed_real=? WHERE id=?", + (ft, real, tw["id"])) + moved.append(tw["id"]) + return moved + def set_false_trigger(self, event_id: str, value: bool) -> bool: """Set or clear the false_trigger flag on an event. Returns True if found.""" with self._connect() as conn: diff --git a/sfm/server.py b/sfm/server.py index ea0c92a..6e2ea1a 100644 --- a/sfm/server.py +++ b/sfm/server.py @@ -2481,6 +2481,13 @@ def db_event_sidecar_patch(event_id: str, body: SidecarPatchBody) -> dict: if body.review is not None: _get_db().update_event_review(event_id, new_sidecar.get("review", {})) + # Propagate the review to the event's histogram/waveform twin(s) so + # flagging one flags both (column-level; twins share serial+PVS+near time). + try: + _get_db().propagate_review_to_twins(event_id) + except Exception as exc: + log.warning("twin review-propagation failed for %s: %s", event_id, exc) + return new_sidecar diff --git a/tests/test_twin_propagation.py b/tests/test_twin_propagation.py new file mode 100644 index 0000000..c3f7386 --- /dev/null +++ b/tests/test_twin_propagation.py @@ -0,0 +1,33 @@ +import sqlite3 +from sfm.database import SeismoDb +from minimateplus.models import Event, Timestamp + + +def _ins(db, key, serial, pvs, ts): + ev = Event(index=0) + ev._waveform_key = bytes.fromhex(key) + ev.timestamp = ts + # peak_vector_sum comes from peak_values; simplest: insert then UPDATE pvs directly + db.insert_events([ev], serial=serial) + row = [r for r in db.query_events(serial=serial) if r["waveform_key"] == key][0] + with sqlite3.connect(db.db_path) as c: + c.execute("UPDATE events SET peak_vector_sum=? WHERE id=?", (pvs, row["id"])) + return row["id"] + + +def test_propagate_copies_flags_to_twins(tmp_path): + db = SeismoDb(tmp_path / "s.db") + base = Timestamp(raw=b"", flag=0x10, year=2026, unknown_byte=0, month=2, day=25, hour=20, minute=19, second=5) + twin = Timestamp(raw=b"", flag=0x10, year=2026, unknown_byte=0, month=2, day=25, hour=20, minute=19, second=45) + other = Timestamp(raw=b"", flag=0x10, year=2026, unknown_byte=0, month=2, day=25, hour=20, minute=19, second=44) + + primary_id = _ins(db, "01110001", "BE1", 0.4763, base) + twin_id = _ins(db, "01110002", "BE1", 0.4763, twin) # twin: same serial+pvs, 40s apart + non_twin_id = _ins(db, "01110003", "BE1", 0.9999, other) # near time but different pvs + + db.update_event_review(primary_id, {"false_trigger": True}) + moved = db.propagate_review_to_twins(primary_id) + + assert twin_id in moved + assert db.get_event(twin_id)["false_trigger"] == 1 + assert db.get_event(non_twin_id)["false_trigger"] == 0 -- 2.54.0 From 4a581e0e67cb5a53d84de0b46cf508820d7ee3f6 Mon Sep 17 00:00:00 2001 From: serversdown Date: Tue, 25 Aug 2026 00:45:33 +0000 Subject: [PATCH 15/30] =?UTF-8?q?chore(release):=20v0.25.0=20=E2=80=94=20r?= =?UTF-8?q?eviewed=5Freal=20+=20twin=20review-propagation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 26 ++++++++++++++++++++++++++ pyproject.toml | 2 +- sfm/server.py | 2 +- tests/test_reviewed_real_in_feed.py | 15 +++++++++++++++ 4 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 tests/test_reviewed_real_in_feed.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b4fa04a..e486bae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,32 @@ All notable changes to seismo-relay are documented here. --- +## v0.25.0 — 2026-08-25 + +**reviewed_real 3-state review flag + twin review-propagation.** The +`events` table and the `/db/events` feed now carry `reviewed_real`, a +3-state review flag mutually exclusive with `false_trigger`, mirrored from +the sidecar review block — plus histogram/waveform twin review-propagation +(flagging one flags both, matched by serial + identical PVS + timestamp +window). + +### Added + +- **`events` column** `reviewed_real` — `INTEGER NOT NULL DEFAULT 0`, added + via the existing incremental `_migrate` ADD COLUMN pass (auto-migrates on + `SeismoDb()` construction, no manual migration). `query_events` / + `get_event` (and thus `/db/events`) return it automatically (`SELECT *`). +- **Mutual exclusivity with `false_trigger`** — setting `reviewed_real=1` + clears `false_trigger`, and vice versa, both via `set_false_trigger` / + the sidecar review PATCH path. +- **`find_twins`** — matches an event's histogram/waveform twins by serial + + identical peak-vector-sum + a timestamp window. +- **`propagate_review_to_twins`** — copies an event's `false_trigger`/ + `reviewed_real` state onto its twins, wired into the + `PATCH /db/events/{id}/sidecar` review path so flagging one flags both. + +--- + ## v0.24.0 — 2026-08-22 **Waveform-shape metrics on events.** The `events` table and the `/db/events` diff --git a/pyproject.toml b/pyproject.toml index 2a51e33..ed0cc90 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "seismo-relay" -version = "0.24.0" +version = "0.25.0" description = "Python client and REST server for MiniMate Plus seismographs" requires-python = ">=3.10" dependencies = [ diff --git a/sfm/server.py b/sfm/server.py index 6e2ea1a..c658196 100644 --- a/sfm/server.py +++ b/sfm/server.py @@ -90,7 +90,7 @@ app = FastAPI( "Implements the minimateplus RS-232 protocol library.\n" "Proxied by terra-view at /api/sfm/*." ), - version="0.24.0", + version="0.25.0", ) # Allow requests from the waveform viewer opened as a local file (file://) diff --git a/tests/test_reviewed_real_in_feed.py b/tests/test_reviewed_real_in_feed.py new file mode 100644 index 0000000..42483c0 --- /dev/null +++ b/tests/test_reviewed_real_in_feed.py @@ -0,0 +1,15 @@ +from __future__ import annotations +import os, sys +from pathlib import Path + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from sfm.database import SeismoDb +from minimateplus.models import Event + + +def test_query_events_includes_reviewed_real(tmp_path: Path): + db = SeismoDb(tmp_path / "s.db") + ev = Event(index=0) + ev._waveform_key = bytes.fromhex("01110000") + db.insert_events([ev], serial="BE1") + assert "reviewed_real" in db.query_events(serial="BE1")[0] -- 2.54.0 From 37043a47e99cf4508535966be0e1bd5df9b2ad22 Mon Sep 17 00:00:00 2001 From: serversdown Date: Tue, 25 Aug 2026 00:55:38 +0000 Subject: [PATCH 16/30] fix(review): quick FT path enforces 3-state exclusivity + twin propagation; changelog + twin caveat set_false_trigger now clears reviewed_real when flagging false_trigger=True (mirrors update_event_review's exclusivity), and the quick PATCH /db/events/{id}/false_trigger endpoint now calls propagate_review_to_twins after the flag write, matching the sidecar PATCH path's try/except-with-log.warning pattern. Previously the quick path could leave both flags set and never touched twins. Also corrects the v0.25.0 CHANGELOG bullet (exclusivity was not actually enforced on the quick path until this commit) and adds a caveat comment on find_twins about rare clamped/saturated-PVS false-positive twin matches. --- CHANGELOG.md | 11 ++++--- sfm/database.py | 33 +++++++++++++++++---- sfm/server.py | 11 ++++++- tests/test_set_false_trigger_exclusivity.py | 29 ++++++++++++++++++ 4 files changed, 74 insertions(+), 10 deletions(-) create mode 100644 tests/test_set_false_trigger_exclusivity.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e486bae..402dd9f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,13 +24,16 @@ window). `SeismoDb()` construction, no manual migration). `query_events` / `get_event` (and thus `/db/events`) return it automatically (`SELECT *`). - **Mutual exclusivity with `false_trigger`** — setting `reviewed_real=1` - clears `false_trigger`, and vice versa, both via `set_false_trigger` / - the sidecar review PATCH path. + clears `false_trigger`, and vice versa, enforced on both review paths: + the sidecar review PATCH (`update_event_review`) and the quick + `PATCH /db/events/{id}/false_trigger` endpoint (`set_false_trigger`). - **`find_twins`** — matches an event's histogram/waveform twins by serial + identical peak-vector-sum + a timestamp window. - **`propagate_review_to_twins`** — copies an event's `false_trigger`/ - `reviewed_real` state onto its twins, wired into the - `PATCH /db/events/{id}/sidecar` review path so flagging one flags both. + `reviewed_real` state onto its twins, wired into both the + `PATCH /db/events/{id}/sidecar` review path and the quick + `PATCH /db/events/{id}/false_trigger` path, so flagging one flags both + regardless of which endpoint made the change. --- diff --git a/sfm/database.py b/sfm/database.py index c0e8654..6986208 100644 --- a/sfm/database.py +++ b/sfm/database.py @@ -610,6 +610,16 @@ class SeismoDb: within ``window_seconds`` of this event's timestamp. Excludes the event itself. Returns [] if the event or any required field (serial / peak_vector_sum / timestamp) is missing. + + Caveat: identical-PVS matching is a proxy for "same physical event + recorded twice," not a guarantee. In the rare case where the device + clamps/saturates PVS (clamped to sqrt(3) * geo_range), two distinct + saturated events on the same serial within the window can share the + same clamped PVS value and be matched as twins even though they are + different events. This is harmless in practice — false_trigger/ + reviewed_real are derived/index columns re-derivable from the + sidecar source of truth — but worth knowing if twin counts look + surprising on a saturated/clamped run. """ row = self.get_event(event_id) if not row: @@ -652,12 +662,25 @@ class SeismoDb: return moved def set_false_trigger(self, event_id: str, value: bool) -> bool: - """Set or clear the false_trigger flag on an event. Returns True if found.""" + """ + Set or clear the false_trigger flag on an event. Returns True if found. + + Mirrors the 3-state exclusivity enforced by `update_event_review`: + setting false_trigger true also clears `reviewed_real` (a false + trigger can't also be a confirmed-real event). Clearing false_trigger + leaves `reviewed_real` untouched. + """ with self._connect() as conn: - cur = conn.execute( - "UPDATE events SET false_trigger=? WHERE id=?", - (1 if value else 0, event_id), - ) + if value: + cur = conn.execute( + "UPDATE events SET false_trigger=1, reviewed_real=0 WHERE id=?", + (event_id,), + ) + else: + cur = conn.execute( + "UPDATE events SET false_trigger=0 WHERE id=?", + (event_id,), + ) return cur.rowcount > 0 def delete_event(self, event_id: str) -> Optional[dict]: diff --git a/sfm/server.py b/sfm/server.py index c658196..eabc45a 100644 --- a/sfm/server.py +++ b/sfm/server.py @@ -1957,7 +1957,10 @@ def db_set_false_trigger( value: bool = Query(..., description="True to flag as false trigger, False to clear"), ) -> dict: """ - Set or clear the false_trigger flag on a single event. + Set or clear the false_trigger flag on a single event. Enforces the + same 3-state exclusivity as the sidecar review PATCH (flagging false_trigger + clears reviewed_real) and propagates the resulting flags to the event's + histogram/waveform twin(s), same as the sidecar path. Used by the terra-view event review UI. Returns 404 if the event_id is not found. @@ -1965,6 +1968,12 @@ def db_set_false_trigger( found = _get_db().set_false_trigger(event_id, value) if not found: raise HTTPException(status_code=404, detail=f"Event {event_id} not found") + + try: + _get_db().propagate_review_to_twins(event_id) + except Exception as exc: + log.warning("twin review-propagation failed for %s: %s", event_id, exc) + return {"status": "ok", "event_id": event_id, "false_trigger": value} diff --git a/tests/test_set_false_trigger_exclusivity.py b/tests/test_set_false_trigger_exclusivity.py new file mode 100644 index 0000000..e72f865 --- /dev/null +++ b/tests/test_set_false_trigger_exclusivity.py @@ -0,0 +1,29 @@ +from sfm.database import SeismoDb +from minimateplus.models import Event + + +def _ins(db, eid_key="01110000", serial="BE1"): + ev = Event(index=0); ev._waveform_key = bytes.fromhex(eid_key) + db.insert_events([ev], serial=serial) + return db.query_events(serial=serial)[0]["id"] + + +def test_set_false_trigger_clears_reviewed_real(tmp_path): + db = SeismoDb(tmp_path / "s.db"); eid = _ins(db) + db.update_event_review(eid, {"reviewed_real": True}) + assert db.get_event(eid)["reviewed_real"] == 1 + + db.set_false_trigger(eid, True) + row = db.get_event(eid) + assert row["false_trigger"] == 1 and row["reviewed_real"] == 0 + + +def test_clearing_false_trigger_leaves_reviewed_real_untouched(tmp_path): + db = SeismoDb(tmp_path / "s.db"); eid = _ins(db) + db.update_event_review(eid, {"false_trigger": True}) + db.set_false_trigger(eid, False) + row = db.get_event(eid) + assert row["false_trigger"] == 0 + # reviewed_real was never set true, so it stays 0 either way; the point + # is set_false_trigger(False) does not touch the column at all. + assert row["reviewed_real"] == 0 -- 2.54.0 From 686ab6e7a61d775b8545c3e6bcb4d8ce7f3f3a50 Mon Sep 17 00:00:00 2001 From: serversdown Date: Tue, 25 Aug 2026 08:11:11 +0000 Subject: [PATCH 17/30] fix(codec): geo full scale is 32000 counts; 4 walker framing cases; channel-id from header MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two independent bugs, both found by diffing 75 production events against their preserved Blastware ASCII exports (//_ASCII.TXT). 1. Geo full scale was wrong — every geophone reading was 2.34% low. The codec emits geo samples in 16-count units with a documented LSB of exactly 0.005 in/s, and decoded_to_adc_counts multiplies by 16, so one ADC count is 0.005/16 in/s and 10.000 in/s is 10.0/(0.005/16) = 32000 counts. sfm/event_hdf5.py and minimateplus/event_file_io.py both divided by 32768 (2^15), scaling every sample and derived peak down by 1 - 32000/32768. The error scales with amplitude, so it was invisible on quiet events and worst on the loud ones that matter for compliance. Mic is unaffected (it back-solves its scale from the device peak). 216 per-channel comparisons: 32768 -> 151/216 exact, worst error 0.238 in/s on a 10 in/s event; 32000 -> 216/216 exact, worst 0.005 = 1 LSB. 2. walk_body silently truncated channels on four unhandled framing cases. An unrecognised tag ends the walk and decode_waveform_v2 returns whatever it got, so this surfaced as short channels, never an error: - wide-NN RLE `0X NN` (runs longer than 252 samples) - `30 NN` with NN > 0x10 (the old cap was arbitrary) - variable-width `40 NN` headers: NN counts previous-channel continuation deltas, so the header is 2*NN + 16 bytes; `40 01` and `40 03` occur alongside `40 02` - tagless segment headers: no `40 NN` tag at all, just the 14-byte tail [field2:2][len:2][channel_id:4][marker:2][anchors:4] Also: the header field documented as a "monotonic uint32 LE counter" is really [channel_id][00][00][segment_index], with 0x46=Tran 0x47=Vert 0x48=Long 0x49=MicL — verified on 1697/1697 segment headers, zero disagreements. decode_waveform_v2 now takes the channel from that field instead of rotation position, which was fragile: one missed header desynced every channel after it. parse_segment_header now returns n_prev_deltas/prev_deltas/marker/ anchors/channel/segment_index; the old fixed_pattern (02 00 00 01) conflated the 2-byte marker with the first anchor. Ground-truth corpus, end to end through the production path: exact 37 -> 72, truncated 23 -> 3, full-length value errors 15 -> 0. Store-wide, 729 of 1388 series-3 waveform events decode differently and 728 gain samples; the scale fix changes float values on all of them, so stored .h5 files need regenerating. Still open: 3 events truncate at a header variant with a variable-width prefix (2/4/6 bytes) before the channel id and an `01 00` marker. Documented in docs/instantel_protocol_reference.md with byte offsets. +20 tests. No regressions: the byte-exact fixture suite still passes and the full-suite failure list is unchanged from baseline (16 pre-existing failures from gitignored fixtures). Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HgTe8CamXAHcAmaQ6QNcog --- CHANGELOG.md | 93 +++++++++++++++ CLAUDE.md | 8 ++ docs/instantel_protocol_reference.md | 157 +++++++++++++++++++++--- docs/waveform_codec_re_status.md | 60 +++++++++- minimateplus/event_file_io.py | 9 +- minimateplus/waveform_codec.py | 171 +++++++++++++++++++++------ sfm/event_hdf5.py | 23 +++- tests/test_event_file_io.py | 9 ++ tests/test_event_hdf5.py | 46 ++++++- tests/test_waveform_codec.py | 137 ++++++++++++++++++++- 10 files changed, 645 insertions(+), 68 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 402dd9f..2f7d1c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,99 @@ All notable changes to seismo-relay are documented here. ## [Unreleased] +### Fixed + +- **Geophone full scale is 32000 ADC counts, not 32768 — every geo reading was + 2.3% low.** The verified body codec emits geo samples in 16-count units whose + documented LSB is exactly 0.005 in/s, and `decoded_to_adc_counts` multiplies + by 16, so one ADC count is `0.005/16` in/s and Normal range (10.000 in/s) is + `10.0 / (0.005/16)` = **32000** counts. Both `sfm/event_hdf5.py` and + `minimateplus/event_file_io.py` divided by 32768, scaling every geophone + sample and every derived peak down by `1 - 32000/32768` = **2.34%**. + + Measured against 216 per-channel comparisons with preserved Blastware ASCII + exports: **32768 → 151/216 exact** (worst error 0.238 in/s on a 10 in/s + event); **32000 → 216/216 exact**, worst error 0.005 in/s (exactly 1 LSB — + pure quantization). The error scales with amplitude, so it was invisible on + quiet events and worst on the loud ones that matter for compliance. + + The mic path is unaffected — it back-solves its own per-count factor from the + device-reported peak. + +- **Series-3 waveform codec: four block-framing cases caused silent channel + truncation.** `walk_body` hit its unknown-tag `break` mid-stream and every + channel decoded after that point came out short — typically Vert/Long/MicL, + sometimes at a third of their true length, with no error raised. + + - **Wide-NN RLE `0X NN`** — the 12-bit NN encoding already handled for + `1X NN` / `2X NN` also applies to the `00 NN` RLE tag. Runs longer than + 252 samples must use the wide form (e.g. `01 0c` = 268 repeats). + - **`30 NN` with NN > 0x10** — the `0 < NN <= 0x10` guard was arbitrary; + data-section `30 NN` blocks reach at least NN = 0x18. The length formula + (`NN × 1.5 + 2`) was already correct. + - **Variable-width `40 NN` segment headers** — NN is the *count of + previous-channel continuation deltas*, so the header is `2 × NN + 16` + bytes and every field after the deltas shifts by `2 × NN`. Only `40 02` + (20 bytes) was handled; `40 01` (18) and `40 03` (22) both occur. + - **Tagless segment headers** — a segment header can appear with no + `40 NN` tag at all: just the 14-byte tail + `[field2:2][len:2][channel_id:4][marker:2][anchors:4]`. This is the NN=0 + case (no continuation deltas needed, so no tag and no delta bytes). It is + where the walk stopped in 7 of the 8 events still truncating after the + first three fixes. + +### Changed + +- **Segment channel now comes from the header's own channel-id byte** rather + than from rotation position. The field previously documented as a + "monotonic uint32 LE counter" is really `[channel][00][00][segment_index]` + with `0x46`=Tran `0x47`=Vert `0x48`=Long `0x49`=MicL — verified on + **1697 of 1697** segment headers across the ground-truth corpus with zero + disagreements. Rotation-by-position is kept only as a fallback for unknown + ids; it was fragile because a single missed or extra header (exactly what + tagless headers caused) desynced every channel after it. + +- **`parse_segment_header` return shape** — now `n_prev_deltas`, + `prev_deltas`, `marker`, `anchors`, `channel`, `segment_index` in place of + the fixed-offset `anchor_bytes` / `fixed_pattern` / `tail` keys. The old + `fixed_pattern` (`02 00 00 01`) conflated the 2-byte constant marker with + the first anchor. `counter` is retained as the raw uint32 of the id field. + +### Verification + +Against the 75 ground-truth events (BW binary paired with its preserved +`_ASCII.TXT` export), decoding end-to-end through the production path: + +| | before | after | +|---|---|---| +| exact (full length, within 1 LSB) | 37 | **72** | +| truncated | 23 | **3** | +| full length, value error > 2 LSB | 15 | **0** | + +Worst remaining error among the 72: 0.0050 in/s = exactly 1 LSB. + +No regressions — the byte-exact fixture suite still passes, and the full-suite +failure list is unchanged from baseline (16 pre-existing failures from +gitignored fixtures). + +### Notes + +- **The "DC offset" symptom is _not_ a decode bug.** Events whose geo trace + sits at a constant level instead of oscillating around zero + (dominant-axis `|mean| / peak` >> 0) reproduce *exactly* in Blastware's own + ASCII export — e.g. `BE12599/N599LQD7.8E0W` Tran reads mean +0.345, + min +0.335, max +0.355 in both. It is a known recurring hardware fault (the + operators call it an "offset"): the affected channel's baseline exceeds the + unit's own geo trigger level, so the unit retriggers continuously and floods + the ACH queue with garbage events. Store-wide it affects 2 units of 21 across + 6 episodes; see `scratch/offset_candidates.csv` and the project memory notes. + +- **Still open:** 3 of 75 ground-truth events truncate at a segment-header + variant with a variable-width prefix before the channel-id field (2, 4 or 6 + bytes observed) and an `01 00` marker instead of `02 00`. See the protocol + reference, "Unmapped: variable-prefix segment descriptors". Examples: + `BE12599/N599LPNB.JF0W` at body offset 1155, `BE9558/K558LOF2.820W` at 1485. + --- ## v0.25.0 — 2026-08-25 diff --git a/CLAUDE.md b/CLAUDE.md index 9198786..1e5b2cb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -264,6 +264,14 @@ then `decoded_to_adc_counts()` to scale to int16 ADC counts (geos × 16; mic pass-through). The `.h5` sidecars SFM produces now contain correct samples for any event without walker edge cases. +**Geo full scale is 32000 ADC counts, NOT 32768** (fixed 2026-08-25). +One decoder unit = 16 ADC counts = exactly 0.005 in/s, so +`10.000 in/s / (0.005/16)` = 32000. Consumers must use +`sfm.event_hdf5._GEO_INT16_FS` / `event_file_io._GEO_INT16_FS` (both +32000). Dividing by 32768 reads every geophone sample 2.34% low — +that was a live bug in both modules until 2026-08-25. Mic is +unaffected (it back-solves its scale from the device-reported peak). + The original int16 LE decoder is preserved as `_decode_a5_waveform_LEGACY` for reference but is not called. diff --git a/docs/instantel_protocol_reference.md b/docs/instantel_protocol_reference.md index e10f9da..6c25d27 100644 --- a/docs/instantel_protocol_reference.md +++ b/docs/instantel_protocol_reference.md @@ -1101,14 +1101,51 @@ Every block starts with a 2-byte tag. Five tag types are confirmed: |-----------|-------------------------------------|-----------------------| | ``10 NN`` | Small-delta data block | NN/2 + 2 bytes | | ``20 NN`` | Literal data block (int8-shaped) | NN + 2 bytes | -| ``00 NN`` | 2-byte marker between data blocks | 2 bytes | +| ``00 NN`` | RLE zero-delta run | 2 bytes | | ``30 NN`` | Trailer summary block | NN × 4 bytes | -| ``40 02`` | Segment header | 20 bytes (fixed) | +| ``40 NN`` | Segment header | 2 × NN + 16 bytes | NN is always a multiple of 4. ``10 NN`` and ``20 NN`` data blocks alternate with ``00 NN`` markers — every ``10/20 NN`` block is followed by a ``00 NN`` marker before the next data block. +###### Wide-NN forms — ``0X NN`` (CONFIRMED 2026-08-25) + +The 12-bit wide-NN encoding already documented for ``1X NN`` / +``2X NN`` (low nibble of the tag byte carries the high nibble of NN, +so effective ``NN = ((tag & 0x0F) << 8) | NN``) **also applies to the +``00 NN`` RLE tag.** A narrow RLE run maxes out at NN = 0xFC, so a +quiet stretch longer than 252 samples must use the wide form. + +Confirmed against six production events, e.g. ``01 0c`` (NN = 268) in +``BE9558/K558LKOF.460W``. Before this was handled, the walker hit its +unknown-tag break at the first long quiet run and silently truncated +every channel decoded after that point. + +###### ``30 NN`` is not capped at NN = 0x10 (CONFIRMED 2026-08-25) + +Data-section ``30 NN`` blocks occur with NN up to at least 0x18 (24), +e.g. ``30 18`` in ``BE18193/T193LQ45.NN0W`` and ``30 14`` in +``BE18193/T193LQ9W.AF0W``. The data-section length formula +(``NN × 1.5 + 2``) holds for these; only the earlier ``NN ≤ 0x10`` +guard was wrong. + +###### ``40 NN`` segment headers are variable width (CONFIRMED 2026-08-25) + +``40 02`` is the common case, but **NN is the count of int16 BE +continuation deltas the header carries for the *previous* channel**, so +the header grows with NN and every field after the deltas shifts by +``2 × NN``: + +``` +length = 2 (tag) + 2 × NN (prev-channel deltas) + 14 (fixed tail) +``` + +``40 01`` (18 bytes) and ``40 03`` (22 bytes) both occur in production +files — see ``BE12599/N599LP1S.UO0W`` and ``BE18438/T438LO30.GA0W``. +In each case the constant ``02 00`` marker sits at ``data[2×NN+8]`` and +the following tag lands exactly on a valid block boundary. + ##### Segments The body is divided into segments separated by ``40 02`` segment headers. @@ -1129,20 +1166,110 @@ fit fewer. Observed first-segment sizes in the bundled fixtures: based on incomplete walks; that figure is wrong. Segments are flash-page-sized in bytes, not sample-count-sized. -The 18-byte ``40 02`` payload structure: +The ``40 NN`` payload structure (offsets shown for the common NN=2 / +18-byte-payload case; add ``2 × (NN − 2)`` to every offset from ``[4:6]`` +onward for other widths): -| Offset | Field | Status | -|-----------|---------------------------------------------|-------------| -| [0:2] | T_delta at first sample of new segment | ✅ confirmed| -| | (int16 BE, in 16-count units) | | -| [2:4] | Likely T_delta at sample seg_start+1 | 🟡 likely | -| [4:6] | Unknown (varies; possibly a checksum) | ❓ open | -| [6:8] | Byte length to next segment header − 2 | ✅ confirmed| -| | (uint16 BE; useful for walker pre-scan) | | -| [8:12] | Monotonic uint32 LE counter | ✅ confirmed| -| | (starts ~0x47, increments by 1 per segment) | | -| [12:14] | Constant ``02 00`` | ✅ confirmed| -| [14:18] | Unknown 4-byte field | ❓ open | +| Offset (NN=2) | Generic | Field | Status | +|---------------|------------------|----------------------------------------|-------------| +| [0:4] | [0 : 2NN] | NN × int16 BE continuation deltas for | ✅ confirmed| +| | | the PREVIOUS channel (16-count units) | | +| [4:6] | [2NN : 2NN+2] | Unknown (varies; possibly a checksum) | ❓ open | +| [6:8] | [2NN+2 : 2NN+4] | Byte length to next segment header − 2 | 🟡 likely | +| | | (uint16 BE; off by ±4 on some files) | | +| [8:12] | [2NN+4 : 2NN+8] | Monotonic uint32 LE counter | ✅ confirmed| +| | | (starts ~0x47, +1 per segment) | | +| [12:14] | [2NN+8 : 2NN+10] | Constant ``02 00`` | ✅ confirmed| +| [14:18] | [2NN+10 : 2NN+14]| THIS channel's 2-sample anchor pair | ✅ confirmed| +| | | (2 × int16 BE) | | + +⚠️ An earlier draft listed ``[14:18]`` as an "unknown 4-byte field" and +``[12:16]`` as a constant ``02 00 00 01``. Both were wrong: the constant +is only the 2-byte ``02 00``, and the four bytes after it are the anchor +pair the decoder needs. Corrected 2026-08-25. + +###### Tagless segment headers (CONFIRMED 2026-08-25) + +A segment header can appear with **no ``40 NN`` tag at all** — just the +14-byte tail: + +``` +[field2:2][len_to_next:2][channel_id:4][marker:2][anchors:4] +``` + +This is the NN=0 case: the previous channel needed no continuation +deltas, so there is no tag and no delta bytes. Detect it by the six +bytes at ``[4:10]`` — a known channel id, two zero bytes, a small +segment index, then the ``01 00`` / ``02 00`` marker. + +It is where the walk stopped in 7 of the 8 events that still truncated +after the wide-RLE / ``30 NN`` / variable-width-``40 NN`` fixes. + +###### The header "counter" is really a channel id (CONFIRMED 2026-08-25) + +The 4-byte field previously documented as a *"monotonic uint32 LE +counter (starts ~0x47, increments by 1 per segment)"* is actually: + +``` +[channel_id:1][00][00][segment_index:1] +``` + +| channel_id | channel | +|---|---| +| ``0x46`` | Tran | +| ``0x47`` | Vert | +| ``0x48`` | Long | +| ``0x49`` | MicL | + +Verified on **1697 of 1697** segment headers across the ground-truth +corpus — every one agrees with the channel the rotation would assign, +zero disagreements, no other id values observed. The old reading was +plausible because the id byte cycles 0x46→0x47→0x48→0x49 and the +segment index increments, which *looks* monotonic in LE. + +Decoders should take the channel from this field rather than from +rotation position: one missed or extra header (exactly what tagless +headers used to cause) desyncs rotation and corrupts every channel +after it. + +###### Geophone full scale is 32000 counts, not 32768 (CONFIRMED 2026-08-25) + +The body codec emits geo samples in 16-count units whose LSB is exactly +**0.005 in/s**. With the consumer-side ``×16`` to ADC counts, one ADC +count is ``0.005 / 16`` in/s, so Normal range (10.000 in/s) is + +``` +10.0 / (0.005 / 16) = 32000 counts +``` + +Dividing by 32768 scales every geophone sample and every derived peak +down by ``1 - 32000/32768`` = **2.34%**. Measured on 216 per-channel +comparisons against preserved Blastware ASCII exports: 32768 gave +151/216 exact (worst error 0.238 in/s on a 10 in/s event); 32000 gives +216/216 exact with a worst error of 0.005 in/s — exactly 1 LSB, i.e. +pure quantization. + +This also explains why Blastware reports geo peaks slightly above +nominal full scale (e.g. 10.14 in/s): the ADC has headroom past 32000. + +###### Unmapped: variable-prefix segment descriptors ❓ OPEN + +Three of 75 ground-truth production events still truncate. In each, +the walk reaches a segment header whose channel-id field is preceded by +a **variable-width prefix** — 2, 4 or 6 bytes have all been observed, +where the standard tagless form always has 4 (``field2`` + ``len``). +These records also carry the ``01 00`` marker rather than ``02 00``, +and appear packed back-to-back with little or no sample data between +them. + +The ``01 00`` marker is *not* simply an anchor count: records carrying +it have been seen with both 2-byte and 4-byte anchor fields in the same +file, so the prefix width and the marker are not yet reconciled. + +The decoder stops cleanly at these rather than emitting garbage. +Examples: ``BE12599/N599LPNB.JF0W`` at body offset 1155 (2-byte +prefix), ``BE12599/N599LPWJ.980W`` at 849 (6-byte prefix), +``BE9558/K558LOF2.820W`` at 1485. Examples from event-c (1 sec single-shot): diff --git a/docs/waveform_codec_re_status.md b/docs/waveform_codec_re_status.md index dd1a6c6..47fbc41 100644 --- a/docs/waveform_codec_re_status.md +++ b/docs/waveform_codec_re_status.md @@ -102,12 +102,24 @@ correct. | | | nibble first; signed 0..7 / 8..F = -8..-1)| | `20 NN` | NN + 2 bytes | int8 signed deltas (1 per byte) | | `00 NN` | 2 bytes | RLE: append NN copies of current value | -| `30 NN` | NN*2 in data section, | Unknown content. Only in loud-from- | -| | NN*4 in trailer | start events. | -| `40 02` | 20 bytes (fixed) | Segment header | +| `30 NN` | NN*1.5 + 2 in data | 12-bit signed deltas (see below). | +| | section, NN*4 trailer | | +| `40 NN` | 2*NN + 16 bytes | Segment header (NN = prev-channel deltas)| NN is always a multiple of 4. +**Wide-NN forms.** `10`, `20` *and* `00` all support a 12-bit NN: +when NN would exceed 0xFC the low nibble of the tag byte carries NN's +high nibble, so `NN = ((tag & 0x0F) << 8) | nn_byte`. Confirmed for +`1X`/`2X` in 2026-05-11 and for `0X` (RLE) in 2026-08-25 — e.g. +`01 0c` = a 268-sample zero-delta run. + +**`40 NN` is variable width.** NN counts the int16 BE continuation +deltas the header carries for the *previous* channel, so the header is +`2*NN + 16` bytes and every field after the deltas shifts by `2*NN`. +`40 01` (18 B) and `40 03` (22 B) both occur alongside the common +`40 02` (20 B). Confirmed 2026-08-25. + Implementation: `walk_body()` in `minimateplus/waveform_codec.py`. ### 7-byte preamble @@ -207,6 +219,48 @@ TL;DR table above are now locked in by pytest regression tests. still bails out partway through. Lower priority since the other 7 events walk cleanly. +4. **Variable-prefix segment descriptors** (found 2026-08-25). + 3 of 75 ground-truth production events still truncate. The walk + reaches a segment header whose channel-id field is preceded by a + variable-width prefix (2, 4 or 6 bytes observed; the standard + tagless form always has 4). These also carry an `01 00` marker + instead of `02 00`. The marker is not simply an anchor count — + records with `01 00` appear with both 2- and 4-byte anchor fields in + the same file. Examples: `BE12599/N599LPNB.JF0W` @1155, + `BE12599/N599LPWJ.980W` @849, `BE9558/K558LOF2.820W` @1485. + +## Segment header: channel id and tagless form — 2026-08-25 + +The 4-byte field previously read as a "monotonic uint32 LE counter" is +`[channel_id][00][00][segment_index]`, with `0x46`=Tran `0x47`=Vert +`0x48`=Long `0x49`=MicL. Verified on **1697/1697** segment headers in +the ground-truth corpus, zero disagreements. `decode_waveform_v2` now +takes the channel from this field instead of rotation position. + +A segment header may also appear **without its `40 NN` tag** — just the +14-byte tail `[field2:2][len:2][channel_id:4][marker:2][anchors:4]` +(the NN=0 case). `is_tagless_segment_header()` detects it from the six +bytes at `[4:10]`. + +## Geo scale: full scale is 32000 counts — 2026-08-25 + +One decoder unit (16 ADC counts) is exactly 0.005 in/s, so Normal range +(10.000 in/s) is `10.0 / (0.005/16)` = **32000** ADC counts. Consumers +that divided by 32768 read every geophone sample 2.34% low. Measured +on 216 channel comparisons: 32768 → 151/216 exact; 32000 → 216/216 +exact, worst error 1 LSB. + +## Ground-truth corpus (2026-08-25) + +Beyond the bundled fixtures, the production waveform store keeps each +event's original Blastware ASCII export at +`//_ASCII.TXT`. 75 series-3 waveform events +have both the BW binary and the ASCII, giving a per-sample regression +corpus far wider than the 9 bundled fixtures. Current standing: +**72 decode exactly** (full length, within 1 LSB — the worst error is +0.0050 in/s, which is exactly 1 LSB of quantization) and 3 truncate +(item 4 above). Zero events have full-length value errors. + ## `30 NN` block format — CRACKED 2026-05-11 late The `30 NN` block carries `NN` 12-bit signed deltas, packed as `NN/4` diff --git a/minimateplus/event_file_io.py b/minimateplus/event_file_io.py index 00cb43e..5d3a01f 100644 --- a/minimateplus/event_file_io.py +++ b/minimateplus/event_file_io.py @@ -659,6 +659,11 @@ def file_sha256(path: Union[str, Path], chunk_size: int = 65536) -> str: _GEO_NORMAL_FS_INS = 10.0 _GEO_SENSITIVE_FS_INS = 1.250 _INT16_FS = 32768.0 +# Geophone full-scale count — 32000, not 32768. One decoder unit (16 ADC +# counts) is exactly 0.005 in/s, so 10.000 in/s = 32000 counts. Must match +# sfm.event_hdf5._GEO_INT16_FS or sidecar peaks disagree with the plotted +# waveform by 2.3%. Confirmed 2026-08-25 against the BW ASCII corpus. +_GEO_INT16_FS = 32000.0 # Microphone scale factor, psi per ADC count. Approximate — exact factor # depends on the geophone-vs-mic ADC scaling and the firmware reference. @@ -728,7 +733,7 @@ def _peaks_from_samples(samples: dict[str, list[int]]) -> PeakValues: if not ch: return 0.0 m = max(abs(int(v)) for v in ch) - return m / _INT16_FS * _GEO_NORMAL_FS_INS + return m / _GEO_INT16_FS * _GEO_NORMAL_FS_INS tran = _peak_ins(samples.get("Tran", [])) vert = _peak_ins(samples.get("Vert", [])) @@ -742,7 +747,7 @@ def _peaks_from_samples(samples: dict[str, list[int]]) -> PeakValues: pvs = 0.0 n = min(len(samples.get("Tran", [])), len(samples.get("Vert", [])), len(samples.get("Long", []))) if n: - scale = _GEO_NORMAL_FS_INS / _INT16_FS + scale = _GEO_NORMAL_FS_INS / _GEO_INT16_FS T = samples["Tran"]; V = samples["Vert"]; L = samples["Long"] for i in range(n): t = T[i] * scale diff --git a/minimateplus/waveform_codec.py b/minimateplus/waveform_codec.py index c0b40ab..0c96422 100644 --- a/minimateplus/waveform_codec.py +++ b/minimateplus/waveform_codec.py @@ -166,8 +166,14 @@ def find_data_start(body: bytes) -> int: # Try fixed offset 7 first (canonical preamble length). if len(body) >= 9: b, nn = body[7], body[8] - if (b in (0x00, 0x10, 0x20, 0x30) and nn % 4 == 0 and 0 < nn <= 0xFC) \ - or (b == 0x40 and nn == 0x02): + # Accept the same tag vocabulary ``walk_body`` accepts, including the + # wide-NN forms (``0X``/``1X``/``2X``) and the variable-width ``40 NN`` + # segment header. + if ((b & 0xF0) in (0x00, 0x10, 0x20) and nn % 4 == 0 + and ((b & 0x0F) != 0 or 0 < nn <= 0xFC)) \ + or (b == 0x30 and nn % 4 == 0 and 0 < nn <= 0xFC) \ + or (b == 0x40 and 0 < nn <= 0x08) \ + or is_tagless_segment_header(body, 7): return 7 # Fall back to scanning the first 20 bytes. for i in range(min(20, len(body) - 1)): @@ -178,6 +184,31 @@ def find_data_start(body: bytes) -> int: return -1 +# Channel-id byte carried in every segment header. Previously mis-read as a +# "monotonic uint32 LE counter"; it is really ``[channel][00][00][segment]``. +# Verified 2026-08-25 on 1697/1697 segment headers across the ground-truth +# corpus with zero disagreements against the decoded channel rotation. +SEGMENT_CHANNEL_IDS = {0x46: "Tran", 0x47: "Vert", 0x48: "Long", 0x49: "MicL"} + +# A tagless segment header: the 14-byte tail of a ``40 NN`` header with no tag +# and no previous-channel continuation deltas (the NN=0 case). +_TAGLESS_HEADER_LEN = 14 + + +def is_tagless_segment_header(body: bytes, i: int) -> bool: + """True if a bare 14-byte segment header starts at *i*. + + Layout ``[field2:2][len_to_next:2][channel_id:4][marker:2][anchors:4]``. + The discriminator is the 6 bytes at ``[4:10]``: a known channel id, two + zero bytes, a small segment index, and the ``01 00`` / ``02 00`` marker. + """ + if i + _TAGLESS_HEADER_LEN > len(body): + return False + return (body[i + 4] in SEGMENT_CHANNEL_IDS + and body[i + 5] == 0x00 and body[i + 6] == 0x00 + and body[i + 8] in (0x01, 0x02) and body[i + 9] == 0x00) + + def walk_body(body: bytes, start: Optional[int] = None) -> List[WaveformBlock]: """Walk the tagged-block sequence starting at *start* (auto-detected by default). @@ -210,9 +241,15 @@ def walk_body(body: bytes, start: Optional[int] = None) -> List[WaveformBlock]: # Wide-NN int8 block: ``2X NN`` extends NN to 12 bits the same way. wide_nn = ((t0 & 0x0F) << 8) | t1 length = wide_nn + 2 - elif t0 == 0x00 and t1 % 4 == 0: + elif (t0 & 0xF0) == 0x00 and t1 % 4 == 0: + # ``00 NN`` RLE zero-delta run, plus its wide form ``0X NN`` + # (X != 0) which extends NN to 12 bits exactly like ``1X``/``2X``: + # NN = ((t0 & 0x0F) << 8) | t1. A narrow run maxes out at + # NN=0xFC, so quiet stretches longer than 252 samples must use + # the wide form. Confirmed 2026-08-25 against six production + # events (e.g. ``01 0c`` = 268 repeats in K558LKOF.460W). length = 2 - elif t0 == 0x30 and t1 % 4 == 0 and 0 < t1 <= 0x10: + elif t0 == 0x30 and t1 % 4 == 0 and 0 < t1 <= 0xFC: # Data-section ``30 NN`` blocks carry NN 12-bit signed deltas packed # as NN/4 groups of (2-byte high-nibble field + 4 × int8 low byte). # Length = NN/4 × 6 + 2 = NN × 1.5 + 2 (= 8 for NN=4, 14 for NN=8, @@ -229,8 +266,28 @@ def walk_body(body: bytes, start: Optional[int] = None) -> List[WaveformBlock]: length = cand_data else: length = cand_trailer - elif t0 == 0x40 and t1 == 0x02: - length = 20 + elif t0 == 0x40 and 0 < t1 <= 0x08: + # ``40 NN`` segment header. NN is the number of int16 BE + # continuation deltas the header carries for the PREVIOUS + # channel, so the header grows with NN: + # length = 2 (tag) + 2*NN (deltas) + 14 (fixed tail) + # ``40 02`` (20 bytes) dominates, but ``40 01`` (18) and + # ``40 03`` (22) both occur in production files. Confirmed + # 2026-08-25; the constant ``02 00`` marker moves with NN too + # (see :func:`parse_segment_header`). + length = 2 * t1 + 16 + elif is_tagless_segment_header(body, i): + # Segment header with no ``40 NN`` tag (NN=0 — the previous channel + # needed no continuation deltas). Emit it as a synthetic ``40 00`` + # block whose ``data`` is the whole 14-byte record, so the nd=0 + # offsets in :func:`decode_waveform_v2` line up unchanged. + blocks.append(WaveformBlock( + offset=i, tag_hi=0x40, tag_lo=0x00, + data=bytes(body[i : i + _TAGLESS_HEADER_LEN]), + length=_TAGLESS_HEADER_LEN, + )) + i += _TAGLESS_HEADER_LEN + continue else: # Unknown tag; stop. Caller can inspect ``i`` to see where. break @@ -256,7 +313,7 @@ def split_segments(blocks: List[WaveformBlock]) -> List[List[WaveformBlock]]: segments: List[List[WaveformBlock]] = [] current: List[WaveformBlock] = [] for b in blocks: - if b.tag_hi == 0x40 and b.tag_lo == 0x02: + if b.tag_hi == 0x40: if current: segments.append(current) current = [b] @@ -268,23 +325,40 @@ def split_segments(blocks: List[WaveformBlock]) -> List[List[WaveformBlock]]: def parse_segment_header(block: WaveformBlock) -> Optional[dict]: - """Decode the 18-byte payload of a ``40 02`` segment header. + """Decode the payload of a ``40 NN`` segment header. - Returns a dict with the labelled fields, or None if *block* is not - a ``40 02`` header. + NN (the tag's low byte) is the number of int16 BE continuation deltas + the header carries for the PREVIOUS channel, so every field after + those deltas shifts by ``2 * NN``. The payload is ``2 * NN + 14`` + bytes. ``40 02`` is the common case; ``40 01`` and ``40 03`` also + occur in production files (confirmed 2026-08-25). + + Returns a dict with the labelled fields, or None if *block* is not a + segment header or is too short. """ - if not (block.tag_hi == 0x40 and block.tag_lo == 0x02): + if block.tag_hi != 0x40 or block.tag_lo > 0x08: return None - if len(block.data) < 18: + nd = block.tag_lo + if len(block.data) < 2 * nd + 14: return None p = block.data - counter = int.from_bytes(p[8:12], "little", signed=False) + counter = int.from_bytes(p[2 * nd + 4 : 2 * nd + 8], "little", signed=False) return { - "anchor_bytes": p[0:4], # 4-byte field, role unconfirmed - "field2": p[4:8], # 4-byte field, role unconfirmed - "counter": counter, # uint32 LE — increments by 1 per segment - "fixed_pattern": p[12:16], # always b"\x02\x00\x00\x01" - "tail": p[16:18], # last 2 bytes + "n_prev_deltas": nd, + # ``nd`` int16 BE deltas extending the previous channel. + "prev_deltas": [ + int.from_bytes(p[2 * k : 2 * k + 2], "big", signed=True) + for k in range(nd) + ], + "field2": p[2 * nd : 2 * nd + 4], # 4-byte field, role unconfirmed + "counter": counter, # legacy: raw uint32 LE of the id field + "channel": SEGMENT_CHANNEL_IDS.get(p[2 * nd + 4]), + "segment_index": p[2 * nd + 7], + "marker": p[2 * nd + 8 : 2 * nd + 10], # always b"\x02\x00" + "anchors": [ + int.from_bytes(p[2 * nd + 10 : 2 * nd + 12], "big", signed=True), + int.from_bytes(p[2 * nd + 12 : 2 * nd + 14], "big", signed=True), + ], } @@ -420,8 +494,11 @@ def decode_waveform_v2(body: bytes) -> Optional[dict]: for byte in blk.data: cur += _i8(byte) out[channel].append(cur) - elif blk.tag_hi == 0x00: - for _ in range(blk.tag_lo): + elif (blk.tag_hi & 0xF0) == 0x00: + # RLE zero-delta run. Wide form ``0X NN`` carries the high + # nibble of a 12-bit NN in the tag byte, same as ``1X``/``2X``. + run = ((blk.tag_hi & 0x0F) << 8) | blk.tag_lo + for _ in range(run): out[channel].append(cur) elif blk.tag_hi == 0x30: # 12-bit signed deltas, packed as NN/4 groups of 6 bytes each: @@ -461,34 +538,54 @@ def decode_waveform_v2(body: bytes) -> Optional[dict]: # previous-channel extension deltas at every segment boundary. last_value = {"Tran": last_tran_value, "Vert": None, "Long": None, "MicL": None} + prev_channel = "Tran" for k, hi in enumerate(seg_idx): - channel = rotation[k % 4] - prev_channel = "Tran" if k == 0 else rotation[(k - 1) % 4] header = blocks[hi] - if len(header.data) < 18: + # Channel comes from the header's own id byte, which is authoritative. + # The old rotation-by-position fallback is kept for headers whose id + # byte isn't one of the four known values — but a single missed or + # extra header would desync rotation and corrupt every later channel, + # which is exactly what tagless headers used to cause. + _nd = header.tag_lo + channel = None + if len(header.data) >= 2 * _nd + 8: + channel = SEGMENT_CHANNEL_IDS.get(header.data[2 * _nd + 4]) + if channel is None: + channel = rotation[k % 4] + # ``40 NN``: NN int16 BE continuation deltas for the previous channel + # come first, so every later field shifts by 2*NN. NN is usually 2 + # but 1 and 3 both occur (confirmed 2026-08-25). + nd = header.tag_lo + if len(header.data) < 2 * nd + 14: continue - # Validate: real segment headers have bytes [12:14] = `02 00`. - # Trailer/footer "40 02" markers contain ASCII serial bytes or other - # non-header data there and would otherwise be mis-interpreted as - # segment headers, adding spurious samples at the tail. - if header.data[12:14] != b"\x02\x00": + # Validate: real segment headers have the constant `02 00` marker + # right after the counter. Trailer/footer "40 NN" markers contain + # ASCII serial bytes or other non-header data there and would + # otherwise be mis-read as segment headers, adding spurious tail + # samples. + if header.data[2 * nd + 8 : 2 * nd + 10] != b"\x02\x00": break - # Extend the PREVIOUS channel by 2 more samples (deltas in bytes [0:4]). - prev_d0 = int.from_bytes(header.data[0:2], "big", signed=True) - prev_d1 = int.from_bytes(header.data[2:4], "big", signed=True) + # Extend the PREVIOUS channel by NN more samples. if last_value[prev_channel] is not None: - v = last_value[prev_channel] + prev_d0 - out[prev_channel].append(v) - v += prev_d1 - out[prev_channel].append(v) + v = last_value[prev_channel] + for d in range(nd): # NB: not `k` — that's the segment index + v += int.from_bytes( + header.data[2 * d : 2 * d + 2], "big", signed=True + ) + out[prev_channel].append(v) last_value[prev_channel] = v # Anchor pair for THIS segment's channel. - c0 = int.from_bytes(header.data[14:16], "big", signed=True) - c1 = int.from_bytes(header.data[16:18], "big", signed=True) + c0 = int.from_bytes( + header.data[2 * nd + 10 : 2 * nd + 12], "big", signed=True + ) + c1 = int.from_bytes( + header.data[2 * nd + 12 : 2 * nd + 14], "big", signed=True + ) out[channel].extend([c0, c1]) # Apply delta blocks for this segment. next_hi = seg_idx[k + 1] if k + 1 < len(seg_idx) else len(blocks) last_value[channel] = apply_blocks(channel, c1, hi + 1, next_hi) + prev_channel = channel return out diff --git a/sfm/event_hdf5.py b/sfm/event_hdf5.py index aa6e6f0..a25b34d 100644 --- a/sfm/event_hdf5.py +++ b/sfm/event_hdf5.py @@ -77,6 +77,20 @@ _GEO_FS_BY_RANGE = { } _INT16_FS = 32768.0 +# Geophone full-scale count. NOT 32768: the verified body codec emits geo +# samples in 16-count units whose documented LSB is exactly 0.005 in/s, and +# ``waveform_codec.decoded_to_adc_counts`` multiplies by 16 — so one ADC count +# is 0.005/16 in/s and Normal range (10.000 in/s) is 10.0 / (0.005/16) = 32000 +# counts. Using 32768 here made every geophone reading 2.3% low +# (1 - 32000/32768 = 0.0234). +# +# Confirmed 2026-08-25 against 216 per-channel comparisons with the preserved +# Blastware ASCII exports: 32000 gives 216/216 exact within 1 LSB (worst error +# 0.005 in/s); 32768 gave 151/216 with a worst error of 0.238 in/s on a +# 10 in/s event. The mic path is unaffected — it back-solves its own scale +# from the device-reported peak (see _mic_scale_factor). +_GEO_INT16_FS = 32000.0 + # Default mic conversion: ADC count → psi. Approximate; exact factor # depends on firmware reference voltage and mic sensitivity, neither of # which is independently confirmed. We try to refine it from the device- @@ -125,15 +139,14 @@ def _samples_to_float( ) -> np.ndarray: """Convert int16 ADC counts → float32 physical units. - Uses _INT16_FS=32768 (not 32767) so that a count of -32768 maps to - exactly -full_scale and +32767 maps to ~+full_scale * 32767/32768. - Matches the device firmware's documented mapping (see CLAUDE.md - geo_hardware_constant rationale). + Uses _GEO_INT16_FS=32000 (see the constant's rationale): one decoder + unit (16 ADC counts) is exactly 0.005 in/s, so full scale is 32000 + counts, not 32768. """ if not samples_int16: return np.array([], dtype=np.float32) arr = np.asarray(samples_int16, dtype=np.int32) # int32 to avoid overflow during scale - return (arr.astype(np.float32) * (full_scale / _INT16_FS)).astype(np.float32) + return (arr.astype(np.float32) * (full_scale / _GEO_INT16_FS)).astype(np.float32) def _mic_scale_factor( diff --git a/tests/test_event_file_io.py b/tests/test_event_file_io.py index 0e043e8..2b0c0d4 100644 --- a/tests/test_event_file_io.py +++ b/tests/test_event_file_io.py @@ -626,3 +626,12 @@ if __name__ == "__main__": failed += 1 print(f"\n{passed} passed, {failed} failed") sys.exit(0 if failed == 0 else 1) + + +def test_peaks_from_samples_uses_32000_full_scale(): + """`_peaks_from_samples` must use the same 32000-count geo full scale as + the .h5 writer, or sidecar peaks disagree with the plotted waveform by + 2.3%. See test_event_hdf5.test_geo_full_scale_count_is_32000.""" + from minimateplus.event_file_io import _peaks_from_samples + pv = _peaks_from_samples({"Tran": [32000], "Vert": [0], "Long": [0], "MicL": []}) + assert abs(pv.tran - 10.0) < 1e-4 diff --git a/tests/test_event_hdf5.py b/tests/test_event_hdf5.py index 86c3336..0560f5d 100644 --- a/tests/test_event_hdf5.py +++ b/tests/test_event_hdf5.py @@ -99,8 +99,14 @@ def test_hdf5_round_trip_preserves_metadata(tmp_path: Path): def test_hdf5_samples_in_physical_units_normal_range(tmp_path: Path): - """Vert hits ADC full-scale (32767) → with Normal range FS=10 in/s, - the HDF5 sample value should be ≈ 10 * 32767/32768 in/s.""" + """Vert hits 32767 ADC counts → with Normal range FS=10 in/s that is + ``10 * 32767/32000`` in/s. + + Geo full scale is 32000 counts, not 32768 (see + test_geo_full_scale_count_is_32000), so 32767 counts sits slightly + ABOVE nominal full scale -- the ADC has headroom past 10.000 in/s, + which is why Blastware reports peaks like 10.14 in/s. This test + previously asserted the 32768 scale and was wrong by 2.3%.""" ev = _make_event_with_samples() h5 = tmp_path / "n.h5" event_hdf5.write_event_hdf5(h5, ev, serial="BE11529", geo_range="normal") @@ -110,7 +116,7 @@ def test_hdf5_samples_in_physical_units_normal_range(tmp_path: Path): assert vert.dtype.name == "float32" assert max(abs(v) for v in vert) > 9.99 # full-scale ≈ 10.0 # The dirac was at n//2 → 32767 ADC counts. - expected_peak = 10.0 * 32767 / 32768 + expected_peak = 10.0 * 32767 / 32000 assert abs(max(vert) - expected_peak) < 1e-3 @@ -122,7 +128,7 @@ def test_hdf5_samples_in_physical_units_sensitive_range(tmp_path: Path): data = event_hdf5.read_event_hdf5(h5) vert = data["samples"]["Vert"] - expected_peak = 1.250 * 32767 / 32768 + expected_peak = 1.250 * 32767 / 32000 assert abs(max(vert) - expected_peak) < 1e-4 @@ -294,3 +300,35 @@ if __name__ == "__main__": failed += 1 print(f"\n{passed} passed, {failed} failed") sys.exit(0 if failed == 0 else 1) + + +# ── Geophone full-scale count ─────────────────────────────────────────────── + +def test_geo_full_scale_count_is_32000(): + """Geo full scale is 32000 ADC counts, not 32768. + + The verified body codec emits geo samples in 16-count units with a + documented LSB of exactly 0.005 in/s, and ``decoded_to_adc_counts`` + multiplies by 16 — so one ADC count is 0.005/16 in/s and Normal range + (10.000 in/s) is 10.0 / (0.005/16) = 32000 counts. + + Using 32768 made every geophone reading 2.3% low (1 - 32000/32768). + Confirmed 2026-08-25 against 216 channel comparisons with preserved + Blastware ASCII exports: 32000 → 216/216 exact within 1 LSB; + 32768 → 151/216, worst error 0.238 in/s on a 10 in/s event. + """ + from sfm.event_hdf5 import _GEO_INT16_FS + assert _GEO_INT16_FS == 32000.0 + + +def test_samples_to_float_lsb_is_exactly_5_milli_ips(): + """One decoder unit (= 16 ADC counts) must be exactly 0.005 in/s.""" + from sfm.event_hdf5 import _samples_to_float + out = _samples_to_float([16], 10.0) + assert abs(float(out[0]) - 0.005) < 1e-9 + + +def test_samples_to_float_full_scale_count_maps_to_full_scale(): + from sfm.event_hdf5 import _samples_to_float + assert abs(float(_samples_to_float([32000], 10.0)[0]) - 10.0) < 1e-4 + assert abs(float(_samples_to_float([32000], 1.25)[0]) - 1.25) < 1e-5 diff --git a/tests/test_waveform_codec.py b/tests/test_waveform_codec.py index ffd84ca..2d3290f 100644 --- a/tests/test_waveform_codec.py +++ b/tests/test_waveform_codec.py @@ -210,9 +210,11 @@ def test_parse_segment_header_decodes_fields(): ) decoded = parse_segment_header(block) assert decoded is not None + assert decoded["n_prev_deltas"] == 2 + assert decoded["prev_deltas"] == [0, 0] assert decoded["counter"] == 0x47 # uint32 LE - assert decoded["fixed_pattern"] == b"\x02\x00\x00\x01" - assert decoded["anchor_bytes"] == b"\x00\x00\x00\x00" + assert decoded["marker"] == b"\x02\x00" + assert decoded["anchors"] == [1, 1] def test_segment_counter_increments(): @@ -516,3 +518,134 @@ def test_decode_a5_frames_empty(): from minimateplus.waveform_codec import decode_a5_frames assert decode_a5_frames([]) is None assert decode_a5_frames(None) is None + + +# ── Wide-NN RLE, wide 30 NN, and variable-width segment headers ────────────── +# +# Three framing cases discovered 2026-08-25 by diffing 75 production events +# against their preserved Blastware ASCII exports. Each caused ``walk_body`` +# to hit its ``else: break`` mid-stream, truncating every channel decoded +# after that point (see CHANGELOG v0.25.1). + +_PREAMBLE = b"\x00\x02\x00\x00\x00\x00\x00" # magic + Tran[0]=0, Tran[1]=0 +_STOP = b"\xff\xff" # unrecognised tag → walker stops + + +def _synth(*chunks: bytes) -> bytes: + return _PREAMBLE + b"".join(chunks) + _STOP + + +def test_walk_body_wide_rle_block(): + """``0X NN`` is a 12-bit-NN RLE run (NN = ((t0 & 0x0F) << 8) | t1). + + Observed as ``01 0c`` (NN=268) in BE9558/K558LKOF.460W and five other + production events. A narrow ``00 NN`` maxes out at NN=0xFC, so runs + longer than 252 samples must use the wide form. + """ + blocks = walk_body(_synth(b"\x01\x0c")) + assert len(blocks) == 1 + assert (blocks[0].tag_hi, blocks[0].tag_lo) == (0x01, 0x0C) + assert blocks[0].length == 2 + + +def test_decode_wide_rle_repeats_full_run(): + """A wide RLE run repeats the running value NN times, not NN & 0xFF.""" + decoded = decode_waveform_v2(_synth(b"\x01\x0c")) + # 2 preamble anchors + 268 repeats + assert len(decoded["Tran"]) == 2 + 268 + assert set(decoded["Tran"]) == {0} + + +def test_walk_body_30_block_nn_above_16(): + """``30 NN`` data blocks are not capped at NN=0x10. + + ``30 18`` (NN=24) appears in BE18193/T193LQ45.NN0W; the old + ``0 < t1 <= 0x10`` guard rejected it and stopped the walk 1033 bytes + into a 4877-byte body. Length is still NN * 1.5 + 2. + """ + payload = bytes(36) # 24 deltas × 1.5 bytes + blocks = walk_body(_synth(b"\x30\x18" + payload, b"\x00\x04")) + assert [b.length for b in blocks] == [38, 2] + + +@pytest.mark.parametrize("nn,hdr_len", [(1, 18), (2, 20), (3, 22)]) +def test_walk_body_segment_header_width_follows_tag_lo(nn, hdr_len): + """``40 NN``: NN is the count of previous-channel continuation deltas. + + Header length = 2 * NN + 16. ``40 02`` (the only form previously + handled) is the NN=2 case at 20 bytes; ``40 01`` (18) and ``40 03`` + (22) both occur in production files. + """ + data = bytearray(hdr_len - 2) + data[2 * nn + 8 : 2 * nn + 10] = b"\x02\x00" # constant marker + blocks = walk_body(_synth(bytes([0x40, nn]) + bytes(data), b"\x00\x04")) + assert [b.length for b in blocks] == [hdr_len, 2] + + +@pytest.mark.parametrize("nn,hdr_len", [(1, 18), (2, 20), (3, 22)]) +def test_segment_header_anchors_track_header_width(nn, hdr_len): + """Anchor pair sits at data[2*NN+10 : 2*NN+14] regardless of width.""" + data = bytearray(hdr_len - 2) + data[2 * nn + 8 : 2 * nn + 10] = b"\x02\x00" + data[2 * nn + 10 : 2 * nn + 12] = (7).to_bytes(2, "big") # anchor 0 + data[2 * nn + 12 : 2 * nn + 14] = (9).to_bytes(2, "big") # anchor 1 + decoded = decode_waveform_v2(_synth(bytes([0x40, nn]) + bytes(data))) + assert decoded["Vert"][:2] == [7, 9] + + +# ── Tagless segment headers ───────────────────────────────────────────────── +# +# A segment header can appear WITHOUT its ``40 NN`` tag: just the 14-byte tail +# ``[field2:2][len:2][channel_id:4][marker:2][anchors:4]``. This is the NN=0 +# case — no continuation deltas for the previous channel, so no tag and no +# delta bytes. Found 2026-08-25: it is where the walk stopped in 7 of the 8 +# remaining truncating production events. +# +# The channel_id field (previously mis-labelled a "monotonic counter") is +# ``[channel][00][00][segment_index]`` with 0x46=Tran 0x47=Vert 0x48=Long +# 0x49=MicL — verified on 1697 of 1697 segment headers across the ground-truth +# corpus, zero disagreements. + +def _tagless(chan_id=0x47, seg=2, marker=b"\x02\x00", a0=0, a1=0): + return (b"\x5d\xee" + b"\x00\xd0" + bytes([chan_id, 0, 0, seg]) + marker + + a0.to_bytes(2, "big", signed=True) + a1.to_bytes(2, "big", signed=True)) + + +def test_walk_body_accepts_tagless_segment_header(): + """A bare 14-byte header is walked as a segment block, not a stop.""" + blocks = walk_body(_synth(b"\x10\x04\x00\x00", _tagless(), b"\x00\x04")) + kinds = [(b.tag_hi, b.tag_lo, b.length) for b in blocks] + assert kinds == [(0x10, 0x04, 4), (0x40, 0x00, 14), (0x00, 0x04, 2)] + + +def test_tagless_header_carries_full_14_bytes_as_data(): + """The synthetic block's data includes the leading bytes (there is no tag + to strip), so decode_waveform_v2's ``2*nd + k`` offsets line up at nd=0.""" + blocks = walk_body(_synth(_tagless())) + hdr = next(b for b in blocks if b.tag_hi == 0x40) + assert len(hdr.data) == 14 + assert hdr.data[8:10] == b"\x02\x00" # marker at 2*0 + 8 + + +def test_tagless_header_anchors_and_channel_id(): + """Anchors decode from data[10:14]; the channel comes from the id byte.""" + decoded = decode_waveform_v2(_synth(_tagless(chan_id=0x48, a0=11, a1=13))) + assert decoded["Long"][:2] == [11, 13] # 0x48 → Long, not rotation + assert decoded["Vert"] == [] + + +@pytest.mark.parametrize("chan_id,name", + [(0x46, "Tran"), (0x47, "Vert"), (0x48, "Long"), (0x49, "MicL")]) +def test_segment_channel_comes_from_id_not_rotation(chan_id, name): + """Channel is taken from the header's id byte. Two headers in a row for + the SAME channel must both land on that channel — rotation-by-position + would put the second one on the next channel and corrupt both.""" + body = _synth(_tagless(chan_id=chan_id, seg=1, a0=5, a1=6), + _tagless(chan_id=chan_id, seg=2, a0=7, a1=8)) + decoded = decode_waveform_v2(body) + # Tran additionally carries the body preamble's 2 anchors (both 0 here). + expected = [0, 0, 5, 6, 7, 8] if name == "Tran" else [5, 6, 7, 8] + assert decoded[name] == expected + for other in ("Tran", "Vert", "Long", "MicL"): + if other != name: + assert decoded[other] == ([0, 0] if other == "Tran" else []) -- 2.54.0 From b6b6ee0331cbc9a1cd59c6f861ad07369207c0b5 Mon Sep 17 00:00:00 2001 From: serversdown Date: Tue, 25 Aug 2026 09:16:23 +0000 Subject: [PATCH 18/30] docs(changelog): record that the 32000 scale fix hit histograms and series-4 too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The scale lives in _samples_to_float, which every event passes through regardless of source codec, so waveforms, histograms and Thor IDF events were all 2.34% low — not just waveforms. Verified after regeneration: series-3 histogram peaks vs ASCII reports now median 1.0000 across 1137 comparisons (0.9766 under 32768); series-4 peaks vs device peaks moved from median 0.960 to 0.983 across 1468. The four block-framing fixes remain waveform-only; histogram_codec is untouched. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HgTe8CamXAHcAmaQ6QNcog --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f7d1c0..a2a7819 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,16 @@ All notable changes to seismo-relay are documented here. The mic path is unaffected — it back-solves its own per-count factor from the device-reported peak. + **Scope:** the scale lives in `_samples_to_float`, which every event passes + through regardless of which codec produced the samples — so this affected + **waveforms, histograms and series-4 (Thor IDF) alike**, not just waveforms. + Verified after regeneration: series-3 histogram peaks vs their ASCII reports + now sit at a median ratio of 1.0000 across 1,137 comparisons (0.9766 under + 32768); series-4 peaks vs device peaks moved from a median 0.960 to 0.983 + across 1,468 comparisons. The four block-framing fixes below are + waveform-only — histograms decode via `histogram_codec.decode_histogram_body`, + which is untouched. + - **Series-3 waveform codec: four block-framing cases caused silent channel truncation.** `walk_body` hit its unknown-tag `break` mid-stream and every channel decoded after that point came out short — typically Vert/Long/MicL, -- 2.54.0 From 0f6c9d930f6c6bd47c9f01a022725f69ddd67d95 Mon Sep 17 00:00:00 2001 From: serversdown Date: Tue, 25 Aug 2026 14:21:59 +0000 Subject: [PATCH 19/30] data: offset-candidate event list from the 2026-08-25 survey MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 274 series-3 waveform events across 6 episodes on 2 units (BE9558, BE18438) whose dominant geo axis sits pinned at a DC offset above that unit's own geo trigger level — the "offset" hardware fault that makes a unit retrigger continuously and flood the ACH queue. Detection rule: dominant-axis |mean|/peak > 0.7 AND |mean| >= 0.9 x the unit's geo trigger level. Bare |mean|/peak is useless on quiet events — a trace at the 0.010 in/s noise floor clears any ratio threshold. Not a decode artifact: these reproduce exactly in Blastware's own ASCII export. Kept as the starting point for the archive-wide analysis. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HgTe8CamXAHcAmaQ6QNcog --- scratch/offset_candidates.csv | 275 ++++++++++++++++++++++++++++++++++ 1 file changed, 275 insertions(+) create mode 100644 scratch/offset_candidates.csv diff --git a/scratch/offset_candidates.csv b/scratch/offset_candidates.csv new file mode 100644 index 0000000..af9f5ee --- /dev/null +++ b/scratch/offset_candidates.csv @@ -0,0 +1,275 @@ +event_id,serial,timestamp,filename,channel,offset_ips,peak_ips,mean_over_peak,trigger_level_ips,already_flagged_ft +1e7a7808-034c-423b-80e9-d6788da80993,BE18438,2025-11-15T08:57:40,T438LBX4.W40W,Vert,0.2722,0.2930,0.929,0.2,0 +346e383f-c5a1-41d1-b972-3d652e599d39,BE18438,2025-11-15T09:13:15,T438LBX5.M30W,Vert,0.2747,0.2930,0.938,0.2,0 +45ef3023-22c5-44a8-a606-a0900bc17861,BE18438,2025-11-15T09:16:13,T438LBX5.R10W,Vert,0.2929,0.3027,0.967,0.2,0 +1cbcd2aa-ec55-4e2f-af42-7178f26cc6a1,BE18438,2025-11-15T09:19:29,T438LBX5.WH0W,Vert,0.2816,0.2979,0.945,0.2,0 +caf43634-4528-4fd5-9674-5f4f563661c0,BE18438,2025-11-15T09:22:32,T438LBX6.1K0W,Vert,0.2883,0.2930,0.984,0.2,0 +456ae646-c834-4e5c-82f1-7df18b3440a1,BE18438,2025-11-15T09:25:31,T438LBX6.6J0W,Vert,0.3189,0.3223,0.989,0.2,0 +afb91ba2-351b-42a0-b33f-acebd5f1829d,BE18438,2025-11-15T09:28:29,T438LBX6.BH0W,Vert,0.3057,0.3076,0.994,0.2,0 +39847146-3537-4300-943d-01c0c771e9cd,BE18438,2025-11-15T09:31:26,T438LBX6.GE0W,Vert,0.3225,0.3271,0.986,0.2,0 +e47335c1-fc3f-4c90-88c6-8edd0f296ce4,BE18438,2025-11-15T09:34:24,T438LBX6.LC0W,Vert,0.3275,0.3320,0.986,0.2,0 +25b6eda6-ba98-422f-8834-94b6cbe34971,BE18438,2025-11-15T09:37:22,T438LBX6.QA0W,Vert,0.3398,0.3418,0.994,0.2,0 +7af99377-e47f-4914-8e91-ac7f7f50b8c6,BE18438,2025-11-15T09:40:20,T438LBX6.V80W,Vert,0.3515,0.3564,0.986,0.2,0 +c4330605-c257-49ef-aede-20deb641e7d7,BE18438,2025-11-15T10:09:48,T438LBX8.8C0W,Vert,0.3605,0.3955,0.912,0.2,0 +b665caf9-64ee-4352-a391-60c1ddeebc1e,BE18438,2026-02-25T10:58:04,T438LH66.GS0W,Vert,0.1803,0.1953,0.923,0.2,0 +3b0c32c3-fd16-4b83-9917-942a56dea168,BE18438,2026-02-25T18:12:04,T438LH6Q.K40W,Vert,0.1869,0.1953,0.957,0.2,0 +ca66e601-d10f-4424-9c26-08a688e7b08c,BE18438,2026-02-25T18:17:16,T438LH6Q.SS0W,Vert,0.1870,0.1953,0.958,0.2,0 +31aa30ba-eb0a-49a9-98ca-c77d2cc31275,BE18438,2026-02-25T18:21:11,T438LH6Q.ZB0W,Vert,0.1868,0.1953,0.956,0.2,0 +fdb55a93-fc91-4fdf-a7f4-fdd1a6b48f59,BE18438,2026-02-25T18:26:27,T438LH6R.830W,Vert,0.1893,0.1953,0.969,0.2,0 +cbd718d7-2378-4d74-8656-842f229d19a8,BE18438,2026-02-25T18:33:25,T438LH6R.JP0W,Vert,0.1887,0.1953,0.966,0.2,0 +bfaf58ec-6163-47eb-9d2a-70da880bd948,BE18438,2026-02-25T18:37:48,T438LH6R.R00W,Vert,0.1872,0.1953,0.959,0.2,0 +ddaa530b-886e-4415-b93a-f4ebf4c4a882,BE18438,2026-02-25T18:59:54,T438LH6S.RU0W,Vert,0.1874,0.1953,0.959,0.2,0 +57f15284-ab69-48fc-9660-8c347c80c681,BE18438,2026-02-25T19:15:58,T438LH6T.IM0W,Vert,0.1873,0.1953,0.959,0.2,0 +03715718-7aa8-4e1d-9b0b-c2ab75fc1975,BE18438,2026-02-25T19:19:58,T438LH6T.PA0W,Vert,0.1891,0.1953,0.968,0.2,0 +9f921cd5-1bf9-4713-aef6-41e7f780eff1,BE18438,2026-02-25T19:24:02,T438LH6T.W20W,Vert,0.1871,0.1953,0.958,0.2,0 +44994949-58d4-44fa-bd78-ed75b8acc667,BE18438,2026-02-25T19:44:57,T438LH6U.UX0W,Vert,0.1877,0.1953,0.961,0.2,0 +c4be01c3-1001-4e17-aecd-d6d26dbb09f3,BE18438,2026-02-25T19:49:19,T438LH6V.270W,Vert,0.1884,0.1953,0.964,0.2,0 +3db004e6-ee34-43c5-ae8e-f22894ead5d0,BE18438,2026-02-25T20:08:22,T438LH6V.XY0W,Vert,0.1878,0.1953,0.962,0.2,0 +d276ae48-0434-41e5-af43-e0fb366acf11,BE18438,2026-02-25T20:11:33,T438LH6W.390W,Vert,0.1880,0.1953,0.963,0.2,0 +c027dc12-88fd-4a45-872b-a6a945f2008e,BE18438,2026-02-25T20:14:35,T438LH6W.8B0W,Vert,0.1874,0.1953,0.959,0.2,0 +9665bc3a-3d08-4852-b008-37ef8bfd0023,BE18438,2026-02-25T20:18:51,T438LH6W.FF0W,Vert,0.1884,0.1953,0.965,0.2,0 +5fb5a05e-d7e2-4268-9b85-cfe8d82aa6c9,BE18438,2026-02-25T20:29:50,T438LH6W.XQ0W,Vert,0.1890,0.1953,0.968,0.2,0 +57ec0b10-9324-4459-a07e-0c8532b8928f,BE18438,2026-02-25T20:35:39,T438LH6X.7F0W,Vert,0.1889,0.1953,0.967,0.2,0 +ac86f73f-1174-4fae-a206-ecdcb73e1ffc,BE18438,2026-02-25T20:39:12,T438LH6X.DC0W,Vert,0.1887,0.1953,0.966,0.2,0 +dcf25ddd-9cf0-41c1-b109-f3ef2e490ce5,BE18438,2026-02-25T20:43:41,T438LH6X.KT0W,Vert,0.1871,0.1953,0.958,0.2,0 +aa79bbdd-2392-4ef6-8a5c-873a9f72746d,BE18438,2026-02-25T20:46:51,T438LH6X.Q30W,Vert,0.1877,0.1953,0.961,0.2,0 +ad46d1ce-b417-4c8c-b029-bb3b008ba809,BE18438,2026-02-25T20:53:33,T438LH6Y.190W,Vert,0.1882,0.1953,0.964,0.2,0 +3d3a8f34-86a7-4d70-a7e8-fa768a723325,BE18438,2026-02-26T07:03:39,T438LH7Q.A30W,Vert,0.1809,0.1953,0.926,0.2,0 +7817525f-dc3d-4f5e-b2cb-5af9ca12ede8,BE18438,2026-02-26T07:09:42,T438LH7Q.K60W,Vert,0.1814,0.1953,0.929,0.2,0 +a2091302-0b34-46c1-ab83-174be2b59bcf,BE18438,2026-02-26T13:12:35,T438LH87.CZ0W,Vert,0.1896,0.1953,0.971,0.2,0 +e1c0c9de-a900-479f-9572-9675b13296a9,BE18438,2026-02-26T13:24:46,T438LH87.XA0W,Vert,0.2237,0.2588,0.864,0.2,0 +76a6dc6b-88e1-434b-a137-f5e9b63f248e,BE18438,2026-02-26T13:27:43,T438LH88.270W,Vert,0.3185,0.3223,0.988,0.2,0 +281b508b-7982-4668-86af-9b0b28be608a,BE18438,2026-02-26T13:30:36,T438LH88.700W,Vert,0.3263,0.3320,0.983,0.2,0 +68bbaf77-bb23-40fd-94f9-f928728d90dc,BE18438,2026-02-26T13:33:28,T438LH88.BS0W,Vert,0.3207,0.3271,0.980,0.2,0 +5c975640-6ff1-441a-aa11-41642f6a6b31,BE18438,2026-02-26T13:36:21,T438LH88.GL0W,Vert,0.3261,0.3320,0.982,0.2,0 +dea248ac-1124-4bba-b054-953f86c852b2,BE18438,2026-02-26T13:40:34,T438LH88.NM0W,Vert,0.3271,0.3320,0.985,0.2,0 +4e463201-f008-41ad-9c52-11fcc3d93095,BE18438,2026-02-26T13:43:29,T438LH88.SH0W,Vert,0.3357,0.3418,0.982,0.2,0 +507262c5-d58f-4a05-a6a3-a6cbb5b4de20,BE18438,2026-02-26T13:46:24,T438LH88.XC0W,Vert,0.3372,0.3418,0.987,0.2,0 +4c2d8ee8-842f-4e44-9e03-ff34d26e4fdb,BE18438,2026-02-26T13:49:17,T438LH89.250W,Vert,0.3253,0.3320,0.980,0.2,0 +b184b112-603f-44f4-ae97-087493fbd8ce,BE18438,2026-02-26T13:52:10,T438LH89.6Y0W,Vert,0.3321,0.3369,0.986,0.2,0 +2ff1a552-5a2e-4396-9083-b9c8d334eed7,BE18438,2026-02-26T13:55:04,T438LH89.BS0W,Vert,0.3378,0.3418,0.988,0.2,0 +928e21a7-1990-4792-803e-58014d8140ce,BE18438,2026-02-26T13:57:57,T438LH89.GL0W,Vert,0.3341,0.3369,0.992,0.2,0 +f521b5db-6100-41e6-954e-0abdbf9660c1,BE18438,2026-02-26T14:00:51,T438LH89.LF0W,Vert,0.3421,0.3467,0.987,0.2,0 +efba962f-e511-41ef-9fc5-313f202a9d96,BE18438,2026-02-26T14:03:46,T438LH89.QA0W,Vert,0.3506,0.3564,0.984,0.2,0 +0b51e273-8d10-4494-bda6-8c75eaeec66f,BE18438,2026-02-26T14:06:41,T438LH89.V50W,Vert,0.3448,0.3516,0.981,0.2,0 +6d46e4ee-1647-4169-b6b2-33da32240eb9,BE18438,2026-02-26T14:09:35,T438LH89.ZZ0W,Vert,0.3463,0.3516,0.985,0.2,0 +1ca294cd-4dc2-41fb-9374-e277d4550761,BE18438,2026-02-26T14:12:28,T438LH8A.4S0W,Vert,0.3442,0.3467,0.993,0.2,0 +7b683dcf-1aed-4d9b-be1e-b10f92374398,BE18438,2026-02-26T14:15:23,T438LH8A.9N0W,Vert,0.3486,0.3516,0.992,0.2,0 +2b5cb8e7-c947-4914-bcb5-504f74603547,BE18438,2026-02-26T14:18:21,T438LH8A.EL0W,Vert,0.3519,0.3613,0.974,0.2,0 +5ea3a6bf-10de-4911-aedc-66daecf86b76,BE18438,2026-02-26T14:21:16,T438LH8A.JG0W,Vert,0.3460,0.3564,0.971,0.2,0 +d93be7f5-39ec-45fd-9949-fa531f2f9d34,BE18438,2026-02-26T14:24:13,T438LH8A.OD0W,Vert,0.3457,0.3516,0.983,0.2,0 +cfa113f2-8cee-4a1d-b1e7-0e936fa771e2,BE18438,2026-02-26T14:28:21,T438LH8A.V90W,Vert,0.3485,0.3564,0.978,0.2,0 +46d322ac-38cb-4c99-a4c2-9e9d7ad06171,BE18438,2026-02-26T14:31:17,T438LH8B.050W,Vert,0.3367,0.3467,0.971,0.2,0 +42500a83-bd67-4deb-8bb2-963ee2b8a7ce,BE18438,2026-02-26T14:34:11,T438LH8B.4Z0W,Vert,0.3365,0.3516,0.957,0.2,0 +3ccf6bda-2e88-473c-951d-9ea86d0d9363,BE18438,2026-02-26T14:37:05,T438LH8B.9T0W,Vert,0.3472,0.3564,0.974,0.2,0 +9050487d-d238-4631-b8e1-5a693a026542,BE18438,2026-02-26T14:39:58,T438LH8B.EM0W,Vert,0.3420,0.3564,0.959,0.2,0 +e62989e4-a326-4bae-8d47-a8becee96462,BE18438,2026-02-26T14:42:52,T438LH8B.JG0W,Vert,0.3430,0.3564,0.962,0.2,0 +f0e30fab-dc57-4189-8439-48df17007a2c,BE18438,2026-02-26T14:45:46,T438LH8B.OA0W,Vert,0.3389,0.3516,0.964,0.2,0 +03e1ca09-78df-4c6f-9915-0b6e4f2141d2,BE18438,2026-02-26T14:48:43,T438LH8B.T70W,Vert,0.3449,0.3564,0.968,0.2,0 +945d39f7-1b98-4a1f-80a8-6a18fd9df25f,BE18438,2026-02-26T14:51:37,T438LH8B.Y10W,Vert,0.3522,0.3613,0.975,0.2,0 +d27eb2e3-e313-4125-98f4-f9d6d3737d28,BE18438,2026-02-26T14:54:32,T438LH8C.2W0W,Vert,0.3502,0.3613,0.969,0.2,0 +916901cb-ebea-441e-a2a1-40f4e49f32f4,BE18438,2026-02-26T14:57:25,T438LH8C.7P0W,Vert,0.3524,0.3662,0.962,0.2,0 +5d8d7e29-fdbb-4fa3-bd25-506c9352a354,BE18438,2026-02-26T15:01:35,T438LH8C.EN0W,Vert,0.3530,0.3613,0.977,0.2,0 +c6c7a3af-b03b-43dc-97de-024f8e09b62f,BE18438,2026-02-26T15:04:27,T438LH8C.JF0W,Vert,0.3514,0.3613,0.973,0.2,0 +b6856898-3d10-4967-9577-7f17902f77ea,BE18438,2026-02-26T15:11:34,T438LH8C.VA0W,Vert,0.3566,0.3613,0.987,0.2,0 +7a4802f6-621e-4230-8d8c-d7f0c9a65bf2,BE18438,2026-02-26T15:14:24,T438LH8D.000W,Vert,0.3594,0.3711,0.968,0.2,0 +79f38b7e-ec56-417e-943a-8c944252c2b8,BE18438,2026-02-26T15:18:32,T438LH8D.6W0W,Vert,0.3550,0.3613,0.983,0.2,0 +1b4190c4-0d54-48ba-8ff5-8ddb0eb91e50,BE18438,2026-02-26T15:22:22,T438LH8D.DA0W,Vert,0.3571,0.3613,0.988,0.2,0 +de57b5ae-a3a1-4ede-9d2b-3e87bfb3fd19,BE9558,2026-04-14T11:16:32,K558LJN3.BK0W,Tran,0.3448,0.3662,0.942,0.2,0 +43afeaf5-02ec-41f6-9e23-0c9772821ed4,BE9558,2026-04-14T11:27:15,K558LJN3.TF0W,Tran,0.3094,0.3223,0.960,0.2,0 +8123c0ef-84c9-4f6c-8d82-9dc32e2e470d,BE9558,2026-04-14T14:45:30,K558LJNC.ZU0W,Tran,0.2721,0.3564,0.763,0.2,0 +8c787af3-596e-411b-9e10-29485fa5114f,BE9558,2026-04-29T16:18:47,K558LKF9.BB0W,Tran,0.2943,0.3027,0.972,0.2,0 +431928ff-b4c4-4caa-b933-acc917f3717c,BE9558,2026-05-04T15:02:30,K558LKOF.460W,Tran,0.4364,0.5225,0.835,0.2,0 +6c3c07f8-c36a-4493-acce-7441c9d22cec,BE9558,2026-05-15T08:50:06,K558LL8B.7I0W,Long,0.2892,0.2930,0.987,0.2,0 +13c268df-b652-422e-a642-6453f6a314aa,BE9558,2026-05-15T10:18:34,K558LL8F.AY0W,Long,0.2907,0.2979,0.976,0.2,0 +9b0d0871-8810-467d-806a-5bbfa4e667fe,BE9558,2026-05-15T15:52:00,K558LL8U.QO0W,Long,0.2659,0.2979,0.893,0.2,0 +092640f9-a944-48b7-b872-364d75c2c5e7,BE9558,2026-05-15T16:13:12,K558LL8V.Q00W,Long,0.2428,0.2979,0.815,0.2,0 +0573741a-96ab-4b36-af7a-b5bc11a79009,BE9558,2026-05-16T03:23:26,K558LL9Q.R20W,Long,0.2861,0.2930,0.976,0.2,0 +321f03ea-6696-47de-ad46-2be43d4d6cae,BE9558,2026-05-16T03:30:49,K558LL9R.3D0W,Long,0.2886,0.2930,0.985,0.2,0 +cc423a6a-3e3c-466a-b39c-6146ca7f34c8,BE9558,2026-05-16T03:33:55,K558LL9R.8J0W,Long,0.2880,0.2881,1.000,0.2,0 +efa52a14-cc68-4bea-aab4-4ee3a3cbff3a,BE9558,2026-05-16T03:36:55,K558LL9R.DJ0W,Long,0.2890,0.2930,0.986,0.2,0 +92895bfe-122f-4cdc-bd3d-40609633d278,BE9558,2026-05-16T03:43:41,K558LL9R.OT0W,Long,0.2880,0.2881,1.000,0.2,0 +2a4c81db-2307-44e5-9b27-d5154d98e38d,BE9558,2026-05-16T03:46:36,K558LL9R.TO0W,Long,0.2921,0.2979,0.981,0.2,0 +656b0fdc-e275-4c9c-8936-9eee267ea04e,BE9558,2026-05-16T03:49:28,K558LL9R.YG0W,Long,0.2966,0.2979,0.996,0.2,0 +3b9989ae-e0f2-4652-9724-c145603543d2,BE9558,2026-05-16T03:52:29,K558LL9S.3H0W,Long,0.2880,0.2930,0.983,0.2,0 +837d0d84-d609-4f4a-b542-b7e2b064ea22,BE9558,2026-05-16T03:56:32,K558LL9S.A80W,Long,0.2880,0.2930,0.983,0.2,0 +affcb008-ab99-4bc5-8f35-83dbe620499d,BE9558,2026-05-16T04:02:50,K558LL9S.KQ0W,Long,0.2896,0.2930,0.989,0.2,0 +e1f1c6d2-545c-451f-9018-714d52e20a05,BE9558,2026-05-16T04:05:47,K558LL9S.PN0W,Long,0.3024,0.3076,0.983,0.2,0 +93ef80f1-7e83-4703-a0f9-89bbfeeae6c3,BE9558,2026-05-16T04:08:41,K558LL9S.UH0W,Long,0.2976,0.2979,0.999,0.2,0 +5c118ca9-fc37-4080-876d-b3f57ce0b121,BE9558,2026-05-16T04:11:42,K558LL9S.ZI0W,Long,0.2881,0.2930,0.983,0.2,0 +d98b2eaa-7763-4c78-88fb-c21195cb9978,BE9558,2026-05-16T04:14:38,K558LL9T.4E0W,Long,0.2898,0.2930,0.989,0.2,0 +6a146a01-0d02-4413-b231-ecb4a6fde776,BE9558,2026-05-16T04:17:35,K558LL9T.9B0W,Long,0.2880,0.2930,0.983,0.2,0 +f6587a55-27e3-454f-997e-57e7516a296c,BE9558,2026-05-16T04:20:32,K558LL9T.E80W,Long,0.3046,0.3076,0.990,0.2,0 +1fb48250-2214-41e6-92cb-362aff1ba62b,BE9558,2026-05-16T04:26:05,K558LL9T.NH0W,Long,0.2876,0.2930,0.982,0.2,0 +2f4f3d06-1189-4e28-a1ac-5454ea586a0c,BE9558,2026-05-16T04:59:47,K558LL9V.7N0W,Long,0.2961,0.3027,0.978,0.2,0 +62b7bdab-b744-41e4-8d0e-c9141b9d4821,BE9558,2026-05-16T05:02:46,K558LL9V.CM0W,Long,0.2903,0.2930,0.991,0.2,0 +cadcb6dc-74c0-48b5-974c-87bb7a57816c,BE9558,2026-05-16T05:05:43,K558LL9V.HJ0W,Long,0.3090,0.3125,0.989,0.2,0 +a0235177-be5a-4f11-b741-c65744698e5b,BE9558,2026-05-16T05:08:35,K558LL9V.MB0W,Long,0.3167,0.3223,0.983,0.2,0 +1968c2f6-f52c-4544-a0ed-f79b91ce0da0,BE9558,2026-05-16T05:11:28,K558LL9V.R40W,Long,0.3446,0.3516,0.980,0.2,0 +caa06b94-8813-4949-b47b-61ffb16e61c4,BE9558,2026-05-16T05:14:22,K558LL9V.VY0W,Long,0.3318,0.3320,0.999,0.2,0 +fdaa8101-f033-47e6-9c65-abd76dea1870,BE9558,2026-05-16T05:17:16,K558LL9W.0S0W,Long,0.3407,0.3418,0.997,0.2,0 +29bd16bd-81b9-4ccd-a981-f1a80858536c,BE9558,2026-05-16T05:23:05,K558LL9W.AH0W,Long,0.3450,0.3516,0.981,0.2,0 +1f8bc862-d5fd-4cce-abc0-e2bd8c2f11fa,BE9558,2026-05-16T05:26:00,K558LL9W.FC0W,Long,0.3417,0.3467,0.986,0.2,0 +20220068-d498-4fca-9a2a-a7fd2ee52ab1,BE9558,2026-05-16T05:28:53,K558LL9W.K50W,Long,0.3516,0.3564,0.986,0.2,0 +9816a261-3118-417a-8334-ace36687ad8f,BE9558,2026-05-16T05:31:47,K558LL9W.OZ0W,Long,0.3416,0.3467,0.985,0.2,0 +5d2c76d7-543d-4557-9caf-996a69ab00bd,BE9558,2026-05-16T05:34:42,K558LL9W.TU0W,Long,0.3537,0.3564,0.992,0.2,0 +c7b5ae00-52e7-4920-aa27-49d3b3daff92,BE9558,2026-05-16T05:37:36,K558LL9W.YO0W,Long,0.3517,0.3564,0.987,0.2,0 +07db8fd8-ccef-4a5a-9d97-9f318f73a478,BE9558,2026-05-16T05:40:31,K558LL9X.3J0W,Long,0.3574,0.3613,0.989,0.2,0 +790fcff9-e266-4a12-87f0-a072990d2533,BE9558,2026-05-16T05:43:26,K558LL9X.8E0W,Long,0.3650,0.3662,0.997,0.2,0 +f4245f61-1dde-43dc-a121-98e102f6a193,BE9558,2026-05-16T05:50:28,K558LL9X.K40W,Long,0.3856,0.3906,0.987,0.2,0 +5a3bce7b-358e-4260-876f-3f853163d7cf,BE9558,2026-05-16T05:53:22,K558LL9X.OY0W,Long,0.3847,0.3857,0.997,0.2,0 +37c2b4b1-7a1b-43e7-b513-2c16f582109c,BE9558,2026-05-16T05:56:15,K558LL9X.TR0W,Long,0.3906,0.3955,0.988,0.2,0 +57fde109-1d8a-45a1-b398-5bd28c3e47f1,BE9558,2026-05-16T05:59:10,K558LL9X.YM0W,Long,0.4000,0.4004,0.999,0.2,0 +1e41ab40-93be-405c-a464-f7a57524d3b3,BE9558,2026-05-16T06:02:02,K558LL9Y.3E0W,Long,0.3825,0.3857,0.992,0.2,0 +e9d61eb5-fae8-4902-8fab-4828ffa8e6da,BE9558,2026-05-16T06:04:57,K558LL9Y.890W,Long,0.3802,0.3809,0.998,0.2,0 +c0e964e0-7788-4e41-bc41-6e479c3dc81b,BE9558,2026-05-16T06:07:51,K558LL9Y.D30W,Long,0.3954,0.4004,0.987,0.2,0 +0b2df8e5-1689-4fe6-abad-30970ba10aa6,BE9558,2026-05-16T06:16:06,K558LL9Y.QU0W,Long,0.4051,0.4053,1.000,0.2,0 +2bc6ce73-464b-469c-9e8f-3bf6f6cfae1d,BE9558,2026-05-16T06:21:51,K558LL9Z.0F0W,Long,0.3910,0.3955,0.989,0.2,0 +b8603ee1-616a-451e-a2cd-61f8f2634cd0,BE9558,2026-05-16T06:24:44,K558LL9Z.580W,Long,0.3969,0.4004,0.991,0.2,0 +1aecf582-66a2-4ebb-80f9-04b3aeba331a,BE9558,2026-05-16T06:27:39,K558LL9Z.A30W,Long,0.3915,0.3955,0.990,0.2,0 +885aedfd-7a1a-4120-8176-7c344dd3d8fe,BE9558,2026-05-16T06:30:30,K558LL9Z.EU0W,Long,0.4025,0.4053,0.993,0.2,0 +ec15930f-47f5-4752-afc2-427ee156ac95,BE9558,2026-05-16T06:33:25,K558LL9Z.JP0W,Long,0.4060,0.4102,0.990,0.2,0 +5ca23f47-2a38-4a54-99dd-041287aa9510,BE9558,2026-05-16T06:36:17,K558LL9Z.OH0W,Long,0.4054,0.4102,0.988,0.2,0 +03f371cf-81bc-4681-80f0-ef6fa437da85,BE9558,2026-05-16T06:39:09,K558LL9Z.T90W,Long,0.4052,0.4102,0.988,0.2,0 +ed386f54-140d-45e8-ad51-6585a58b4375,BE9558,2026-05-16T06:44:54,K558LLA0.2U0W,Long,0.3993,0.4004,0.997,0.2,0 +579f1c3f-4c1e-42a3-aa89-69aa93514033,BE9558,2026-05-16T06:47:47,K558LLA0.7N0W,Long,0.4001,0.4053,0.987,0.2,0 +57024de6-07ea-41f4-b916-c6adc3f51636,BE9558,2026-05-16T06:50:40,K558LLA0.CG0W,Long,0.4103,0.4150,0.989,0.2,0 +36bf5adf-afb5-4a5e-a106-1afbd1f56c7b,BE9558,2026-05-16T06:54:49,K558LLA0.JD0W,Long,0.4069,0.4150,0.980,0.2,0 +3e2f831b-5228-46ff-9acb-75d8c86f8bc1,BE9558,2026-05-16T06:57:43,K558LLA0.O70W,Long,0.4108,0.4150,0.990,0.2,0 +1775fa20-568b-4294-b9ca-561201a66c5b,BE9558,2026-05-16T07:00:38,K558LLA0.T20W,Long,0.3967,0.4004,0.991,0.2,0 +fd391469-03f9-4d8d-bccd-3af50c91e5b7,BE9558,2026-05-16T07:03:33,K558LLA0.XX0W,Long,0.3927,0.3955,0.993,0.2,0 +02b52ef6-b707-4dcb-b58f-43da02698832,BE9558,2026-05-16T07:06:27,K558LLA1.2R0W,Long,0.3824,0.3857,0.991,0.2,0 +8d45333f-e101-4249-9e27-913c583a0a8d,BE9558,2026-05-16T07:09:21,K558LLA1.7L0W,Long,0.4085,0.4150,0.984,0.2,0 +361d11ce-f42a-4d04-8270-ee3771fe4f09,BE9558,2026-05-16T07:12:14,K558LLA1.CE0W,Long,0.4017,0.4053,0.991,0.2,0 +59218125-ed5d-4457-aa8b-35ba03441363,BE9558,2026-05-16T07:15:07,K558LLA1.H70W,Long,0.3934,0.3955,0.995,0.2,0 +a603e55a-2ad6-4fc9-8c8c-0dc7be95c3e7,BE9558,2026-05-16T07:23:43,K558LLA1.VJ0W,Long,0.3806,0.3857,0.987,0.2,0 +c8d07902-93ea-4b03-9e69-9ad6ed5bffb1,BE9558,2026-05-16T07:26:36,K558LLA2.0C0W,Long,0.3807,0.3857,0.987,0.2,0 +4c46c0c7-42f0-4648-827e-1995f733b18f,BE9558,2026-05-16T07:30:44,K558LLA2.780W,Long,0.3667,0.3711,0.988,0.2,0 +04759a00-ba3e-4959-a4cb-be8fc1f3004e,BE9558,2026-05-16T07:36:28,K558LLA2.GS0W,Long,0.3514,0.3516,1.000,0.2,0 +0b6b0856-bd87-4a07-a5b4-f13d99e04a12,BE9558,2026-05-16T07:39:20,K558LLA2.LK0W,Long,0.3512,0.3516,0.999,0.2,0 +3d81a8bd-464d-4050-8a76-25e8373d1ddb,BE9558,2026-05-16T07:42:12,K558LLA2.QC0W,Long,0.3350,0.3369,0.994,0.2,0 +b8b9c0c4-c388-4acb-955a-be4ac356afce,BE9558,2026-05-16T07:45:04,K558LLA2.V40W,Long,0.3563,0.3564,1.000,0.2,0 +cb3d142a-fb84-4a21-a298-f61bb869b036,BE9558,2026-05-16T07:49:13,K558LLA3.210W,Long,0.3663,0.3711,0.987,0.2,0 +7401b19b-bc9d-43f0-b395-f4323aca5c72,BE9558,2026-05-16T07:52:06,K558LLA3.6U0W,Long,0.3612,0.3662,0.986,0.2,0 +2236df76-a9e6-401f-9176-ad565242f2cf,BE9558,2026-05-16T07:57:52,K558LLA3.GG0W,Long,0.3366,0.3369,0.999,0.2,0 +a645846c-cae4-4aa5-a2e6-85166bd4dd62,BE9558,2026-05-16T08:00:44,K558LLA3.L80W,Long,0.3265,0.3271,0.998,0.2,0 +7212bd33-99a9-4659-9f0b-c98cc8d1e8bc,BE9558,2026-05-16T08:04:51,K558LLA3.S30W,Long,0.3418,0.3467,0.986,0.2,0 +2cc4bf33-c63a-4c65-b0e2-5e61a9146a5d,BE9558,2026-05-16T08:07:43,K558LLA3.WV0W,Long,0.3405,0.3418,0.996,0.2,0 +4fedb486-5f1f-4623-a01f-083e829f0565,BE9558,2026-05-16T08:10:35,K558LLA4.1N0W,Long,0.3515,0.3564,0.986,0.2,0 +58aff0ba-0f9b-45e1-9bbe-a990d16192ae,BE9558,2026-05-16T08:13:27,K558LLA4.6F0W,Long,0.3513,0.3516,0.999,0.2,0 +e56d87e5-e1d4-4947-b6a5-3bbd2773c54a,BE9558,2026-05-16T08:20:03,K558LLA4.HF0W,Long,0.3564,0.3613,0.986,0.2,0 +0e96e61e-6a21-4e07-8e09-72d09dbfa1e6,BE9558,2026-05-16T08:25:21,K558LLA4.Q90W,Long,0.3565,0.3613,0.987,0.2,0 +19611f28-2da8-4362-a58e-dbed2e061379,BE9558,2026-05-16T08:34:19,K558LLA5.570W,Long,0.3647,0.3662,0.996,0.2,0 +bd580c1d-44d4-41fd-9f83-f462cda4099a,BE9558,2026-05-16T08:37:15,K558LLA5.A30W,Long,0.3536,0.3564,0.992,0.2,0 +2782df3b-3cb4-471a-a6f5-bbc4903a8ab9,BE9558,2026-05-16T08:44:00,K558LLA5.LC0W,Long,0.3612,0.3662,0.986,0.2,0 +d15a08f3-1c7c-4f80-8d9b-f5008c58c3d9,BE9558,2026-05-16T08:49:18,K558LLA5.U60W,Long,0.3610,0.3613,0.999,0.2,0 +03a10a6a-193a-476c-9543-f63818f69417,BE9558,2026-05-16T08:54:36,K558LLA6.300W,Long,0.3515,0.3516,1.000,0.2,0 +1c271856-8100-4c0d-bc8c-9357b47d7626,BE9558,2026-05-16T08:59:54,K558LLA6.BU0W,Long,0.3617,0.3662,0.988,0.2,0 +a85f125e-bfa2-42be-b877-2604b8d068c8,BE9558,2026-05-16T09:05:17,K558LLA6.KT0W,Long,0.3510,0.3516,0.998,0.2,0 +dde9aa71-d2e3-41ac-8388-f2bbddb6dae6,BE9558,2026-05-16T09:08:11,K558LLA6.PN0W,Long,0.3447,0.3467,0.994,0.2,0 +2f3f2be2-0ee3-422e-b3ad-83fb87512a2b,BE9558,2026-05-16T09:14:39,K558LLA7.0F0W,Long,0.3580,0.3613,0.991,0.2,0 +357a3034-8a9c-485e-8cb6-f526dd2baf6f,BE9558,2026-05-16T09:29:11,K558LLA7.ON0W,Long,0.3708,0.3760,0.986,0.2,0 +f8ad0274-8e6f-4714-82ff-d229e60f1442,BE9558,2026-05-16T09:32:06,K558LLA7.TI0W,Long,0.3567,0.3613,0.987,0.2,0 +42bb007a-2726-401a-b085-c25151b0dc24,BE9558,2026-05-16T09:35:02,K558LLA7.YE0W,Long,0.3575,0.3613,0.989,0.2,0 +c99bfc4a-db02-4bea-bea4-943c67485dba,BE9558,2026-05-16T09:37:58,K558LLA8.3A0W,Long,0.3527,0.3564,0.989,0.2,0 +8fc17de0-73f7-48bd-8d93-5451dce47e8e,BE9558,2026-05-16T09:40:52,K558LLA8.840W,Long,0.3613,0.3662,0.987,0.2,0 +bbee5276-03dd-4ca4-a7b9-aec02baac7e8,BE9558,2026-05-16T09:48:13,K558LLA8.KD0W,Long,0.3590,0.3613,0.993,0.2,0 +8dcdee10-db01-4224-9506-4181c3105d6f,BE9558,2026-05-16T09:53:31,K558LLA8.T70W,Long,0.3588,0.3613,0.993,0.2,0 +48c8be72-64d9-4da3-95b5-bbec2c5e1dbd,BE9558,2026-05-16T09:58:49,K558LLA9.210W,Long,0.3582,0.3662,0.978,0.2,0 +0da20919-7ce9-4feb-8ae9-12d9bb40c2b5,BE9558,2026-05-16T10:04:07,K558LLA9.AV0W,Long,0.3506,0.3516,0.997,0.2,0 +c2f7e9bc-8b93-42c3-96d3-e3af217dde35,BE9558,2026-05-16T10:09:25,K558LLA9.JP0W,Long,0.3506,0.3516,0.997,0.2,0 +522a0e42-d443-4447-bc71-af1832c6604e,BE9558,2026-05-16T10:14:43,K558LLA9.SJ0W,Long,0.3624,0.3662,0.990,0.2,0 +67f7e732-3c0b-439f-87c5-3c478a381418,BE9558,2026-05-16T10:20:01,K558LLAA.1D0W,Long,0.3494,0.3516,0.994,0.2,0 +251767e8-56f7-4091-a001-364d8e75c15c,BE9558,2026-05-16T10:25:19,K558LLAA.A70W,Long,0.3463,0.3516,0.985,0.2,0 +9457239d-a4ca-4476-b385-5e01fc8f6420,BE9558,2026-05-16T10:30:37,K558LLAA.J10W,Long,0.3436,0.3467,0.991,0.2,0 +43718d7c-9353-49e1-89b6-0841aeb1b276,BE9558,2026-05-16T10:35:55,K558LLAA.RV0W,Long,0.3452,0.3516,0.982,0.2,0 +f54ba557-ddfe-41d3-8f4c-8bbec9a3783c,BE9558,2026-05-16T10:41:13,K558LLAB.0P0W,Long,0.3419,0.3467,0.986,0.2,0 +dc26c758-ad99-49c1-b518-931368cdf8cf,BE9558,2026-05-16T10:46:31,K558LLAB.9J0W,Long,0.3463,0.3516,0.985,0.2,0 +10917eb2-d151-461c-a8cb-e9660da5a5b0,BE9558,2026-05-16T10:51:49,K558LLAB.ID0W,Long,0.3520,0.3564,0.987,0.2,0 +4938a5e6-d53b-40ee-be2b-06ec0e60398c,BE9558,2026-05-16T10:57:07,K558LLAB.R70W,Long,0.3541,0.3564,0.993,0.2,0 +0711b51d-55bd-48be-b51e-2aad67107a60,BE9558,2026-05-16T11:13:01,K558LLAC.HP0W,Long,0.3661,0.3711,0.986,0.2,0 +065564b6-aa49-40e7-9ea6-c75defdc60fe,BE9558,2026-05-16T11:18:19,K558LLAC.QJ0W,Long,0.3622,0.3662,0.989,0.2,0 +ffb6b230-1d70-4e0e-ba30-e945339a122a,BE9558,2026-05-16T11:23:37,K558LLAC.ZD0W,Long,0.3711,0.3760,0.987,0.2,0 +52b4da82-c1e0-4a8a-8365-51b090787005,BE9558,2026-05-16T11:28:55,K558LLAD.870W,Long,0.3644,0.3662,0.995,0.2,0 +cdbff3d8-fbfe-4bea-8bd6-c898800e81ba,BE9558,2026-05-16T11:39:31,K558LLAD.PV0W,Long,0.3609,0.3662,0.986,0.2,0 +3fb0bc30-1463-4cfc-9e7f-386c0eb2a34a,BE9558,2026-05-16T11:44:49,K558LLAD.YP0W,Long,0.3553,0.3564,0.997,0.2,0 +ba1f5ec8-aef7-4a9e-ac1f-276cf016e285,BE9558,2026-05-16T11:50:07,K558LLAE.7J0W,Long,0.3479,0.3516,0.990,0.2,0 +b1dd4df7-34a9-4512-a903-f8242b236fd2,BE9558,2026-05-16T11:55:25,K558LLAE.GD0W,Long,0.3427,0.3467,0.988,0.2,0 +1f23aa28-c026-4bc7-b3d1-7c78fd76a673,BE9558,2026-05-16T12:00:43,K558LLAE.P70W,Long,0.3454,0.3467,0.996,0.2,0 +0d91f54d-b20d-4a50-b7dd-d071c39a0690,BE9558,2026-05-16T12:06:01,K558LLAE.Y10W,Long,0.3431,0.3467,0.990,0.2,0 +12ccdecd-5fd9-4465-9813-3d6006ec32f3,BE9558,2026-05-16T12:11:19,K558LLAF.6V0W,Long,0.3433,0.3467,0.990,0.2,0 +a7024f1a-f071-48ab-820f-99bdf87366b5,BE9558,2026-05-16T12:16:37,K558LLAF.FP0W,Long,0.3361,0.3369,0.998,0.2,0 +de0fa439-30b8-4607-a3c3-7e17191e4624,BE9558,2026-05-16T12:21:55,K558LLAF.OJ0W,Long,0.3274,0.3320,0.986,0.2,0 +f9374ec3-05e5-477f-8e1a-0c862fed2a50,BE9558,2026-05-16T12:27:13,K558LLAF.XD0W,Long,0.3449,0.3467,0.995,0.2,0 +424cb275-3743-4174-95f0-0ca1205b374c,BE9558,2026-05-16T12:32:31,K558LLAG.670W,Long,0.3370,0.3418,0.986,0.2,0 +9accde13-cefa-41b8-bd0a-cef2ffed59b0,BE9558,2026-05-16T12:37:49,K558LLAG.F10W,Long,0.3465,0.3467,0.999,0.2,0 +4665be38-1956-4b14-b931-068ba9af5c5e,BE9558,2026-05-16T12:48:25,K558LLAG.WP0W,Long,0.3453,0.3516,0.982,0.2,0 +1aedf624-1a99-43e1-a8f2-f9d358b97f61,BE9558,2026-05-16T12:53:43,K558LLAH.5J0W,Long,0.3366,0.3418,0.985,0.2,0 +c15e1799-91d7-425e-ad17-2ba3d170d81e,BE9558,2026-05-16T13:04:19,K558LLAH.N70W,Long,0.3348,0.3369,0.994,0.2,0 +72d4c4ea-19ca-4cc6-9b04-7a2b70122c67,BE9558,2026-05-16T13:09:37,K558LLAH.W10W,Long,0.3272,0.3320,0.986,0.2,0 +b4ebf2c9-1f6d-4594-9810-40479e87c0a5,BE9558,2026-05-16T13:14:55,K558LLAI.4V0W,Long,0.3297,0.3320,0.993,0.2,0 +bb297e9e-548f-4dae-8b1a-0f161af2e766,BE9558,2026-05-16T13:20:13,K558LLAI.DP0W,Long,0.3272,0.3320,0.986,0.2,0 +87562c24-63db-48bd-9962-72a20a40f2ff,BE9558,2026-05-16T13:36:07,K558LLAJ.470W,Long,0.3174,0.3223,0.985,0.2,0 +01e9cc58-3bea-4f38-8499-a2c20e5f8550,BE9558,2026-05-16T13:46:52,K558LLAJ.M40W,Long,0.2880,0.2930,0.983,0.2,0 +5ae7503d-ae17-4063-9208-6a420699e1ab,BE9558,2026-05-16T14:00:35,K558LLAK.8Z0W,Long,0.2879,0.2930,0.983,0.2,0 +e009ea3c-1534-4154-8623-e180ed4db9fb,BE9558,2026-05-16T14:06:19,K558LLAK.IJ0W,Long,0.2882,0.2930,0.984,0.2,0 +cd4e21f5-512b-4b2b-8054-bec01aa06400,BE9558,2026-05-16T14:12:03,K558LLAK.S30W,Long,0.2857,0.2930,0.975,0.2,0 +ae5b780f-2038-4ccf-8bfc-b15b87be139e,BE9558,2026-05-16T14:17:22,K558LLAL.0Y0W,Long,0.2899,0.2930,0.989,0.2,0 +d8c3fb81-8c66-4ebc-b33a-0559bc03334b,BE9558,2026-05-16T14:22:40,K558LLAL.9S0W,Long,0.3009,0.3027,0.994,0.2,0 +990a4c7a-3c51-4d4a-8ab7-416ccb97d29c,BE9558,2026-05-16T14:27:58,K558LLAL.IM0W,Long,0.3057,0.3076,0.994,0.2,0 +68d7a2c3-d899-4e75-96fb-3bf16ae6ea6f,BE9558,2026-05-16T14:33:16,K558LLAL.RG0W,Long,0.3110,0.3125,0.995,0.2,0 +73505b8c-f912-4116-b903-a8a4c75524d3,BE9558,2026-05-16T14:38:34,K558LLAM.0A0W,Long,0.3172,0.3223,0.984,0.2,0 +8d3c9602-7efa-488a-801b-3228bd85ab14,BE9558,2026-05-16T14:43:51,K558LLAM.930W,Long,0.3136,0.3174,0.988,0.2,0 +243e16ee-860a-4960-bc15-b797ad7e9735,BE9558,2026-05-16T14:49:08,K558LLAM.HW0W,Long,0.3135,0.3174,0.988,0.2,0 +e90c8d90-6d41-4788-9ee6-a29e4bfcc16b,BE9558,2026-05-16T15:19:35,K558LLAN.WN0W,Long,0.3132,0.3174,0.987,0.2,0 +4182b0e7-7811-46b8-87d1-a4dd0c152f3b,BE9558,2026-05-16T15:28:45,K558LLAO.BX0W,Long,0.3023,0.3076,0.983,0.2,0 +28a790da-0e48-4fe4-b8ba-2c82cc6bfa79,BE9558,2026-05-16T15:34:02,K558LLAO.KQ0W,Long,0.3176,0.3223,0.985,0.2,0 +093d2fa3-7a4b-4f0f-abda-71155ebe2dfb,BE9558,2026-05-16T15:39:19,K558LLAO.TJ0W,Long,0.3223,0.3271,0.985,0.2,0 +68dc9560-4082-49b6-af04-9244773ffbc1,BE9558,2026-05-16T15:44:36,K558LLAP.2C0W,Long,0.3227,0.3271,0.986,0.2,0 +ad6b268a-165d-4b1d-b0b3-f58db7c9b0e4,BE9558,2026-05-16T15:49:53,K558LLAP.B50W,Long,0.3177,0.3223,0.986,0.2,0 +046c5cb4-56b3-48c0-83c2-bf1f30837940,BE9558,2026-05-16T15:55:10,K558LLAP.JY0W,Long,0.3174,0.3223,0.985,0.2,0 +1b76edf4-3b1c-4bcb-a73c-27334981c350,BE9558,2026-05-16T16:00:27,K558LLAP.SR0W,Long,0.3127,0.3174,0.985,0.2,0 +a0264171-a682-4396-ad8a-cda3b2964533,BE9558,2026-05-16T16:06:55,K558LLAQ.3J0W,Long,0.2885,0.2930,0.985,0.2,0 +707da1f6-ee81-4e57-abd5-98295c14651b,BE9558,2026-05-16T16:12:30,K558LLAQ.CU0W,Long,0.2880,0.2930,0.983,0.2,0 +b8115b64-8739-42c8-b541-713ab6b68bcd,BE9558,2026-05-16T16:17:53,K558LLAQ.LT0W,Long,0.2881,0.2930,0.983,0.2,0 +fb179c36-fef1-4df6-b0a0-2bd7792ea910,BE9558,2026-05-16T16:23:11,K558LLAQ.UN0W,Long,0.2933,0.2979,0.985,0.2,0 +0cec8793-c3c1-486b-a33a-4dbf9cb069b0,BE9558,2026-05-16T16:33:46,K558LLAR.CA0W,Long,0.3017,0.3027,0.997,0.2,0 +c531b052-ee04-4c66-888b-21e97445a615,BE9558,2026-05-16T16:49:37,K558LLAS.2P0W,Long,0.2922,0.2979,0.981,0.2,0 +0dc402f9-cab6-4c4a-8317-b8fc3c4b1ced,BE9558,2026-05-16T16:55:08,K558LLAS.BW0W,Long,0.2898,0.2930,0.989,0.2,0 +fd73733e-b42d-4a51-81d0-4acad6edfbd4,BE9558,2026-05-16T17:16:17,K558LLAT.B50W,Long,0.3115,0.3125,0.997,0.2,0 +6a254daa-ae27-4ac1-b334-58f776a6b276,BE9558,2026-05-16T17:26:51,K558LLAT.SR0W,Long,0.3174,0.3223,0.985,0.2,0 +0cc8e82f-5270-4508-beae-0d3af56095ca,BE9558,2026-05-16T17:37:27,K558LLAU.AF0W,Long,0.2899,0.2930,0.989,0.2,0 +00557a73-8dc3-4cd2-b880-c7820dc8b171,BE9558,2026-05-16T17:43:17,K558LLAU.K50W,Long,0.2881,0.2930,0.983,0.2,0 +8e5518d9-ff0a-4c09-9cf9-a681c84c0b6e,BE9558,2026-05-16T17:53:53,K558LLAV.1T0W,Long,0.2898,0.2930,0.989,0.2,0 +91dbd7e9-7344-400b-82b1-379f2ac4b49a,BE9558,2026-05-16T17:59:11,K558LLAV.AN0W,Long,0.2925,0.2930,0.998,0.2,0 +fa256408-d480-4376-95b5-d1a549c483ea,BE9558,2026-05-16T18:04:29,K558LLAV.JH0W,Long,0.2977,0.3027,0.983,0.2,0 +4ab6e66a-8364-49b6-82f1-86e86199e676,BE9558,2026-05-16T18:09:46,K558LLAV.SA0W,Long,0.2944,0.2979,0.988,0.2,0 +0a65fb7c-96e1-4ba0-85de-4620d88658c1,BE9558,2026-05-16T18:15:03,K558LLAW.130W,Long,0.2965,0.2979,0.995,0.2,0 +92b1ef3e-1696-4123-8662-163947737c83,BE9558,2026-05-16T18:20:20,K558LLAW.9W0W,Long,0.2976,0.3027,0.983,0.2,0 +9e3cb24c-4426-4220-9a73-389d0a96c94f,BE9558,2026-05-16T18:25:37,K558LLAW.IP0W,Long,0.2983,0.3027,0.985,0.2,0 +a3473215-4c46-45ab-9aa7-1720486ec4a8,BE9558,2026-05-16T18:30:54,K558LLAW.RI0W,Long,0.2956,0.2979,0.993,0.2,0 +449665b1-47c8-4bd2-984c-8f45b627e6d8,BE9558,2026-05-16T18:36:11,K558LLAX.0B0W,Long,0.2989,0.3027,0.987,0.2,0 +600cfabc-cffd-42e1-800c-70e75f312316,BE9558,2026-05-16T20:00:12,K558LLB0.WC0W,Long,0.3077,0.3125,0.985,0.2,0 +12bf5944-0194-4f5b-b486-a7acd7141c7f,BE9558,2026-05-16T20:04:28,K558LLB1.3G0W,Long,0.2888,0.2930,0.986,0.2,0 +f1fc913a-91bc-4240-a595-7dfe3091f9a2,BE9558,2026-05-16T20:07:29,K558LLB1.8H0W,Long,0.2977,0.3027,0.983,0.2,0 +e74f7ec3-20bb-4177-beb2-e802e5f801fd,BE9558,2026-05-16T20:10:26,K558LLB1.DE0W,Long,0.2978,0.3027,0.984,0.2,0 +27491c3a-b631-4643-818a-714523911178,BE9558,2026-05-16T20:16:21,K558LLB1.N90W,Long,0.2921,0.2930,0.997,0.2,0 +9d9196ce-11e2-4f28-b01c-3dcc60e60d42,BE9558,2026-05-16T20:22:28,K558LLB1.XG0W,Long,0.2881,0.2930,0.983,0.2,0 +a592d7c8-b643-4b25-9186-3d611ae22709,BE9558,2026-05-16T20:25:37,K558LLB2.2P0W,Long,0.2893,0.2930,0.987,0.2,0 +f709a3d5-27cd-4bb1-b91c-669e8ea11d77,BE9558,2026-05-16T20:28:57,K558LLB2.890W,Long,0.2880,0.2930,0.983,0.2,0 +cc9cad87-b87f-4b67-854a-ffb21e876a92,BE9558,2026-05-16T20:32:45,K558LLB2.EL0W,Long,0.2873,0.2930,0.980,0.2,0 +10732d26-6b49-4b14-a474-6c75e259d278,BE9558,2026-05-16T20:49:32,K558LLB3.6K0W,Long,0.2881,0.2930,0.983,0.2,0 +aa63957a-fdb2-4f17-af52-a63bb015119d,BE9558,2026-05-16T21:21:45,K558LLB4.O90W,Long,0.2880,0.2930,0.983,0.2,0 +8350440f-019d-45a4-b9f9-23058628e6f0,BE9558,2026-05-16T21:32:43,K558LLB5.6J0W,Long,0.2925,0.2930,0.999,0.2,0 +edfc10b2-dbe1-4527-a061-b0d670b5e5d2,BE9558,2026-05-16T21:35:37,K558LLB5.BD0W,Long,0.2891,0.2930,0.987,0.2,0 +f98001a3-88b2-4830-8112-b7c5164e99c3,BE9558,2026-05-16T21:38:40,K558LLB5.GG0W,Long,0.2877,0.2930,0.982,0.2,0 +f1e53e7b-b124-4fc5-82c2-504fb0c8b80d,BE9558,2026-05-16T21:41:46,K558LLB5.LM0W,Long,0.2881,0.2930,0.983,0.2,0 +7890637f-142e-4775-b48c-fd4f3b8ada77,BE9558,2026-05-16T21:45:14,K558LLB5.RE0W,Long,0.2880,0.2930,0.983,0.2,0 +7c97c10f-b413-467c-9b56-c18e423cdcfa,BE9558,2026-05-16T21:48:26,K558LLB5.WQ0W,Long,0.2886,0.2930,0.985,0.2,0 +8178cf22-8428-4f85-8199-0a43f7334a8f,BE9558,2026-05-16T21:51:47,K558LLB6.2B0W,Long,0.2881,0.2930,0.983,0.2,0 +eceb2997-deb4-403a-802f-50e93d23d423,BE9558,2026-05-16T21:57:18,K558LLB6.BI0W,Long,0.2881,0.2930,0.984,0.2,0 +9c26690c-1983-4b21-b8af-7c7967c014c3,BE9558,2026-05-16T22:12:21,K558LLB7.0L0W,Long,0.2876,0.2930,0.982,0.2,0 +78f67fc3-08b7-4317-96ba-ee5544e0b95b,BE9558,2026-05-16T22:19:34,K558LLB7.CM0W,Long,0.2880,0.2930,0.983,0.2,0 +7f9dad14-4f1b-492d-aa26-845a908ea894,BE9558,2026-05-16T22:27:40,K558LLB7.Q40W,Long,0.2881,0.2930,0.983,0.2,0 +f75c52b4-21b9-4065-baeb-5cfbd27d130a,BE9558,2026-05-16T22:33:50,K558LLB8.0E0W,Long,0.2881,0.2930,0.984,0.2,0 +10f7496d-fa76-40a7-8bef-dc14e8a41869,BE9558,2026-05-16T22:40:48,K558LLB8.C00W,Long,0.2878,0.2930,0.982,0.2,0 -- 2.54.0 From 5d3963b54579d539f8bbb1840c95b83d2d2e7d3b Mon Sep 17 00:00:00 2001 From: serversdown Date: Tue, 25 Aug 2026 14:25:26 +0000 Subject: [PATCH 20/30] docs: record the 2026-08-25 body-codec and geo-scale findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Brings the protocol reference, CLAUDE.md and the codec RE status doc up to date with everything confirmed in this pass. instantel_protocol_reference.md - Changelog row for the five findings. - S7.6.1: scope table showing the 32000 scale correction applies to series-3 waveform, series-3 histogram and series-4 Thor alike, with the measured before/after ratios for each. - S15: closed "Full channel ID mapping in SUB 5A stream" — resolved by the segment-header channel id ([channel][00][00][segment], 0x46=Tran 0x47=Vert 0x48=Long 0x49=MicL, 1697/1697 verified). Four new open questions: variable-prefix segment descriptors, the histogram codec missing peak intervals (26% of channels), UM-series IDF decoding ~1000x low, and the Thor per-count LSB residual. - NEW Appendix E — Known Device Faults. Documents the field-observed "offset" fault: symptom, why it floods the ACH queue (pedestal exceeds the unit's own geo trigger level), the episode table, the detection rule that works, what the data rules out (not the geophone, not the battery, not environmental, not condensation), and the two remaining candidate mechanisms with the test that separates them. Explicitly flags that it is NOT a decode artifact, since that mistake has already been made once. CLAUDE.md - Body-codec section: the four framing cases and the channel-id finding, with the corpus result. - "What's NOT solved": replaced the stale walker-edge-cases bullet with the four genuinely open items. waveform_codec_re_status.md - Scale scope table matching the protocol reference. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HgTe8CamXAHcAmaQ6QNcog --- CLAUDE.md | 64 ++++++++++++++- docs/instantel_protocol_reference.md | 117 ++++++++++++++++++++++++++- docs/waveform_codec_re_status.md | 13 +++ 3 files changed, 189 insertions(+), 5 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 1e5b2cb..9fc0941 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -223,6 +223,44 @@ custom delta + RLE + variable-width codec. `NN + 2` for int8 blocks). Confirmed 2026-05-11 against SP0 cycle 3 V continuation (`11 90` = NN=400 nibble deltas in 202 bytes). +### Framing cases added 2026-05-11 → 2026-08-25 + +Four more block-framing cases, each of which had been causing **silent +channel truncation** — `walk_body` ends its loop on an unrecognised tag +and `decode_waveform_v2` returns whatever channels it got, so an +unhandled tag surfaces as short channels with no error raised. Found by +diffing 75 production events against their preserved Blastware ASCII +exports (`//_ASCII.TXT`). + +- **Wide-NN RLE `0X NN`** — the 12-bit NN encoding documented above for + `1X`/`2X` **also applies to the `00 NN` RLE tag**. A narrow run maxes + out at NN=0xFC, so a quiet stretch longer than 252 samples must use + the wide form (e.g. `01 0c` = 268 repeats). +- **`30 NN` is not capped at NN=0x10** — data-section blocks reach at + least NN=0x18. The `NN × 1.5 + 2` length formula was already right; + only the guard was wrong. +- **`40 NN` segment headers are variable width** — NN is the *count of + int16 BE continuation deltas for the PREVIOUS channel*, so the header + is `2*NN + 16` bytes and every field after the deltas shifts by + `2*NN`. `40 01` (18 B) and `40 03` (22 B) both occur alongside the + common `40 02` (20 B). +- **Tagless segment headers** — a header can appear with **no `40 NN` + tag at all**: just the 14-byte tail + `[field2:2][len:2][channel_id:4][marker:2][anchors:4]`. This is the + NN=0 case (previous channel needed no continuation deltas). + +**The header "counter" is really a channel id.** The 4-byte field long +documented as a "monotonic uint32 LE counter" is +`[channel_id][00][00][segment_index]`, with `0x46`=Tran `0x47`=Vert +`0x48`=Long `0x49`=MicL — verified on **1697/1697** segment headers +across the corpus, zero disagreements. `decode_waveform_v2` now takes +the channel from this field rather than from rotation position; a single +missed or extra header (exactly what tagless headers caused) desyncs +rotation and corrupts every channel after it. + +Corpus result, end to end through the production path: +**exact 37 → 72, truncated 23 → 3, full-length value errors 15 → 0.** + ### What's NOT solved - **MicL channel conversion to dB(L)** — the codec emits MicL as @@ -230,9 +268,29 @@ custom delta + RLE + variable-width codec. shows mic in dB(L) with ~6 dB quantization steps. Need to map ADC counts → dB(L) for direct comparison; likely `dB = 20*log10(|counts|) + offset` or similar. -- **Walker edge cases** — SP0/SS0/SV0 don't walk the full event due - to block-length quirks past the first few segments. Every sample - reached is correct; the walker just needs robustness improvements. +- **Variable-prefix segment descriptors** — 3 of the 75 ground-truth + production events still truncate. The walk reaches a segment header + whose channel-id field is preceded by a *variable-width* prefix (2, 4 + or 6 bytes observed; the standard tagless form always has 4), carrying + an `01 00` marker instead of `02 00`. The marker is **not** simply an + anchor count — `01 00` records appear with both 2- and 4-byte anchor + fields in the same file. Examples: `BE12599/N599LPNB.JF0W` @1155, + `BE12599/N599LPWJ.980W` @849, `BE9558/K558LOF2.820W` @1485. +- **Histogram codec misses the peak interval** — 301 of 1141 series-3 + histogram channels (26%) decode a peak below 95% of the + device-reported PPV (one reads 0.0300 against a device PPV of 0.1200, + exactly 1/4). Not a scale error — that would be a uniform 2.34%. + Lives in `histogram_codec.decode_histogram_body`, untouched by the + 2026-08-25 waveform pass. +- **Micromate (UM-series) IDF decode is ~1000× low** — e.g. + `UM11402_20260406130113.IDFW` gives a Tran peak of 0.0009 in/s against + a device-reported 1.1168. The Thor IDF path decodes sanely, so this + is UM-specific. +- **Thor IDF per-count LSB** — after the 32000 geo full-scale + correction, series-4 Thor peaks sit at a median 0.983 of the + device-reported peak (was 0.960 under 32768). Closer but not exact; + Thor likely uses its own per-count LSB rather than the BW + 16-count/0.005 in/s convention. ### Decoded sample counts (across the fixture bundle) diff --git a/docs/instantel_protocol_reference.md b/docs/instantel_protocol_reference.md index 6c25d27..f4b118f 100644 --- a/docs/instantel_protocol_reference.md +++ b/docs/instantel_protocol_reference.md @@ -11,6 +11,7 @@ | Date | Section | Change | |---|---|---| +| 2026-08-25 | §7.6.1, §15, Appendix E (NEW) | **BODY CODEC + SCALE PASS — five findings, all verified against 75 production events paired with their preserved Blastware ASCII exports.** (1) **Geo full scale is 32000 ADC counts, not 32768** — one decoder unit (16 counts) is exactly 0.005 in/s, so 10.000 in/s = 32000 counts. Consumers dividing by 32768 read every geophone sample and derived peak **2.34% low**; the error scales with amplitude so it was invisible on quiet events and worst on loud ones. 216 per-channel comparisons: 32768 → 151/216 exact (worst 0.238 in/s on a 10 in/s event); 32000 → 216/216 exact, worst 1 LSB. Affects waveforms, histograms and series-4 alike. (2) **Wide-NN RLE `0X NN`** — the 12-bit NN encoding already known for `1X`/`2X` also applies to the `00 NN` RLE tag (runs > 252 samples). (3) **`30 NN` is not capped at NN=0x10** — data-section blocks reach at least 0x18; the length formula was already right. (4) **`40 NN` segment headers are variable width** — NN counts the previous-channel continuation deltas, so the header is `2*NN + 16` bytes; `40 01` and `40 03` occur alongside `40 02`. A header can also appear **tagless** (the NN=0 case): just the 14-byte tail. (5) **The header field documented as a "monotonic uint32 LE counter" is really `[channel_id][00][00][segment_index]`** with 0x46=Tran 0x47=Vert 0x48=Long 0x49=MicL — verified on 1697/1697 segment headers, zero disagreements. Decoders should take the channel from this field, not from rotation position. Items (2)–(5) each caused **silent channel truncation**: an unhandled tag ends the walk and the decoder returns short channels with no error. Corpus result end-to-end: exact 37 → 72, truncated 23 → 3, full-length value errors 15 → 0. New Appendix E documents the field-observed "offset" device fault. | | 2026-05-20 | §2, §3, §4.2, §5.1, §5.3, §6, §7.5b, §7.6.1, §7.6.3, §7.6.4, §7.7.2, §7.7.3, §7.7.5, §7.8.4, §7.8.7, §7.9, §8, §11, §12, §13, §14, §15, Appendix D | **DOC AUDIT PASS — accuracy sweep against `CLAUDE.md` + `minimateplus/` code.** Fixed: (1) S3 frames terminate on bare ETX, not DLE+ETX — §2/§3 rewritten. (2) §3 payload layout corrected — byte[1]=flags, byte[2]=SUB (was wrongly labelled DLE/ADDR). (3) §4.2 — probe responses do NOT carry data length; lengths are hardcoded `DATA_LENGTHS` constants. (4) §5.1 — removed stale duplicate "SUB 1C = TRIGGER CONFIG READ" row; SUB 0A lengths corrected from `0x30/0x26` to `0x46/0x2C` (real event / boundary marker). (5) §5.3 — added missing write-frame format (BW_CMD-only doubling, DLE-aware checksum, offset formula, ack format, SUB 71 chunk parameters). (6) §6 — fixed "SUB 06 → channel config read" → event storage range. (7) §7.5b / §8 — added the 10-byte `sub_code=0x03` continuous-mode timestamp variant alongside the 9-byte single-shot layout; peak vector sum location corrected from "fixed offset 87" to `tran_pos − 12` (label-relative). (8) §7.6 / §7.6.1 / §7.6.3 / §7.6.4 — switched compliance-anchor convention from the 10-byte form to the canonical 6-byte `\xbe\x80\x00\x00\x00\x00`; recording_mode confirmed at anchor−8 in BOTH read and write (was wrongly listed as anchor−3 write / anchor−4 read); sample_rate at anchor−6, histogram_interval at anchor−4, record_time at anchor+6; geo_range row added at channel_label+33. (9) §7.7.2 — token byte position corrected from `params[6]` to `params[7]`. (10) §7.8.4 — fi==9 skip marked FIXED (already removed from code); chunk-count totals updated. (11) §7.8.7 — TODO replaced with current state of `_decode_a5_metadata_into`. (12) §7.9 — Histogram Interval upgraded ❓ → ✅. (13) §11 — POLL example wire bytes corrected; SUB 5A row added to checksum table. (14) §13 — device-under-test updated for current primary unit (BE11529 / S338.17). (15) §14 — TCP Idle Timeout fixed (0→2 min); Data Forwarding Timeout units clarified. (16) §15 (renumbered from second §14) — open-question items already resolved in CLAUDE.md closed out. (17) Appendix D — extension taxonomy rewritten to reflect the AB0T timestamp encoding (D.5.2/D.5.3); EXTENSION REFUTED warning replaced with the resolved encoding. | | 2026-05-08 | §7.6.1 (RETRACTION) | **❌ RETRACTED — "raw int16 LE 8 bytes/sample-set" body codec was never validated.** The original 4-2-26 confirmation was based on misreading broken-decoder output (full-scale ±32K noise) as evidence the signal had saturated. BW's own 0C peaks for that capture (Tran=0.420 / Vert=3.870 / Long=0.495 in/s) prove the signal was NOT saturated — none of those exceed 13K ADC counts. No event in the project's archive has ever come close to saturation, yet the decoder consistently produces ±32K noise on every event. Conclusion: the body codec is not raw int16 LE; the actual encoding is open. Body byte distribution is heavily skewed (24% `0x00`, 10.5% `0x10`, lots of `10 XX` pairs) — likely a delta encoding with `0x10` as escape, but unverified. Retraction box added at top of §7.6.1; "fully-saturating event" claim removed from channel-identification note. The histogram codec in §7.6.2 IS verified and decoded correctly (different recording mode, 32-byte blocks); use it as a structural hint when reverse-engineering the waveform codec. | | 2026-02-26 | Initial | Document created from first hex dump analysis | @@ -1252,6 +1253,20 @@ pure quantization. This also explains why Blastware reports geo peaks slightly above nominal full scale (e.g. 10.14 in/s): the ADC has headroom past 32000. +**Scope — this is not waveform-specific.** The scale is applied where +ADC counts become physical units, which every event passes through +regardless of which codec produced the samples. Verified after +re-deriving the whole production store: + +| source | median ratio ours/device, 32768 | with 32000 | +|---|---|---| +| series-3 waveform (vs ASCII sample table) | 0.9766 | **1.0000** | +| series-3 histogram (vs ASCII PPV, n=1137) | 0.9766 | **1.0000** | +| series-4 Thor IDF (vs device peak, n=1468) | 0.960 | **0.983** | + +The series-4 figure is closer to correct but not exact — the Thor +per-count LSB is its own open question (see §15). + ###### Unmapped: variable-prefix segment descriptors ❓ OPEN Three of 75 ground-truth production events still truncate. In each, @@ -2994,7 +3009,7 @@ The `.bin` files produced by `s3_bridge` are **not raw wire bytes**. The logger |---|---|---|---| | Timestamp 6-byte format byte[3] — purpose of the separator `0x00` byte | LOW | 2026-02-26 | Not blocking; 9-byte waveform record format (§8.2) fully confirmed without this byte. | | `trail[0]` in serial number response — unit-specific byte, derivation unknown. `trail[1]` resolved as firmware minor version. | MEDIUM | 2026-02-26 | | -| Full channel ID mapping in SUB `5A` stream (01/02/03/04 → which sensor?) | MEDIUM | 2026-02-26 | | +| ~~Full channel ID mapping in SUB `5A` stream~~ — **RESOLVED 2026-08-25 for the waveform body:** every `40 NN` segment header carries `[channel_id][00][00][segment_index]` at `data[2*NN+4]`, with `0x46`=Tran `0x47`=Vert `0x48`=Long `0x49`=MicL. Verified on 1697/1697 segment headers with zero disagreements against the decoded channel rotation. See §7.6.1. | RESOLVED | 2026-02-26 | Resolved 2026-08-25 | | ~~Exact byte boundaries of project string fields in SUB `71` write frame~~ — **RESOLVED 2026-05-05:** project/client/operator/seis-loc/extended-notes come from SUB 5A metadata pages at counter `0x1002` / `0x1004` (§7.8.7), NOT from the SUB 71 write payload. `_decode_a5_metadata_into` locates them via ASCII label scans. | RESOLVED | 2026-02-26 | Resolved 2026-05-05 | | Purpose of SUB `09` / response `F6` — 202-byte read block | MEDIUM | 2026-02-26 | | | Purpose of SUB `2E` / response `D1` — 26-byte read block | MEDIUM | 2026-02-26 | | @@ -3021,6 +3036,10 @@ The `.bin` files produced by `s3_bridge` are **not raw wire bytes**. The logger | **ACH inbound server — RESOLVED.** `bridges/ach_server.py` implements full inbound ACH pipeline. `--clear-after-download` flag for delete-after-upload workflow. Post-erase key-reuse detection via `max_downloaded_key` high-water mark. | RESOLVED | 2026-04-11 | | | **Sensor Check dropdown byte location** — byte offset in 1A compliance config payload for the "Sensor Check: Before monitoring / After each event / Disabled" setting is NOT YET LOCATED. Confirmed: unit always runs with "Before monitoring" set. Need a capture with "Disabled" to diff. | MEDIUM | 2026-04-08 | Still open | | **RV55 DCD/DTR default** — newer Sierra Wireless RV55 firmware does not assert DCD/DTR by default, so the MiniMate Plus never detects TCP disconnect and stays idle instead of resuming monitoring. Root cause: RV55 ACEmanager `DCD Control` setting. Workaround not yet found. | MEDIUM | 2026-04-11 | Still open | +| **Variable-prefix segment descriptors** — 3 of 75 ground-truth events still truncate. The walk reaches a segment header whose channel-id field is preceded by a *variable-width* prefix (2, 4 or 6 bytes observed; the standard tagless form always has 4), carrying an `01 00` marker instead of `02 00`. The marker is **not** simply an anchor count — `01 00` records appear with both 2- and 4-byte anchor fields in the same file, so prefix width and marker are not yet reconciled. Examples: `BE12599/N599LPNB.JF0W` @1155, `BE12599/N599LPWJ.980W` @849, `BE9558/K558LOF2.820W` @1485. | MEDIUM | 2026-08-25 | Still open | +| **Histogram codec misses the peak interval** — 301 of 1141 series-3 histogram channels (26%) decode a peak below 95% of the device-reported PPV; one example reads 0.0300 against a device PPV of 0.1200 (exactly 1/4). This is not a scale error (that would be a uniform 2.34%) — the interval carrying the peak is being missed or mis-attributed. Affects `histogram_codec.decode_histogram_body`, untouched by the 2026-08-25 waveform pass. | MEDIUM | 2026-08-25 | Still open | +| **Micromate (UM-series) IDF decode is ~1000x low** — e.g. `UM11402_20260406130113.IDFW` decodes a Tran peak of 0.0009 in/s against a device-reported 1.1168. Distinct from the Thor IDF path, which decodes sanely. Suspect a different per-count LSB or a body offset that does not hold for UM-series files. | MEDIUM | 2026-08-25 | Still open | +| **Thor IDF per-count LSB** — after the 32000 geo full-scale correction, series-4 Thor peaks sit at a median 0.983 of the device-reported peak (was 0.960 under 32768). Closer, but the residual ~1.7% suggests Thor uses its own per-count LSB rather than the BW 16-count/0.005 in/s convention. A code comment in `sfm/waveform_store.py` claims Thor's LSB is 0.0003 in/s, which would predict Thor reading *high* — the measurement shows the opposite, so that comment is unverified. | LOW | 2026-08-25 | Still open | --- @@ -3446,4 +3465,98 @@ file[anc−3]: histogram_interval_LO *All findings reverse-engineered from live RS-232 bridge captures.* *Cross-referenced from 2026-03-02 with Instantel MiniMate Plus Operator Manual (716U0101 Rev 15).* -*This is a living document — append changelog entries and timestamps as new findings are confirmed or corrected.* \ No newline at end of file +*This is a living document — append changelog entries and timestamps as new findings are confirmed or corrected.* + +--- + +## Appendix E — Known Device Faults (field-observed, 2026-08-25) + +This appendix records *device* behaviour, not protocol. It exists +because these signatures are easy to mistake for decoder bugs — and one +of them was, for a while. + +### E.1 The "offset" fault + +**Symptom.** One geophone channel's baseline steps away from zero and +stays there. The trace still carries the real AC signal, but it rides +on a DC pedestal of a few tenths of an in/s. Operators call this an +"offset"; it is a known recurring hardware fault, historically resolved +by returning the unit to Instantel, shelving it until calibration, or a +per-channel re-zero in an advanced/professional Blastware build. + +**Why it floods the queue.** The pedestal exceeds the unit's own geo +trigger level, so the channel sits permanently above threshold and the +unit retriggers as fast as it can rearm — one event every 3–7 minutes +for as long as the fault persists. In the surveyed snapshot the worst +episode produced 193 events in 38 hours. + +| unit | window | events | channel | pedestal | that unit's trigger level | +|---|---|---|---|---|---| +| BE9558 | 2026-05-15 → 05-16 | 193 | Long | +0.335 | 0.200 | +| BE18438 | 2026-02-25 → 02-26 | 64 | Vert | +0.327 | 0.200 | +| BE18438 | 2025-11-15 | 12 | Vert | +0.312 | 0.200 | +| BE9558 | 2026-04-14 / 04-29 / 05-04 | 5 total | Tran | +0.29…+0.44 | 0.200 | + +**It is NOT a decode artifact.** These events reproduce *exactly* in +Blastware's own ASCII export — e.g. `BE12599/N599LQD7.8E0W` Tran reads +mean +0.345, min +0.335, max +0.355 in both our decode and BW's. Any +shape-based false-trigger detector must treat this as a real device +condition, not corrupt data. + +**Detection rule that works:** + +``` +dominant geo axis: |mean| / peak > 0.7 + AND: |mean| >= 0.9 x that unit's geo trigger level +group hits into episodes by serial with a 12 h gap +``` + +A bare `|mean|/peak` threshold is useless — a quiet trace sitting at the +0.010 in/s noise floor clears any ratio test. On the surveyed store the +bare rule flagged 7,184 events; the rule above flags 274, all real. +Candidate list: `scratch/offset_candidates.csv`. + +**What the data rules out.** + +- *Not the geophone.* The on-device sensor check passes on every unit, + offset or not — test frequency 7.2–8.1 Hz, damping ratio 3.3–4.7, + zero failures fleet-wide, including mid-episode. The coil and its + mechanical response are healthy. +- *Not the battery.* 6.6–6.8 V on the affected units, same as the rest. +- *Not environmental.* Only 2 units of 21 ever show it. BE17353 logged + 395 waveform events over nine months with zero occurrences; BE7145, + 295 events, zero. Weather acts on all of them equally. +- *Not condensation.* BE18438's Vert held 0.179 → 0.189 for 34 hours + **including a 10-hour overnight gap**, then stepped to 0.33 at 13:24 + in the afternoon. Dew would peak overnight; it was flat overnight. + +**What the data shows.** The pedestal is *piecewise constant* — it +holds rock-steady, survives power-off gaps, and changes only in discrete +steps. A small common-mode diurnal wobble rides on top of it, but that +wobble is present on the healthy channels too (it is the unit's normal +thermal breathing) and is not the fault. + +**Open: two candidate mechanisms.** + +1. *A latched bad zero.* These units run Sensor Check "Before + monitoring", so a baseline is captured at session start. Disturb the + geophone at that instant and the unit bakes a non-zero reading in as + "zero". Fits the discrete steps, the rock-steadiness between them, + surviving power cycles, the sensor check still passing, and clearing + on a clean re-zero. Also explains why it is episodic rather than + constant — it needs a disturbance at the moment of zeroing. +2. *A degrading analog front-end.* Fits BE9558 better: it was clean for + nine months, then escalated 3 → 1 → 1 → 193 events over five weeks + **and moved from Tran to Long**, which a single bad channel would + not do. + +BE18438 (same channel twice, months apart) looks like (1); BE9558 +(escalating, channel-migrating) looks like (2). The decisive test is to +force a clean re-zero on a faulted unit: if the offset clears and stays +clear it is (1) and fixable in the field — potentially over the wire, +since we already speak SUB `0x0E` (channel sensor data) and `0x98` +(trigger test), see §7.10. + +**Note:** the surveyed population is subject to survivorship bias — +flooded events were routinely deleted before this snapshot, so units +that look clean here may have had episodes that were scrubbed. diff --git a/docs/waveform_codec_re_status.md b/docs/waveform_codec_re_status.md index 47fbc41..1a0b864 100644 --- a/docs/waveform_codec_re_status.md +++ b/docs/waveform_codec_re_status.md @@ -250,6 +250,19 @@ that divided by 32768 read every geophone sample 2.34% low. Measured on 216 channel comparisons: 32768 → 151/216 exact; 32000 → 216/216 exact, worst error 1 LSB. +**Scope — not waveform-specific.** The scale is applied where ADC +counts become physical units, which every event passes through +regardless of source codec: + +| source | median ratio ours/device, 32768 | with 32000 | +|---|---|---| +| series-3 waveform (vs ASCII sample table) | 0.9766 | **1.0000** | +| series-3 histogram (vs ASCII PPV, n=1137) | 0.9766 | **1.0000** | +| series-4 Thor IDF (vs device peak, n=1468) | 0.960 | **0.983** | + +The four block-framing fixes are waveform-only — `histogram_codec` is +untouched by them. + ## Ground-truth corpus (2026-08-25) Beyond the bundled fixtures, the production waveform store keeps each -- 2.54.0 From 4f8224a751e328e345263a13f9f765b182d99517 Mon Sep 17 00:00:00 2001 From: serversdown Date: Tue, 25 Aug 2026 15:06:34 +0000 Subject: [PATCH 21/30] =?UTF-8?q?docs(appendix-e):=20offset=20fault=20is?= =?UTF-8?q?=20geophone-side=20=E2=80=94=20operator=20swap=20test=20+=20Mic?= =?UTF-8?q?L=20evidence?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Operator report: attaching a different geophone to an affected unit makes the offset go away. That rules out the unit's analog front-end and any stored per-channel zero constant (a constant lives in the unit and would survive a sensor swap). The stored data agrees — MicL, a separate transducer on its own cable, shows no offset during either episode (|mean|/peak 0.17 and 0.02) while the geo channels on the same unit at the same moment are pinned. Two distinct sensor-side patterns recorded: BE18438 Vert 0.97, Tran 0.16, Long 0.18 -> one conductor pair BE9558 Long 0.99, Tran 0.90, Vert 0.81 -> shared return / ground Candidate mechanisms narrowed to three, since a geophone coil is passive and cannot generate sustained DC: galvanic corrosion at a connector or splice (matches the ~46 mV referred to the ADC input), a leakage path to shield, or changed coil DC resistance interacting with the amplifier's input bias current. Also records the confound: swapping a sensor requires a monitoring restart, and these units run Sensor Check "Before monitoring", so the restart re-zeros too. The swap does not cleanly separate "new sensor" from "the restart re-zeroed it". Controls and the single best measurement (open-circuit DC across the suspect connector) documented. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HgTe8CamXAHcAmaQ6QNcog --- docs/instantel_protocol_reference.md | 75 +++++++++++++++++++++------- 1 file changed, 57 insertions(+), 18 deletions(-) diff --git a/docs/instantel_protocol_reference.md b/docs/instantel_protocol_reference.md index f4b118f..8a8115e 100644 --- a/docs/instantel_protocol_reference.md +++ b/docs/instantel_protocol_reference.md @@ -3536,26 +3536,65 @@ steps. A small common-mode diurnal wobble rides on top of it, but that wobble is present on the healthy channels too (it is the unit's normal thermal breathing) and is not the fault. -**Open: two candidate mechanisms.** +**The fault is on the geophone side.** Operator report (2026-08-25): +*attaching a different geophone to an affected unit makes the offset go +away.* That is the channel-swap test, already run in the field many +times, and it rules out the unit's analog front-end and any stored +per-channel zero constant — a constant lives in the unit and would +survive a sensor swap. -1. *A latched bad zero.* These units run Sensor Check "Before - monitoring", so a baseline is captured at session start. Disturb the - geophone at that instant and the unit bakes a non-zero reading in as - "zero". Fits the discrete steps, the rock-steadiness between them, - surviving power cycles, the sensor check still passing, and clearing - on a clean re-zero. Also explains why it is episodic rather than - constant — it needs a disturbance at the moment of zeroing. -2. *A degrading analog front-end.* Fits BE9558 better: it was clean for - nine months, then escalated 3 → 1 → 1 → 193 events over five weeks - **and moved from Tran to Long**, which a single bad channel would - not do. +The stored data agrees. MicL — a separate transducer on its own cable — +shows no offset during either episode (`|mean|/peak` = 0.17 and 0.02), +while the geo channels on the same unit at the same moment are pinned. -BE18438 (same channel twice, months apart) looks like (1); BE9558 -(escalating, channel-migrating) looks like (2). The decisive test is to -force a clean re-zero on a faulted unit: if the offset clears and stays -clear it is (1) and fixable in the field — potentially over the wire, -since we already speak SUB `0x0E` (channel sensor data) and `0x98` -(trigger test), see §7.10. +**Two distinct failure patterns**, both geophone-side: + +| unit | Tran | Vert | Long | MicL | reading | +|---|---|---|---|---|---| +| BE18438 | 0.16 | **0.97** | 0.18 | 0.02 | one element only | +| BE9558 | 0.90 | 0.81 | **0.99** | 0.17 | all three, Long dominant | + +(values are `\|mean\|/peak` on the dominant axis) + +BE18438 has a single faulted conductor pair with the other two clean. +BE9558 has all three geo channels offset by different amounts +(Long +0.340, Tran +0.037, Vert −0.020) — that points at a shared return +or ground path in the housing rather than one element. + +**Candidate mechanisms.** A geophone coil is passive and cannot +generate sustained DC on its own, so an offset originating on the sensor +side has to come from one of: + +1. *Galvanic corrosion at a connector or splice* — dissimilar metals + plus moisture form a literal cell. This generates DC directly, in + the tens-of-millivolts range that matches the measured offset + (0.29 in/s ÷ 6.206053 in/s per volt ≈ 46 mV referred to the ADC + input). Fits the discrete steps, surviving power cycles, and being + cured by swapping the sensor. +2. *A leakage path to shield or another conductor*, letting the unit's + own bias/reference divide into the signal pair. Best fit for the + BE9558 all-three-channels pattern. +3. *Changed coil DC resistance* (partial short or degraded winding) + interacting with the amplifier's input bias current — + `V_offset = I_bias × R_source`. Best fit for the BE18438 + single-element pattern. Note the on-device sensor check would not + catch this: it measures ring-down frequency and damping, not DC. + +**Caveat on the swap test.** Swapping a geophone requires stopping and +restarting monitoring, and these units run Sensor Check "Before +monitoring" — so the restart also re-zeros the channels. The swap +therefore does not cleanly separate *"new sensor"* from *"the restart +re-zeroed it"*. Two cheap controls settle it: restart monitoring +**without** swapping (if the offset clears, it was the re-zero), or swap +the **original** sensor back (if the offset returns, it is the sensor). +The operator's remedy being a swap rather than a restart is weak +evidence for the sensor, since a restart is the easier thing to try +first. + +**Best single measurement when it recurs:** DC voltage across the +suspect geophone's connector pins with the sensor **disconnected from +the unit**. A galvanic cell (mechanism 1) shows an open-circuit +voltage; mechanisms 2 and 3 do not. **Note:** the surveyed population is subject to survivorship bias — flooded events were routinely deleted before this snapshot, so units -- 2.54.0 From e449ac04af22bb2381fbe72ceea3977ac15cbd35 Mon Sep 17 00:00:00 2001 From: serversdown Date: Tue, 25 Aug 2026 16:23:51 +0000 Subject: [PATCH 22/30] =?UTF-8?q?docs:=20sharpen=20the=20series-3=20histog?= =?UTF-8?q?ram=20open=20item=20=E2=80=94=20dropped=20intervals,=20not=20wr?= =?UTF-8?q?ong=20values?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-measured properly. The first pass compared h5 max against the ASCII header PPV and reported "26% of channels miss the peak". The histogram ASCII actually carries a full per-interval data table (Tran/Vert/Long peak + freq + PVS per interval), which is real ground truth, so the comparison should have been per-interval from the start. Per-interval result, n=1196 series-3 histograms: - decoded VALUES are right: 1031/1196 (86%) match within 1 LSB across the overlapping prefix - the interval COUNT is short in 1195 of 1196 files: median 1 missing, 1088 short by 1-2, 65 by 3-10, 39 by 11-100, 3 by >100 (max 205) - decoded max falls below the device PPV in 169/1196 files (14%), not 26% — that happens when a dropped interval held the peak So it is a termination bug in histogram_codec.decode_histogram_body, the same family as the waveform-walker truncation fixed earlier today, rather than mis-decoded interval values. Series-3 only; there is no preserved series-4 ASCII in the snapshot to compare against. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HgTe8CamXAHcAmaQ6QNcog --- CLAUDE.md | 19 +++++++++++++------ docs/instantel_protocol_reference.md | 2 +- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 9fc0941..8623df4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -276,12 +276,19 @@ Corpus result, end to end through the production path: anchor count — `01 00` records appear with both 2- and 4-byte anchor fields in the same file. Examples: `BE12599/N599LPNB.JF0W` @1155, `BE12599/N599LPWJ.980W` @849, `BE9558/K558LOF2.820W` @1485. -- **Histogram codec misses the peak interval** — 301 of 1141 series-3 - histogram channels (26%) decode a peak below 95% of the - device-reported PPV (one reads 0.0300 against a device PPV of 0.1200, - exactly 1/4). Not a scale error — that would be a uniform 2.34%. - Lives in `histogram_codec.decode_histogram_body`, untouched by the - 2026-08-25 waveform pass. +- **Histogram codec drops trailing intervals (series-3)** — the + per-interval *values* are right: measured against the histogram ASCII + data table (Tran/Vert/Long peak + freq + PVS per interval), 1031 of + 1196 files (86%) match within 1 LSB across the overlapping prefix. + The interval **count** is short in 1195 of 1196 files — median 1 + missing; 1088 short by 1–2, 65 by 3–10, 39 by 11–100, 3 by >100 (max + 205). When a dropped interval holds the event peak, the decoded max + falls below the device-reported PPV: 169/1196 files (14%). A + termination bug in `histogram_codec.decode_histogram_body` — same + family as the waveform-walker truncation fixed 2026-08-25, and + untouched by it. Note: the histogram ASCII carries a full + per-interval table, so this has proper ground truth available (1211 + files in the prod snapshot). - **Micromate (UM-series) IDF decode is ~1000× low** — e.g. `UM11402_20260406130113.IDFW` gives a Tran peak of 0.0009 in/s against a device-reported 1.1168. The Thor IDF path decodes sanely, so this diff --git a/docs/instantel_protocol_reference.md b/docs/instantel_protocol_reference.md index 8a8115e..bf272cd 100644 --- a/docs/instantel_protocol_reference.md +++ b/docs/instantel_protocol_reference.md @@ -3037,7 +3037,7 @@ The `.bin` files produced by `s3_bridge` are **not raw wire bytes**. The logger | **Sensor Check dropdown byte location** — byte offset in 1A compliance config payload for the "Sensor Check: Before monitoring / After each event / Disabled" setting is NOT YET LOCATED. Confirmed: unit always runs with "Before monitoring" set. Need a capture with "Disabled" to diff. | MEDIUM | 2026-04-08 | Still open | | **RV55 DCD/DTR default** — newer Sierra Wireless RV55 firmware does not assert DCD/DTR by default, so the MiniMate Plus never detects TCP disconnect and stays idle instead of resuming monitoring. Root cause: RV55 ACEmanager `DCD Control` setting. Workaround not yet found. | MEDIUM | 2026-04-11 | Still open | | **Variable-prefix segment descriptors** — 3 of 75 ground-truth events still truncate. The walk reaches a segment header whose channel-id field is preceded by a *variable-width* prefix (2, 4 or 6 bytes observed; the standard tagless form always has 4), carrying an `01 00` marker instead of `02 00`. The marker is **not** simply an anchor count — `01 00` records appear with both 2- and 4-byte anchor fields in the same file, so prefix width and marker are not yet reconciled. Examples: `BE12599/N599LPNB.JF0W` @1155, `BE12599/N599LPWJ.980W` @849, `BE9558/K558LOF2.820W` @1485. | MEDIUM | 2026-08-25 | Still open | -| **Histogram codec misses the peak interval** — 301 of 1141 series-3 histogram channels (26%) decode a peak below 95% of the device-reported PPV; one example reads 0.0300 against a device PPV of 0.1200 (exactly 1/4). This is not a scale error (that would be a uniform 2.34%) — the interval carrying the peak is being missed or mis-attributed. Affects `histogram_codec.decode_histogram_body`, untouched by the 2026-08-25 waveform pass. | MEDIUM | 2026-08-25 | Still open | +| **Histogram codec drops trailing intervals (series-3)** — measured per-interval against the histogram ASCII data table (which carries Tran/Vert/Long peak + freq + PVS per interval), n=1196: the decoded per-interval **values are right** — 1031/1196 (86%) match the ASCII table within 1 LSB across the overlapping prefix — but the interval **count** is short in 1195 of 1196 files. Median 1 interval missing; 1088 files short by 1–2, 65 by 3–10, 39 by 11–100, 3 by more than 100 (max 205). When a dropped interval is the one holding the event peak, the decoded max falls below the device-reported PPV — 169/1196 (14%) of files. This is a termination bug in `histogram_codec.decode_histogram_body`, the same family as the waveform-walker truncation fixed 2026-08-25, and it was untouched by that pass. | MEDIUM | 2026-08-25 | Still open | | **Micromate (UM-series) IDF decode is ~1000x low** — e.g. `UM11402_20260406130113.IDFW` decodes a Tran peak of 0.0009 in/s against a device-reported 1.1168. Distinct from the Thor IDF path, which decodes sanely. Suspect a different per-count LSB or a body offset that does not hold for UM-series files. | MEDIUM | 2026-08-25 | Still open | | **Thor IDF per-count LSB** — after the 32000 geo full-scale correction, series-4 Thor peaks sit at a median 0.983 of the device-reported peak (was 0.960 under 32768). Closer, but the residual ~1.7% suggests Thor uses its own per-count LSB rather than the BW 16-count/0.005 in/s convention. A code comment in `sfm/waveform_store.py` claims Thor's LSB is 0.0003 in/s, which would predict Thor reading *high* — the measurement shows the opposite, so that comment is unverified. | LOW | 2026-08-25 | Still open | -- 2.54.0 From ef1e99b0a0211eb1f113bb85bfaec36d305d47cd Mon Sep 17 00:00:00 2001 From: serversdown Date: Tue, 25 Aug 2026 18:59:25 +0000 Subject: [PATCH 23/30] =?UTF-8?q?fix(histogram):=20block=20is=20big-endian?= =?UTF-8?q?=20+=20terminal=20block=20tail=20=E2=80=94=201/1196=20to=201211?= =?UTF-8?q?/1211?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two errors in the series-3 histogram block model, both found by diffing against the per-interval data table in the preserved Blastware ASCII exports (1211 files in the prod snapshot — far stronger ground truth than the header PPV used previously). 1. The block is uniformly BIG-ENDIAN. Peaks and half-periods are uint16 BE (T_peak [5:7], T_halfperiod [7:9], V_peak [9:11], V_halfperiod [11:13], L_peak [13:15], L_halfperiod [15:17], M_peak [17:19], M_halfperiod [19:21]); only block_ctr [2:4] is little-endian. The old uint8-peak model silently CLIPPED any peak above 1.275 in/s: the final interval of BE18193/T193LQ9K.OE0H reads 8.270 in/s in BW's export (1654 counts = 0x0676) and decoded as 0x76 = 118 = 0.590. The byte documented as a per-channel "annotation" was never an annotation — it is the half-period's high byte, which is exactly why it was non-zero on the sub-Hz intervals BW renders as "<1.0". The marker is block[4] alone. Testing [4:6] as a uint16 LE marker forced block[5] == 0, which is what capped the peak at one byte. 2. The final block of each stream carries tail 9c 06 00 42 instead of 1e 0a 00 00, and holds arbitrary bytes at [21:23]. Rejecting it dropped the last interval of nearly every histogram — frequently the interval holding the event peak, so the file's PPV read low. Verified end to end through the production path: 1211/1211 histograms decode exactly (interval count + every per-interval peak), plus 842,442 per-interval frequency comparisons with zero mismatches. Previously 1 of 1196 files was fully correct. decode_histogram_body_full records expose `is_terminal` in place of the removed `annotations` tuple. +6 tests. No regressions: full-suite failure list unchanged from baseline. NOTE: stored histogram .h5 files need regenerating to pick this up. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HgTe8CamXAHcAmaQ6QNcog --- CHANGELOG.md | 30 ++++++ CLAUDE.md | 41 +++++--- docs/histogram_codec_re_status.md | 27 +++++ docs/instantel_protocol_reference.md | 3 +- minimateplus/histogram_codec.py | 148 ++++++++++++++++----------- tests/test_histogram_codec.py | 104 +++++++++++++++++-- 6 files changed, 269 insertions(+), 84 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a2a7819..07dc116 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,36 @@ All notable changes to seismo-relay are documented here. ### Fixed +- **Series-3 histogram block is uniformly big-endian, and the stream's final + block has its own tail — the codec was clipping large peaks and dropping the + last interval of nearly every histogram.** + + - **Peaks and half-periods are `uint16` big-endian**, not `uint8` plus an + "annotation" byte: `T_peak` `[5:7]`, `T_halfperiod` `[7:9]`, `V_peak` + `[9:11]`, and so on. Only `block_ctr` `[2:4]` is little-endian. The old + model silently **clipped any peak above 1.275 in/s** — the final interval + of `BE18193/T193LQ9K.OE0H` reads 8.270 in/s in Blastware's own export and + decoded as 0.590. The "annotation" byte was the half-period's high byte, + which is why it was non-zero exactly on the sub-Hz intervals BW renders + as `<1.0`. + - **The marker is `block[4]` alone.** Testing `[4:6]` as a `uint16 LE` + marker forced `block[5] == 0` — which is what capped the peak at one byte + in the first place. + - **The last block of each stream carries tail `9c 06 00 42`** instead of + `1e 0a 00 00`, with arbitrary bytes at `[21:23]`. Rejecting it dropped + the final interval of nearly every histogram, and that interval is + frequently the one holding the event peak — so the file's reported PPV + came out low. + + Verified against **1211 production histograms** paired with their preserved + Blastware ASCII exports, which carry a full per-interval data table: + **1211/1211 now decode exactly** (interval count plus every per-interval + peak), and 842,442 per-interval frequency comparisons match with zero + mismatches. Before this fix: **1 of 1196**. + + `decode_histogram_body_full` records now expose `is_terminal` in place of + the removed `annotations` tuple. + - **Geophone full scale is 32000 ADC counts, not 32768 — every geo reading was 2.3% low.** The verified body codec emits geo samples in 16-count units whose documented LSB is exactly 0.005 in/s, and `decoded_to_adc_counts` multiplies diff --git a/CLAUDE.md b/CLAUDE.md index 8623df4..9af798e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -261,6 +261,32 @@ rotation and corrupts every channel after it. Corpus result, end to end through the production path: **exact 37 → 72, truncated 23 → 3, full-length value errors 15 → 0.** +### Histogram codec — corrected 2026-08-25 + +The histogram block is **uniformly big-endian**, and the stream's final +block has its own tail signature. Two long-standing errors: + +- **Peaks and half-periods are `uint16` big-endian**, not `uint8` + + an "annotation" byte. `T_peak` is `[5:7]`, `T_halfperiod` `[7:9]`, + `V_peak` `[9:11]`, and so on; only `block_ctr` at `[2:4]` is LE. + The old model silently **clipped any peak above 1.275 in/s** — the + final interval of `BE18193/T193LQ9K.OE0H` reads 8.270 in/s in BW's + export and decoded as 0.590. The "annotation" byte was the + half-period's high byte, which is why it was non-zero exactly on the + sub-Hz intervals BW renders as `<1.0`. +- **The marker is `block[4]` alone.** Testing `[4:6]` as a uint16 LE + marker forced `block[5] == 0`, which is what capped the peak at one + byte in the first place. +- **The last block of the stream carries tail `9c 06 00 42`** instead of + `1e 0a 00 00`, with arbitrary bytes at `[21:23]`. Rejecting it + dropped the final interval of nearly every histogram — frequently the + one holding the event peak, so the file's PPV read low. + +Verified against 1211 production histograms paired with their BW ASCII +exports: **1211/1211 decode exactly** (interval count plus every +per-interval peak), and 842,442 per-interval frequency comparisons match +with zero mismatches. Before: 1 of 1196. + ### What's NOT solved - **MicL channel conversion to dB(L)** — the codec emits MicL as @@ -276,19 +302,8 @@ Corpus result, end to end through the production path: anchor count — `01 00` records appear with both 2- and 4-byte anchor fields in the same file. Examples: `BE12599/N599LPNB.JF0W` @1155, `BE12599/N599LPWJ.980W` @849, `BE9558/K558LOF2.820W` @1485. -- **Histogram codec drops trailing intervals (series-3)** — the - per-interval *values* are right: measured against the histogram ASCII - data table (Tran/Vert/Long peak + freq + PVS per interval), 1031 of - 1196 files (86%) match within 1 LSB across the overlapping prefix. - The interval **count** is short in 1195 of 1196 files — median 1 - missing; 1088 short by 1–2, 65 by 3–10, 39 by 11–100, 3 by >100 (max - 205). When a dropped interval holds the event peak, the decoded max - falls below the device-reported PPV: 169/1196 files (14%). A - termination bug in `histogram_codec.decode_histogram_body` — same - family as the waveform-walker truncation fixed 2026-08-25, and - untouched by it. Note: the histogram ASCII carries a full - per-interval table, so this has proper ground truth available (1211 - files in the prod snapshot). +(The series-3 histogram codec was fixed 2026-08-25 — see below.) + - **Micromate (UM-series) IDF decode is ~1000× low** — e.g. `UM11402_20260406130113.IDFW` gives a Tran peak of 0.0009 in/s against a device-reported 1.1168. The Thor IDF path decodes sanely, so this diff --git a/docs/histogram_codec_re_status.md b/docs/histogram_codec_re_status.md index 6fa388c..2f4c323 100644 --- a/docs/histogram_codec_re_status.md +++ b/docs/histogram_codec_re_status.md @@ -1,3 +1,30 @@ +> ## SUPERSEDED 2026-08-25 — the block is uniformly BIG-ENDIAN +> +> The `uint8` peak / `annotation` byte model described below is wrong, +> though it decoded quiet data correctly. The real layout: +> +> - **Every per-channel field is `uint16` big-endian.** `T_peak` is +> `[5:7]`, `T_halfperiod` `[7:9]`, `V_peak` `[9:11]`, and so on. +> Only `block_ctr` at `[2:4]` is little-endian. +> - The **marker is `block[4]` alone**, not a `uint16 LE` at `[4:6]`. +> Testing `[4:6] == 10` forced `block[5] == 0`, which is exactly what +> capped every geo peak at one byte (255 counts = 1.275 in/s). +> - The **"annotation" byte was never an annotation** — it is the high +> byte of the big-endian half-period. That is why it was non-zero +> precisely on the sub-Hz intervals Blastware renders as `<1.0`. +> - The **final block of the stream carries tail `9c 06 00 42`** instead +> of `1e 0a 00 00`, and arbitrary bytes at `[21:23]`. Rejecting it +> dropped the last interval of nearly every histogram — often the one +> holding the event peak, so the file's PPV read low. +> +> Verified against 1211 production histograms paired with their Blastware +> ASCII exports: **1211/1211 decode exactly** (interval count plus every +> per-interval peak), and 842,442 per-interval frequency comparisons match +> with zero mismatches. The uint8 model scored 1204/1211 — the seven +> failures are exactly the files containing a peak above 1.275 in/s. +> +> The section below is retained as the reasoning trail. + # Histogram body codec — FULLY DECODED (2026-05-20) Clean working status doc for the MiniMate Plus histogram-mode event diff --git a/docs/instantel_protocol_reference.md b/docs/instantel_protocol_reference.md index bf272cd..9059658 100644 --- a/docs/instantel_protocol_reference.md +++ b/docs/instantel_protocol_reference.md @@ -11,6 +11,7 @@ | Date | Section | Change | |---|---|---| +| 2026-08-25 (2) | S7.6.2, S15 | **HISTOGRAM BLOCK IS BIG-ENDIAN + terminal block tail.** The 32-byte histogram block's per-channel fields are uint16 **big-endian** (`T_peak` [5:7], `T_halfperiod` [7:9], `V_peak` [9:11], `V_halfperiod` [11:13], `L_peak` [13:15], `L_halfperiod` [15:17], `M_peak` [17:19], `M_halfperiod` [19:21]); only `block_ctr` [2:4] is little-endian. The marker is `block[4]` alone - the previous uint16 LE marker test at [4:6] forced `block[5] == 0` and thereby capped every geo peak at 255 counts (1.275 in/s), silently clipping larger peaks. The byte previously documented as a per-channel "annotation" is the high byte of the big-endian half-period, which is why it was non-zero exactly on sub-Hz intervals. Separately, the **final block of each stream carries tail `9c 06 00 42`** rather than `1e 0a 00 00` and holds arbitrary bytes at [21:23]; rejecting it dropped the last interval of nearly every histogram. Verified against 1211 production histograms paired with their Blastware ASCII exports: 1211/1211 decode exactly, plus 842,442 per-interval frequency comparisons with zero mismatches (previously 1 of 1196 files fully correct). | | 2026-08-25 | §7.6.1, §15, Appendix E (NEW) | **BODY CODEC + SCALE PASS — five findings, all verified against 75 production events paired with their preserved Blastware ASCII exports.** (1) **Geo full scale is 32000 ADC counts, not 32768** — one decoder unit (16 counts) is exactly 0.005 in/s, so 10.000 in/s = 32000 counts. Consumers dividing by 32768 read every geophone sample and derived peak **2.34% low**; the error scales with amplitude so it was invisible on quiet events and worst on loud ones. 216 per-channel comparisons: 32768 → 151/216 exact (worst 0.238 in/s on a 10 in/s event); 32000 → 216/216 exact, worst 1 LSB. Affects waveforms, histograms and series-4 alike. (2) **Wide-NN RLE `0X NN`** — the 12-bit NN encoding already known for `1X`/`2X` also applies to the `00 NN` RLE tag (runs > 252 samples). (3) **`30 NN` is not capped at NN=0x10** — data-section blocks reach at least 0x18; the length formula was already right. (4) **`40 NN` segment headers are variable width** — NN counts the previous-channel continuation deltas, so the header is `2*NN + 16` bytes; `40 01` and `40 03` occur alongside `40 02`. A header can also appear **tagless** (the NN=0 case): just the 14-byte tail. (5) **The header field documented as a "monotonic uint32 LE counter" is really `[channel_id][00][00][segment_index]`** with 0x46=Tran 0x47=Vert 0x48=Long 0x49=MicL — verified on 1697/1697 segment headers, zero disagreements. Decoders should take the channel from this field, not from rotation position. Items (2)–(5) each caused **silent channel truncation**: an unhandled tag ends the walk and the decoder returns short channels with no error. Corpus result end-to-end: exact 37 → 72, truncated 23 → 3, full-length value errors 15 → 0. New Appendix E documents the field-observed "offset" device fault. | | 2026-05-20 | §2, §3, §4.2, §5.1, §5.3, §6, §7.5b, §7.6.1, §7.6.3, §7.6.4, §7.7.2, §7.7.3, §7.7.5, §7.8.4, §7.8.7, §7.9, §8, §11, §12, §13, §14, §15, Appendix D | **DOC AUDIT PASS — accuracy sweep against `CLAUDE.md` + `minimateplus/` code.** Fixed: (1) S3 frames terminate on bare ETX, not DLE+ETX — §2/§3 rewritten. (2) §3 payload layout corrected — byte[1]=flags, byte[2]=SUB (was wrongly labelled DLE/ADDR). (3) §4.2 — probe responses do NOT carry data length; lengths are hardcoded `DATA_LENGTHS` constants. (4) §5.1 — removed stale duplicate "SUB 1C = TRIGGER CONFIG READ" row; SUB 0A lengths corrected from `0x30/0x26` to `0x46/0x2C` (real event / boundary marker). (5) §5.3 — added missing write-frame format (BW_CMD-only doubling, DLE-aware checksum, offset formula, ack format, SUB 71 chunk parameters). (6) §6 — fixed "SUB 06 → channel config read" → event storage range. (7) §7.5b / §8 — added the 10-byte `sub_code=0x03` continuous-mode timestamp variant alongside the 9-byte single-shot layout; peak vector sum location corrected from "fixed offset 87" to `tran_pos − 12` (label-relative). (8) §7.6 / §7.6.1 / §7.6.3 / §7.6.4 — switched compliance-anchor convention from the 10-byte form to the canonical 6-byte `\xbe\x80\x00\x00\x00\x00`; recording_mode confirmed at anchor−8 in BOTH read and write (was wrongly listed as anchor−3 write / anchor−4 read); sample_rate at anchor−6, histogram_interval at anchor−4, record_time at anchor+6; geo_range row added at channel_label+33. (9) §7.7.2 — token byte position corrected from `params[6]` to `params[7]`. (10) §7.8.4 — fi==9 skip marked FIXED (already removed from code); chunk-count totals updated. (11) §7.8.7 — TODO replaced with current state of `_decode_a5_metadata_into`. (12) §7.9 — Histogram Interval upgraded ❓ → ✅. (13) §11 — POLL example wire bytes corrected; SUB 5A row added to checksum table. (14) §13 — device-under-test updated for current primary unit (BE11529 / S338.17). (15) §14 — TCP Idle Timeout fixed (0→2 min); Data Forwarding Timeout units clarified. (16) §15 (renumbered from second §14) — open-question items already resolved in CLAUDE.md closed out. (17) Appendix D — extension taxonomy rewritten to reflect the AB0T timestamp encoding (D.5.2/D.5.3); EXTENSION REFUTED warning replaced with the resolved encoding. | | 2026-05-08 | §7.6.1 (RETRACTION) | **❌ RETRACTED — "raw int16 LE 8 bytes/sample-set" body codec was never validated.** The original 4-2-26 confirmation was based on misreading broken-decoder output (full-scale ±32K noise) as evidence the signal had saturated. BW's own 0C peaks for that capture (Tran=0.420 / Vert=3.870 / Long=0.495 in/s) prove the signal was NOT saturated — none of those exceed 13K ADC counts. No event in the project's archive has ever come close to saturation, yet the decoder consistently produces ±32K noise on every event. Conclusion: the body codec is not raw int16 LE; the actual encoding is open. Body byte distribution is heavily skewed (24% `0x00`, 10.5% `0x10`, lots of `10 XX` pairs) — likely a delta encoding with `0x10` as escape, but unverified. Retraction box added at top of §7.6.1; "fully-saturating event" claim removed from channel-identification note. The histogram codec in §7.6.2 IS verified and decoded correctly (different recording mode, 32-byte blocks); use it as a structural hint when reverse-engineering the waveform codec. | @@ -3037,7 +3038,7 @@ The `.bin` files produced by `s3_bridge` are **not raw wire bytes**. The logger | **Sensor Check dropdown byte location** — byte offset in 1A compliance config payload for the "Sensor Check: Before monitoring / After each event / Disabled" setting is NOT YET LOCATED. Confirmed: unit always runs with "Before monitoring" set. Need a capture with "Disabled" to diff. | MEDIUM | 2026-04-08 | Still open | | **RV55 DCD/DTR default** — newer Sierra Wireless RV55 firmware does not assert DCD/DTR by default, so the MiniMate Plus never detects TCP disconnect and stays idle instead of resuming monitoring. Root cause: RV55 ACEmanager `DCD Control` setting. Workaround not yet found. | MEDIUM | 2026-04-11 | Still open | | **Variable-prefix segment descriptors** — 3 of 75 ground-truth events still truncate. The walk reaches a segment header whose channel-id field is preceded by a *variable-width* prefix (2, 4 or 6 bytes observed; the standard tagless form always has 4), carrying an `01 00` marker instead of `02 00`. The marker is **not** simply an anchor count — `01 00` records appear with both 2- and 4-byte anchor fields in the same file, so prefix width and marker are not yet reconciled. Examples: `BE12599/N599LPNB.JF0W` @1155, `BE12599/N599LPWJ.980W` @849, `BE9558/K558LOF2.820W` @1485. | MEDIUM | 2026-08-25 | Still open | -| **Histogram codec drops trailing intervals (series-3)** — measured per-interval against the histogram ASCII data table (which carries Tran/Vert/Long peak + freq + PVS per interval), n=1196: the decoded per-interval **values are right** — 1031/1196 (86%) match the ASCII table within 1 LSB across the overlapping prefix — but the interval **count** is short in 1195 of 1196 files. Median 1 interval missing; 1088 files short by 1–2, 65 by 3–10, 39 by 11–100, 3 by more than 100 (max 205). When a dropped interval is the one holding the event peak, the decoded max falls below the device-reported PPV — 169/1196 (14%) of files. This is a termination bug in `histogram_codec.decode_histogram_body`, the same family as the waveform-walker truncation fixed 2026-08-25, and it was untouched by that pass. | MEDIUM | 2026-08-25 | Still open | +| ~~**Histogram codec drops trailing intervals (series-3)**~~ - **RESOLVED 2026-08-25.** Two errors, both in the block model. (1) The block is **uniformly big-endian**: peaks and half-periods are uint16 BE (`T_peak` [5:7], `T_halfperiod` [7:9], `V_peak` [9:11], ...); only `block_ctr` [2:4] is LE. The old uint8-peak + "annotation"-byte model silently clipped any peak above 1.275 in/s, and the "annotation" byte was really the half-period high byte - non-zero exactly on the sub-Hz intervals BW renders `<1.0`. The marker is `block[4]` alone; testing [4:6] as a uint16 LE marker forced `block[5] == 0`, which is what capped the peak at one byte. (2) The **final block of the stream carries tail `9c 06 00 42`** instead of `1e 0a 00 00`, with arbitrary bytes at [21:23]; rejecting it dropped the last interval of nearly every histogram - often the one holding the event peak. Verified on 1211 production histograms vs their BW ASCII exports: **1211/1211 exact** (interval count + every per-interval peak) and 842,442 frequency comparisons with zero mismatches; was 1/1196. | RESOLVED | 2026-08-25 | Resolved 2026-08-25 | | **Micromate (UM-series) IDF decode is ~1000x low** — e.g. `UM11402_20260406130113.IDFW` decodes a Tran peak of 0.0009 in/s against a device-reported 1.1168. Distinct from the Thor IDF path, which decodes sanely. Suspect a different per-count LSB or a body offset that does not hold for UM-series files. | MEDIUM | 2026-08-25 | Still open | | **Thor IDF per-count LSB** — after the 32000 geo full-scale correction, series-4 Thor peaks sit at a median 0.983 of the device-reported peak (was 0.960 under 32768). Closer, but the residual ~1.7% suggests Thor uses its own per-count LSB rather than the BW 16-count/0.005 in/s convention. A code comment in `sfm/waveform_store.py` claims Thor's LSB is 0.0003 in/s, which would predict Thor reading *high* — the measurement shows the opposite, so that comment is unverified. | LOW | 2026-08-25 | Still open | diff --git a/minimateplus/histogram_codec.py b/minimateplus/histogram_codec.py index 36e399d..d60bf07 100644 --- a/minimateplus/histogram_codec.py +++ b/minimateplus/histogram_codec.py @@ -25,38 +25,60 @@ iterate 32-stride and stop before the tail. ──────────────────────────────────────────────────────────────────────────── [0] 0x00 always-zero tag - [1] segment_id (uint8) 0x00..0x03 — 256 blocks per segment - [2:4] block_ctr (uint16 LE) resets each segment (0x0100, 0x0101, …) - [4:6] 0x000a (uint16 LE) constant marker (= 10) - [6] T_peak_count uint8 Tran peak (count × 0.005 → in/s, max 1.275 in/s) - [7] T_annotation uint8 empirically non-zero on intervals with sub-Hz - or unmeasurable Tran freq; meaning not fully RE'd - [8:10] T_halfperiod uint16 LE Tran half-period in samples (freq = 512 / halfp Hz) - [10] V_peak_count uint8 - [11] V_annotation uint8 - [12:14] V_halfperiod uint16 LE - [14] L_peak_count uint8 - [15] L_annotation uint8 - [16:18] L_halfperiod uint16 LE - [18] M_peak_count uint8 MicL peak (count → dB via mic_count_to_db) - [19] M_annotation uint8 - [20:22] M_halfperiod uint16 LE MicL half-period in samples (freq = 512 / halfp Hz) - [22:24] 0x00 0x00 constant + [1] segment_id (uint8) 0x00..0x03 - 256 blocks per segment + [2:4] block_ctr (uint16 LE) resets each segment (0x0100, 0x0101, ...) + [4] 0x0a (uint8) constant marker (= 10) + [5:7] T_peak_count uint16 BE Tran peak (count x 0.005 -> in/s) + [7:9] T_halfperiod uint16 BE Tran half-period in samples (freq = 512 / halfp) + [9:11] V_peak_count uint16 BE + [11:13] V_halfperiod uint16 BE + [13:15] L_peak_count uint16 BE + [15:17] L_halfperiod uint16 BE + [17:19] M_peak_count uint16 BE MicL peak (count -> dB via mic_count_to_db) + [19:21] M_halfperiod uint16 BE MicL half-period in samples + [21:23] 0x00 0x00 constant on standard blocks [24:28] 4-byte variable purpose unknown (possibly CRC or timestamp delta) - [28:32] 0x1e 0x0a 0x00 0x00 constant block-end signature + [28:32] block-end signature see "Two block tails" below -NOTE on peak-count width: an earlier interpretation treated the peak -fields as uint16 LE spanning [6:8] / [10:12] / [14:16] / [18:20]. -That happened to be byte-exact against the N844 fixture corpus only -because every annotation byte in those fixtures was zero, making -``uint16 LE == uint8``. Cross-correlating BE9558 (K558) Tran-drift -and BE18003 (T003) Histogram+Continuous events against the BW ASCII -export proved peak is uint8 alone — see test_histogram_codec.py -and docs/histogram_codec_re_status.md. +**Every per-channel field is uint16 BIG-endian** (confirmed 2026-08-25). +Only ``block_ctr`` at [2:4] is little-endian. -Block-identification anchor: ``block[22:24] == b"\\x00\\x00"`` AND -``block[28:32] == b"\\x1e\\x0a\\x00\\x00"``. This is the reliable -distinguisher from non-block content in the file. +HISTORY - two earlier readings of this block were wrong in ways that +cancelled out on quiet data: + + 1. *peak as uint16 LE at [6:8]* - produced 268 in/s peaks on any + interval whose next byte was non-zero. + 2. *peak as uint8 at [6] with an "annotation" byte at [7]* - correct + for every peak below 256 counts (1.275 in/s), but it silently + **clipped larger peaks**: the final interval of + BE18193/T193LQ9K.OE0H reads 8.270 in/s in BW's export + (1654 counts = 0x0676) and decoded as 0x76 = 118 = 0.590 in/s. + The "annotation" byte was never an annotation - it is the high + byte of the big-endian half-period, which is why it was non-zero + exactly on the sub-Hz intervals BW renders as "<1.0". + +Both readings also forced ``block[5] == 0`` via a bogus ``uint16 LE`` +marker check at [4:6], which is what capped the peak at one byte. +The marker is ``block[4]`` alone. + +Verified 2026-08-25 against 1211 production histograms paired with +their Blastware ASCII exports: **1211/1211 decode exactly** (interval +count plus every per-interval peak), and 842,442 per-interval +frequency comparisons match with **zero** mismatches. + +Two block tails +--------------- +Standard blocks end with ``1e 0a 00 00``. The **final block of the +stream** ends with ``9c 06 00 42`` instead, and carries arbitrary bytes +at [21:23]. Rejecting it dropped the last interval of nearly every +histogram - and the last interval is frequently the one holding the +event peak, so the file's reported PPV came out low. Observed in 1206 +of 1211 production histograms, always positioned after every +standard-tail block. + +Block-identification anchor: ``block[0] == 0x00`` AND +``block[4] == 0x0A`` AND the tail is one of the two signatures above; +standard-tail blocks additionally require ``block[22] == 0x00``. ──────────────────────────────────────────────────────────────────────────── Per-channel encoding @@ -109,6 +131,12 @@ from typing import List, Optional, Tuple # real data block. More distinctive than the byte-22 `00 00` (which # matches many false positives), so we anchor on this. _BLOCK_TAIL = b"\x1e\x0a\x00\x00" + +# The final block of a histogram stream ends with this instead. It is a +# real data block - same layout - and holds the last interval. See the +# module docstring, "Two block tails". +_BLOCK_TAIL_TERMINAL = b"\x9c\x06\x00\x42" + _BLOCK_SIZE = 32 # Marker byte at block[4:6] of every histogram data block. Used as @@ -127,19 +155,25 @@ _FREQ_NUMERATOR = 512 def _is_data_block(block: bytes) -> bool: - """Tight identification of a histogram data block.""" + """Tight identification of a histogram data block. + + Accepts both tail signatures. ``block[4]`` alone is the marker - + ``block[5]`` is the high byte of the Tran peak and is non-zero on any + interval above 1.275 in/s, so it must not be part of the marker test. + The ``block[22] == 0`` constraint is what keeps trailer content out, + but it applies only to standard-tail blocks: terminal blocks carry + arbitrary bytes there. + """ if len(block) < _BLOCK_SIZE: return False - if block[28:32] != _BLOCK_TAIL: - return False - if block[22:24] != b"\x00\x00": - return False if block[0] != 0x00: return False - marker = block[4] | (block[5] << 8) - if marker != _BLOCK_MARKER: + if block[4] != _BLOCK_MARKER: return False - return True + tail = block[28:32] + if tail == _BLOCK_TAIL: + return block[22] == 0x00 + return tail == _BLOCK_TAIL_TERMINAL def _decode_block(block: bytes) -> Optional[dict]: @@ -149,33 +183,23 @@ def _decode_block(block: bytes) -> Optional[dict]: Returns a record with per-channel peak counts (uint8) and half-periods (uint16 LE). """ - # Peak counts are uint8 at bytes [6] / [10] / [14] / [18]. The - # adjacent bytes [7] / [11] / [15] / [19] hold an annotation field - # whose meaning isn't fully understood (empirically non-zero in - # intervals with sub-Hz or unmeasurable geo frequencies, mostly - # zero otherwise — see test fixtures from BE9558/BE18003 corpora). - # Crucially, those annotation bytes are NOT the high byte of the - # peak count: cross-correlating against BW's per-interval ASCII - # export proves the peak is uint8 alone. - # - # Reading the peak as uint16 LE (the original interpretation) was - # accidentally correct only because every block in the N844 fixture - # corpus had a zero annotation byte; non-N844 events with non-zero - # annotation bytes decoded to physically impossible peaks (e.g. - # 268 in/s per channel) and produced 35× inflated PVS sums when - # first run against prod data. See histogram_codec_re_status.md. - t_peak = block[6] - v_peak = block[10] - l_peak = block[14] - m_peak = block[18] - t_halfp = block[8] | (block[9] << 8) - v_halfp = block[12] | (block[13] << 8) - l_halfp = block[16] | (block[17] << 8) - m_halfp = block[20] | (block[21] << 8) + # Every per-channel field is uint16 BIG-endian; only block_ctr is LE. + # See the module docstring for the two superseded readings and why + # each looked correct on quiet data. + def _be16(i: int) -> int: + return (block[i] << 8) | block[i + 1] + + t_peak = _be16(5) + t_halfp = _be16(7) + v_peak = _be16(9) + v_halfp = _be16(11) + l_peak = _be16(13) + l_halfp = _be16(15) + m_peak = _be16(17) + m_halfp = _be16(19) segment_id = block[1] block_ctr = block[2] | (block[3] << 8) var_meta = bytes(block[24:28]) - annotations = (block[7], block[11], block[15], block[19]) return { "segment_id": segment_id, "block_ctr": block_ctr, @@ -188,7 +212,7 @@ def _decode_block(block: bytes) -> Optional[dict]: "m_peak": m_peak, "m_halfp": m_halfp, "meta_var": var_meta, - "annotations": annotations, + "is_terminal": block[28:32] == _BLOCK_TAIL_TERMINAL, } diff --git a/tests/test_histogram_codec.py b/tests/test_histogram_codec.py index 6a42e27..e37f60e 100644 --- a/tests/test_histogram_codec.py +++ b/tests/test_histogram_codec.py @@ -354,23 +354,29 @@ _K558_INTERVAL_12_BLOCK = bytes.fromhex( def test_extension_byte_does_not_inflate_peak(): - """The annotation byte at [7]/[11]/[15]/[19] must NOT contribute to - the peak count. Decoded T_peak must be 3 (uint8 byte[6]), NOT - 53763 (uint16 LE byte[6:8]).""" + """The byte after each peak must NOT contribute to the peak count. + + Still true, but for a different reason than originally recorded: the + block is uniformly **big-endian**, so T_peak is uint16 BE at [5:7] + (= 3 here) and the 0xd2 at [7] is the HIGH BYTE of the big-endian + half-period at [7:9], not an "annotation" field. Reading the peak as + uint16 LE at [6:8] gave 53763 → 268 in/s, which is what this test was + written to prevent. + """ body = _K558_INTERVAL_12_BLOCK records = decode_histogram_body_full(body) assert records is not None assert len(records) == 1 r = records[0] - assert r["t_peak"] == 3, f"T_peak should be 3 (uint8), got {r['t_peak']}" + assert r["t_peak"] == 3, f"T_peak should be 3, got {r['t_peak']}" assert r["v_peak"] == 2 assert r["l_peak"] == 2 assert r["m_peak"] == 16 - # Half-periods unchanged — still uint16 LE. - assert r["t_halfp"] == 0x0045 # 69 → 7.4 Hz + # Half-period is uint16 BE — 0xd245 = 53829 samples → 0.0095 Hz, which + # is exactly the sub-Hz drift BW rendered as "<1.0" for this interval. + assert r["t_halfp"] == 0xd245 + assert half_period_to_hz(r["t_halfp"]) < 1.0 assert r["m_halfp"] == 6 # → 85.3 Hz - # Annotation byte is preserved (for future RE) but does not affect peak. - assert r["annotations"] == (0xd2, 0x00, 0x00, 0x00) def test_extension_byte_decoded_to_correct_in_s(): @@ -383,3 +389,85 @@ def test_extension_byte_decoded_to_correct_in_s(): assert channels["Vert"] == [2] assert channels["Long"] == [2] assert channels["MicL"] == [16] + + +# ── Big-endian block layout + terminal block (2026-08-25) ─────────────────── +# +# Verified against 1211 production histograms paired with their Blastware +# ASCII exports: 1211/1211 decode exactly (interval count + every +# per-interval peak), and 842,442 per-interval frequency comparisons match +# with zero mismatches. + +def _mk_block(t_peak=3, t_halfp=69, v_peak=2, v_halfp=69, l_peak=2, l_halfp=69, + m_peak=16, m_halfp=6, seg=0, ctr=256, tail=b"\x1e\x0a\x00\x00", + b22=0x00): + """Build one synthetic 32-byte histogram block, big-endian throughout.""" + b = bytearray(32) + b[0] = 0x00 + b[1] = seg + b[2], b[3] = ctr & 0xFF, (ctr >> 8) & 0xFF # block_ctr is uint16 LE + b[4] = 0x0A # marker, uint8 + for off, val in ((5, t_peak), (7, t_halfp), (9, v_peak), (11, v_halfp), + (13, l_peak), (15, l_halfp), (17, m_peak), (19, m_halfp)): + b[off], b[off + 1] = (val >> 8) & 0xFF, val & 0xFF # uint16 BE + b[22] = b22 + b[28:32] = tail + return bytes(b) + + +def test_geo_peak_is_uint16_be_not_uint8(): + """A peak above the uint8 ceiling (255 counts = 1.275 in/s) must decode. + + Real example: BE18193/T193LQ9K.OE0H's final interval reads 8.270 in/s + in the BW export = 1654 counts = 0x0676, which needs both bytes. + Reading only byte[6] gave 0x76 = 118 = 0.590 in/s. + """ + r = decode_histogram_body_full(_mk_block(t_peak=1654)) + assert r is not None and len(r) == 1 + assert r[0]["t_peak"] == 1654 + assert geo_count_to_ins(r[0]["t_peak"]) == pytest.approx(8.27) + + +def test_half_period_is_uint16_be(): + """Half-period spans two bytes big-endian; a large value is sub-Hz.""" + r = decode_histogram_body_full(_mk_block(t_halfp=53829)) + assert r[0]["t_halfp"] == 53829 + assert half_period_to_hz(53829) == pytest.approx(512 / 53829) + + +def test_terminal_block_tail_is_accepted(): + """The LAST block of a histogram stream carries tail `9c 06 00 42` + instead of `1e 0a 00 00`. Rejecting it dropped the final interval — + which is where the event peak often lives. Observed in 1206 of 1211 + production histograms, always after every standard-tail block.""" + body = _mk_block(ctr=256) + _mk_block(ctr=257, t_peak=1654, + tail=b"\x9c\x06\x00\x42") + ch = decode_histogram_body(body) + assert ch is not None + assert ch["Tran"] == [3, 1654], "terminal block must not be dropped" + + +def test_terminal_block_exempt_from_byte22_constraint(): + """Terminal blocks carry arbitrary bytes at [22:24]; only standard + blocks hold 0x00 there. Requiring it dropped the final interval on + files such as BE18438/T438LO30.DC0H.""" + body = _mk_block(ctr=256) + _mk_block(ctr=257, tail=b"\x9c\x06\x00\x42", + b22=0x01) + ch = decode_histogram_body(body) + assert ch is not None and len(ch["Tran"]) == 2 + + +def test_standard_block_still_requires_byte22_zero(): + """The [22] == 0x00 constraint is what keeps trailer content out, so it + must still apply to standard-tail blocks.""" + ch = decode_histogram_body(_mk_block(b22=0x01)) + assert ch is None + + +def test_marker_is_single_byte_not_uint16(): + """block[4] alone is the 0x0A marker. Treating [4:6] as a uint16 LE + marker forced block[5] to zero, which capped every geo peak at 255 + counts — block[5] is the peak's high byte.""" + r = decode_histogram_body_full(_mk_block(t_peak=0x0676)) + assert r is not None, "block[5] != 0 must not disqualify the block" + assert r[0]["t_peak"] == 0x0676 -- 2.54.0 From 260bf0bc67f1089373e31bfa5f869c4bc1490f99 Mon Sep 17 00:00:00 2001 From: serversdown Date: Tue, 25 Aug 2026 19:11:39 +0000 Subject: [PATCH 24/30] fix(backfill): clear stale shape_* when the .h5 can no longer yield a shape MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit backfill_event_shape.py skipped rows whose .h5 produced no shape and left the previously stored value in place. A stale shape outlives the decode it came from and silently feeds the false-trigger detector. Found while re-running the backfill after the histogram codec fix: 493 rows in the prod snapshot were carrying shape metrics that no longer matched their .h5 — e.g. BE17353/S353LDOK.XZ0H held crest_factor from a 223-sample decode while its .h5 holds a single interval. These predate today's work (present in the pre-32000 snapshot), so this is pre-existing behaviour rather than fallout from the codec fixes. Now NULLs shape_crest_factor / shape_near_peak_count / shape_sample_count / shape_axis in that case and reports a `cleared_stale` count. Verified on the snapshot: 493 cleared, 0 stale rows remaining, 11570 rows matching their .h5 exactly. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HgTe8CamXAHcAmaQ6QNcog --- scripts/backfill_event_shape.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/scripts/backfill_event_shape.py b/scripts/backfill_event_shape.py index 9de2f85..975c716 100644 --- a/scripts/backfill_event_shape.py +++ b/scripts/backfill_event_shape.py @@ -11,7 +11,8 @@ from sfm.shape_metrics import shape_from_h5 log = logging.getLogger("backfill_event_shape") def backfill_shape(db: SeismoDb, store: WaveformStore, *, dry_run: bool = False) -> dict: - counts = {"updated": 0, "skipped_no_h5": 0, "skipped_no_samples": 0} + counts = {"updated": 0, "skipped_no_h5": 0, "skipped_no_samples": 0, + "cleared_stale": 0} for row in db.query_events(limit=1_000_000): serial, filename = row.get("serial"), row.get("blastware_filename") if not serial or not filename: @@ -21,6 +22,20 @@ def backfill_shape(db: SeismoDb, store: WaveformStore, *, dry_run: bool = False) counts["skipped_no_h5"] += 1; continue shape = shape_from_h5(h5_path) if shape is None: + # The .h5 can no longer yield a shape (fewer than 2 samples, or a + # flat trace). Clear any previously stored value rather than + # leaving it behind — a stale shape outlives the decode it came + # from and silently feeds the false-trigger detector. Seen after + # a decoder fix shrinks an event: 493 rows in the prod snapshot + # were carrying metrics from a superseded decode (2026-08-25). + if row.get("shape_crest_factor") is not None: + if not dry_run: + with db._connect() as conn: + conn.execute( + "UPDATE events SET shape_crest_factor=NULL, " + "shape_near_peak_count=NULL, shape_sample_count=NULL, " + "shape_axis=NULL WHERE id=?", (row["id"],)) + counts["cleared_stale"] += 1 counts["skipped_no_samples"] += 1; continue if not dry_run: with db._connect() as conn: -- 2.54.0 From 9bb95003e948027ed2f4ad1d082e8d7153328415 Mon Sep 17 00:00:00 2001 From: serversdown Date: Tue, 25 Aug 2026 22:13:29 +0000 Subject: [PATCH 25/30] fix(codec): the waveform body is a record chain, not a tag stream MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Supersedes the segment-header model entirely, including the fixes made earlier today. Found via multi-agent structural analysis of the 25 files that stalled the walker, then verified independently. Records are self-delimiting: off+2 is a uint16 BE length, next_record = off + 2 + len, and the chain ends on a record whose chan_id is 0x06. off+8 carries a 3-valued mode enum: 02 00 14-byte header, 2 anchors, then CUMULATIVE delta blocks 01 00 10-byte header, no anchors, blocks are ABSOLUTE values 00 03 10-byte header, NO TAGS AT ALL - raw 12-bit packed absolute `40 NN` is an ordinary int16 BE data block (2*NN + 2), never a header. Reading it as a 2*NN + 16 header is what made walks drift — the "variable-prefix segment descriptors" reported earlier today were not a format feature, just walker drift of exactly 4 - (old_stop - true_record_start), on all 25 affected files. Measured on the production snapshot: all four channels equal length 156/1388 -> 1388/1388 ASCII sample-count exact 72/75 -> 75/75 ASCII fully exact 70/75 -> 73/75 device PPV waveform (live) 1288/1306 -> 1306/1306 (mean err 0.00000) device PPV histogram (live) 4434/4459 -> 4458/4459 Also eliminates the walker-over-read class: 24 of those 35 files were histograms that read_blastware_file fed to the waveform codec first; the old walker accepted them and returned garbage (one yielded 98,923 "intervals"), while the record-chain decoder returns None so they fall through to histogram_codec. 00 03 records are DECODED, not skipped. Skipping them silently shifts the time base of everything after them on that channel — BE9558/ K558LOF2.820W had MicL displaced by exactly 512 samples with nothing marking the gap. Footer detection now prefers the 0e 08 candidate whose body yields a chain terminating on 0x06; the signature can occur inside a sample stream. Blast radius 1 file of 1388. The superseded model survives as decode_waveform_legacy, pinned by micromate/idf_file.py: its Thor IDFW body-offset search trial-decodes candidates and keeps whichever yields the most samples, so the new decoder returning None where the old returned garbage changes that heuristic's winner. Deferred until that search uses the record chain. Tests: 253 passed (+11), failure list unchanged from baseline. The 9 tests pinning the superseded model are retargeted at decode_waveform_legacy, which still implements it. NOTE: stored .h5 files need regenerating — nearly all get longer. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HgTe8CamXAHcAmaQ6QNcog --- CHANGELOG.md | 56 +++++++ CLAUDE.md | 23 +++ docs/instantel_protocol_reference.md | 94 ++++++++--- docs/waveform_codec_re_status.md | 19 +++ micromate/idf_file.py | 14 +- minimateplus/event_file_io.py | 27 ++- minimateplus/waveform_codec.py | 238 ++++++++++++++++++++++++++- tests/test_waveform_codec.py | 154 ++++++++++++++++- 8 files changed, 591 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07dc116..94bc028 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,62 @@ All notable changes to seismo-relay are documented here. ### Fixed +- **The series-3 waveform body is a RECORD CHAIN, not a tag stream — this + supersedes the segment-header model, including the fixes made earlier the + same day.** + + Records are self-delimiting. `off+2` is a `uint16 BE` length and + `next_record = off + 2 + len`; the chain ends on a record whose `chan_id` is + `0x06`. `off+8` carries a 3-valued mode enum: + + | mode | header | data section | + |---|---|---| + | `02 00` | 14 B | anchors, then **cumulative deltas** | + | `01 00` | 10 B | no anchors, **absolute** values | + | `00 03` | 10 B | **no tags at all** — raw 12-bit packed absolute | + + **`40 NN` is an ordinary int16 BE data block** (`2*NN + 2`), never a segment + header. Reading it as a `2*NN + 16` header is what made walks drift — and the + "variable-prefix segment descriptors" reported earlier today were not a format + feature at all, just walker drift of exactly + `4 - (old_stop - true_record_start)` on all 25 affected files. + + Measured against the production snapshot: + + | | before | after | + |---|---|---| + | all four channels equal length | 156 / 1388 | **1388 / 1388** | + | ASCII sample-count exact | 72 / 75 | **75 / 75** | + | ASCII fully exact | 70 / 75 | **73 / 75** | + | device PPV, waveform (live decode) | 1288 / 1306 | **1306 / 1306** | + | device PPV, histogram (live decode) | 4434 / 4459 | **4458 / 4459** | + + Mean absolute PPV ratio error on waveforms is now 0.00000. The 2 remaining + ASCII imperfections differ by exactly 1 LSB on samples sitting at the + ±10.000 in/s rail. + + **This also eliminated the walker-over-read class.** 24 of those 35 files + were histograms that `read_blastware_file` fed to the *waveform* codec first; + the old walker accepted them and returned garbage (one yielded 98,923 + "intervals"), while the record-chain decoder correctly returns `None` so they + fall through to `histogram_codec`. + + `00 03` records are decoded rather than skipped. Skipping them does not merely + lose samples — it silently shifts the time base of everything after them on + that channel (observed on `BE9558/K558LOF2.820W`, MicL displaced by exactly + 512 samples with nothing marking the gap). + + Footer detection now prefers whichever `0e 08` candidate yields a chain + terminating on `0x06`, since the signature can occur inside a sample stream. + Blast radius: 1 file of 1,388. + + The superseded model is retained as `decode_waveform_legacy` and pinned by + `micromate/idf_file.py`, whose Thor IDFW body-offset search trial-decodes + candidate offsets and keeps whichever yields the most samples — the new + decoder correctly returns `None` where the old one returned garbage, which + changes that heuristic's winner. Switching Thor over is deferred until that + search is reworked to use the record chain directly. + - **Series-3 histogram block is uniformly big-endian, and the stream's final block has its own tail — the codec was clipping large peaks and dropping the last interval of nearly every histogram.** diff --git a/CLAUDE.md b/CLAUDE.md index 9af798e..0af777e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -223,6 +223,29 @@ custom delta + RLE + variable-width codec. `NN + 2` for int8 blocks). Confirmed 2026-05-11 against SP0 cycle 3 V continuation (`11 90` = NN=400 nibble deltas in 202 bytes). +### ⚠ SUPERSEDED 2026-08-25 — the body is a RECORD CHAIN + +Everything in this section below about `40 NN` segment headers, tagless +headers, variable header widths and channel rotation describes a model that +is **wrong**. The body is a chain of self-delimiting per-channel records: + + off+2 len uint16 BE -> next_record = off + 2 + len + off+4 chan_id 0x46 Tran / 0x47 Vert / 0x48 Long / 0x49 MicL / 0x06 = END + off+8 mode 02 00 = deltas+anchors (14B hdr) + 01 00 = ABSOLUTE values (10B hdr) + 00 03 = raw 12-bit absolute, NO TAGS (10B hdr) + +`40 NN` is an ordinary int16 BE data block (`2*NN + 2`), never a header. The +"variable prefix" of 0/2/4/6/8 bytes was walker drift, exactly +`4 - (old_stop - true_record_start)`. + +All four channels now come out equal length in **1388/1388** files (was +156/1388); ASCII sample-count exact **75/75**, fully exact **73/75**; device +PPV on a live decode **1306/1306** waveform, **4458/4459** histogram. + +The old model survives as `decode_waveform_legacy` because +`micromate/idf_file.py` pins it for Thor IDFW body-offset search. + ### Framing cases added 2026-05-11 → 2026-08-25 Four more block-framing cases, each of which had been causing **silent diff --git a/docs/instantel_protocol_reference.md b/docs/instantel_protocol_reference.md index 9059658..1242d6a 100644 --- a/docs/instantel_protocol_reference.md +++ b/docs/instantel_protocol_reference.md @@ -11,6 +11,7 @@ | Date | Section | Change | |---|---|---| +| 2026-08-25 (3) | S7.6.1, S15 | **THE WAVEFORM BODY IS A RECORD CHAIN, NOT A TAG STREAM — supersedes the segment-header model entirely.** Records are self-delimiting: `off+2` is a uint16 BE length and `next = off + 2 + len`; the chain ends on a record whose chan_id is `0x06`. `off+8` holds a 3-valued mode enum - `02 00` (14-byte header, anchors, cumulative deltas), `01 00` (10-byte, no anchors, ABSOLUTE values), `00 03` (10-byte, no tags at all, raw 12-bit absolute). **`40 NN` is an ordinary int16 BE data block of length 2*NN+2**, never a header; reading it as a 2*NN+16 header is what made walks drift, and the 'variable prefix' of 0/2/4/6/8 bytes reported earlier the same day was walker drift, exactly `4 - (old_stop - true_record_start)`. Verified: chain terminates on `06` in 1387/1388 files; all four channels equal length in **1388/1388** (was 156/1388); ASCII sample-count exact 72/75 -> **75/75**, fully exact 70/75 -> **73/75**; device PPV on a live decode **1306/1306** waveform (mean abs ratio error 0.00000) and **4458/4459** histogram. Also eliminated the walker-over-read class: 24 of those files were histograms the waveform codec was wrongly accepting. The superseded model is retained as `decode_waveform_legacy` because `micromate/idf_file.py` pins it for Thor IDFW body-offset search. | | 2026-08-25 (2) | S7.6.2, S15 | **HISTOGRAM BLOCK IS BIG-ENDIAN + terminal block tail.** The 32-byte histogram block's per-channel fields are uint16 **big-endian** (`T_peak` [5:7], `T_halfperiod` [7:9], `V_peak` [9:11], `V_halfperiod` [11:13], `L_peak` [13:15], `L_halfperiod` [15:17], `M_peak` [17:19], `M_halfperiod` [19:21]); only `block_ctr` [2:4] is little-endian. The marker is `block[4]` alone - the previous uint16 LE marker test at [4:6] forced `block[5] == 0` and thereby capped every geo peak at 255 counts (1.275 in/s), silently clipping larger peaks. The byte previously documented as a per-channel "annotation" is the high byte of the big-endian half-period, which is why it was non-zero exactly on sub-Hz intervals. Separately, the **final block of each stream carries tail `9c 06 00 42`** rather than `1e 0a 00 00` and holds arbitrary bytes at [21:23]; rejecting it dropped the last interval of nearly every histogram. Verified against 1211 production histograms paired with their Blastware ASCII exports: 1211/1211 decode exactly, plus 842,442 per-interval frequency comparisons with zero mismatches (previously 1 of 1196 files fully correct). | | 2026-08-25 | §7.6.1, §15, Appendix E (NEW) | **BODY CODEC + SCALE PASS — five findings, all verified against 75 production events paired with their preserved Blastware ASCII exports.** (1) **Geo full scale is 32000 ADC counts, not 32768** — one decoder unit (16 counts) is exactly 0.005 in/s, so 10.000 in/s = 32000 counts. Consumers dividing by 32768 read every geophone sample and derived peak **2.34% low**; the error scales with amplitude so it was invisible on quiet events and worst on loud ones. 216 per-channel comparisons: 32768 → 151/216 exact (worst 0.238 in/s on a 10 in/s event); 32000 → 216/216 exact, worst 1 LSB. Affects waveforms, histograms and series-4 alike. (2) **Wide-NN RLE `0X NN`** — the 12-bit NN encoding already known for `1X`/`2X` also applies to the `00 NN` RLE tag (runs > 252 samples). (3) **`30 NN` is not capped at NN=0x10** — data-section blocks reach at least 0x18; the length formula was already right. (4) **`40 NN` segment headers are variable width** — NN counts the previous-channel continuation deltas, so the header is `2*NN + 16` bytes; `40 01` and `40 03` occur alongside `40 02`. A header can also appear **tagless** (the NN=0 case): just the 14-byte tail. (5) **The header field documented as a "monotonic uint32 LE counter" is really `[channel_id][00][00][segment_index]`** with 0x46=Tran 0x47=Vert 0x48=Long 0x49=MicL — verified on 1697/1697 segment headers, zero disagreements. Decoders should take the channel from this field, not from rotation position. Items (2)–(5) each caused **silent channel truncation**: an unhandled tag ends the walk and the decoder returns short channels with no error. Corpus result end-to-end: exact 37 → 72, truncated 23 → 3, full-length value errors 15 → 0. New Appendix E documents the field-observed "offset" device fault. | | 2026-05-20 | §2, §3, §4.2, §5.1, §5.3, §6, §7.5b, §7.6.1, §7.6.3, §7.6.4, §7.7.2, §7.7.3, §7.7.5, §7.8.4, §7.8.7, §7.9, §8, §11, §12, §13, §14, §15, Appendix D | **DOC AUDIT PASS — accuracy sweep against `CLAUDE.md` + `minimateplus/` code.** Fixed: (1) S3 frames terminate on bare ETX, not DLE+ETX — §2/§3 rewritten. (2) §3 payload layout corrected — byte[1]=flags, byte[2]=SUB (was wrongly labelled DLE/ADDR). (3) §4.2 — probe responses do NOT carry data length; lengths are hardcoded `DATA_LENGTHS` constants. (4) §5.1 — removed stale duplicate "SUB 1C = TRIGGER CONFIG READ" row; SUB 0A lengths corrected from `0x30/0x26` to `0x46/0x2C` (real event / boundary marker). (5) §5.3 — added missing write-frame format (BW_CMD-only doubling, DLE-aware checksum, offset formula, ack format, SUB 71 chunk parameters). (6) §6 — fixed "SUB 06 → channel config read" → event storage range. (7) §7.5b / §8 — added the 10-byte `sub_code=0x03` continuous-mode timestamp variant alongside the 9-byte single-shot layout; peak vector sum location corrected from "fixed offset 87" to `tran_pos − 12` (label-relative). (8) §7.6 / §7.6.1 / §7.6.3 / §7.6.4 — switched compliance-anchor convention from the 10-byte form to the canonical 6-byte `\xbe\x80\x00\x00\x00\x00`; recording_mode confirmed at anchor−8 in BOTH read and write (was wrongly listed as anchor−3 write / anchor−4 read); sample_rate at anchor−6, histogram_interval at anchor−4, record_time at anchor+6; geo_range row added at channel_label+33. (9) §7.7.2 — token byte position corrected from `params[6]` to `params[7]`. (10) §7.8.4 — fi==9 skip marked FIXED (already removed from code); chunk-count totals updated. (11) §7.8.7 — TODO replaced with current state of `_decode_a5_metadata_into`. (12) §7.9 — Histogram Interval upgraded ❓ → ✅. (13) §11 — POLL example wire bytes corrected; SUB 5A row added to checksum table. (14) §13 — device-under-test updated for current primary unit (BE11529 / S338.17). (15) §14 — TCP Idle Timeout fixed (0→2 min); Data Forwarding Timeout units clarified. (16) §15 (renumbered from second §14) — open-question items already resolved in CLAUDE.md closed out. (17) Appendix D — extension taxonomy rewritten to reflect the AB0T timestamp encoding (D.5.2/D.5.3); EXTENSION REFUTED warning replaced with the resolved encoding. | @@ -1268,36 +1269,79 @@ re-deriving the whole production store: The series-4 figure is closer to correct but not exact — the Thor per-count LSB is its own open question (see §15). -###### Unmapped: variable-prefix segment descriptors ❓ OPEN +###### SUPERSEDED 2026-08-25 — the body is a RECORD CHAIN, not a tag stream -Three of 75 ground-truth production events still truncate. In each, -the walk reaches a segment header whose channel-id field is preceded by -a **variable-width prefix** — 2, 4 or 6 bytes have all been observed, -where the standard tagless form always has 4 (``field2`` + ``len``). -These records also carry the ``01 00`` marker rather than ``02 00``, -and appear packed back-to-back with little or no sample data between -them. +Everything above about ``40 NN`` segment headers, tagless headers, variable +header widths and channel rotation describes a model that is **wrong**. It +produced nearly-correct output because the block table happens to tile the +data sections correctly, but the framing is not what the device writes. -The ``01 00`` marker is *not* simply an anchor count: records carrying -it have been seen with both 2-byte and 4-byte anchor fields in the same -file, so the prefix width and the marker are not yet reconciled. - -The decoder stops cleanly at these rather than emitting garbage. -Examples: ``BE12599/N599LPNB.JF0W`` at body offset 1155 (2-byte -prefix), ``BE12599/N599LPWJ.980W`` at 849 (6-byte prefix), -``BE9558/K558LOF2.820W`` at 1485. - -Examples from event-c (1 sec single-shot): +The body is a chain of **self-delimiting per-channel records**: ``` -Segment header 1 (offset 235): - 40 02 | 00 00 00 00 | 0a 4b 01 1e | 47 00 00 00 | 02 00 00 01 | 00 01 - ^counter=0x47 -Segment header 2 (offset 523): - 40 02 | ff fe ff fe | 13 f5 01 06 | 48 00 00 00 | 02 00 00 01 | 00 02 - ^counter=0x48 (+1) +off+0 field2 uint16 purpose unknown (not a length, not a checksum) +off+2 len uint16 BE next_record = off + 2 + len <- authoritative +off+4 chan_id 0x46 Tran / 0x47 Vert / 0x48 Long / 0x49 MicL + 0x06 = END OF WAVEFORM STREAM +off+5 0x00 +off+6 0x00 +off+7 segment index +off+8 mode 2 bytes, a 3-valued enum +off+10 anchors 2 x int16 BE, ABSOLUTE -- present ONLY when mode == 02 00 ``` +**Mode enum**, all three ground-truth verified: + +| mode | header | data section | +|---|---|---| +| ``02 00`` | 14 bytes | anchors emitted, then blocks are **cumulative deltas** | +| ``01 00`` | 10 bytes | no anchors, blocks carry **absolute** sample values | +| ``00 03`` | 10 bytes | **no tags at all** — raw 12-bit packed absolute samples | + +Census over the 1,388 production series-3 waveform binaries: +``02 00`` x 32,617, ``01 00`` x 74, ``00 03`` x 71. + +**``40 NN`` is an ordinary int16 BE data block** of length ``2*NN + 2`` +(1 <= NN <= 8), never a header. The superseded model read it as a header of +length ``2*NN + 16``, which is exactly why walks drifted: the "variable prefix" +of 0/2/4/6/8 bytes reported earlier was walker drift, precisely +``4 - (old_stop - true_record_start)``, on all 25 affected files. + +Block table for data sections (``NN = ((tag_hi & 0x0F) << 8) | tag_lo``): + +| tag | length | samples | +|---|---|---| +| ``0X NN`` | 2 | NN (RLE hold — holds the previous value in BOTH delta and absolute modes) | +| ``1X NN`` | NN/2 + 2 | NN (4-bit nibble) | +| ``2X NN`` | NN + 2 | NN (int8) | +| ``30 NN`` | NN*1.5 + 2 | NN (12-bit packed) | +| ``40 NN`` | 2*NN + 2 | NN (int16 BE) | + +The ``30 NN`` "trailer length = NN*4" fallback must NOT be applied inside a +record — it corrupts records whose ``30 NN`` sits near a boundary. + +**The preamble is segment 0's implicit Tran record.** ``body[1:3]`` carries +the same mode pair: ``00 02 00`` (1,387 of 1,388 files) means two int16 BE +anchors at ``body[3:7]`` then delta blocks; ``00 00 03`` (1 file, +``BE13121/O121L4L1.KF0W``) means raw 12-bit absolute from ``body[3]``, which +cannot be block-walked — the first record must be located by scanning. + +**Verification.** The length chain terminates on a ``0x06`` record in 1,387 of +1,388 files (the exception has an ambiguous footer signature inside its sample +stream). All four channels come out at identical length in **1,388/1,388**, +against 156/1,388 under the superseded model. Against the 75 events with a +preserved Blastware ASCII export: sample-count exact **72/75 -> 75/75**, fully +exact **70/75 -> 73/75** (the 2 remaining differ by exactly 1 LSB on samples +sitting at the +-10.000 in/s rail). Against device-reported PPVs on a live +decode: waveform **1306/1306** exact with mean absolute ratio error 0.00000; +histogram **4458/4459**. + +This also eliminated the walker-over-read class entirely. 24 of those 35 +"histogram" over-reads were histogram files that ``read_blastware_file`` fed to +the *waveform* codec first; the old walker accepted them and returned garbage +(one produced 98,923 "intervals"), while the record-chain decoder correctly +returns None so they fall through to ``histogram_codec``. + ##### Trailer The trailer (after the last segment's data) is a sequence of 32-byte @@ -3037,7 +3081,7 @@ The `.bin` files produced by `s3_bridge` are **not raw wire bytes**. The logger | **ACH inbound server — RESOLVED.** `bridges/ach_server.py` implements full inbound ACH pipeline. `--clear-after-download` flag for delete-after-upload workflow. Post-erase key-reuse detection via `max_downloaded_key` high-water mark. | RESOLVED | 2026-04-11 | | | **Sensor Check dropdown byte location** — byte offset in 1A compliance config payload for the "Sensor Check: Before monitoring / After each event / Disabled" setting is NOT YET LOCATED. Confirmed: unit always runs with "Before monitoring" set. Need a capture with "Disabled" to diff. | MEDIUM | 2026-04-08 | Still open | | **RV55 DCD/DTR default** — newer Sierra Wireless RV55 firmware does not assert DCD/DTR by default, so the MiniMate Plus never detects TCP disconnect and stays idle instead of resuming monitoring. Root cause: RV55 ACEmanager `DCD Control` setting. Workaround not yet found. | MEDIUM | 2026-04-11 | Still open | -| **Variable-prefix segment descriptors** — 3 of 75 ground-truth events still truncate. The walk reaches a segment header whose channel-id field is preceded by a *variable-width* prefix (2, 4 or 6 bytes observed; the standard tagless form always has 4), carrying an `01 00` marker instead of `02 00`. The marker is **not** simply an anchor count — `01 00` records appear with both 2- and 4-byte anchor fields in the same file, so prefix width and marker are not yet reconciled. Examples: `BE12599/N599LPNB.JF0W` @1155, `BE12599/N599LPWJ.980W` @849, `BE9558/K558LOF2.820W` @1485. | MEDIUM | 2026-08-25 | Still open | +| ~~**Variable-prefix segment descriptors**~~ - **RESOLVED 2026-08-25:** there is no variable prefix. The body is a chain of self-delimiting records (see S7.6.1); the 0/2/4/6/8-byte prefix was walker drift from reading `40 NN` as a segment header. All 25 affected files now chain cleanly to the `06` terminator. | RESOLVED | 2026-08-25 | Resolved 2026-08-25 | | ~~**Histogram codec drops trailing intervals (series-3)**~~ - **RESOLVED 2026-08-25.** Two errors, both in the block model. (1) The block is **uniformly big-endian**: peaks and half-periods are uint16 BE (`T_peak` [5:7], `T_halfperiod` [7:9], `V_peak` [9:11], ...); only `block_ctr` [2:4] is LE. The old uint8-peak + "annotation"-byte model silently clipped any peak above 1.275 in/s, and the "annotation" byte was really the half-period high byte - non-zero exactly on the sub-Hz intervals BW renders `<1.0`. The marker is `block[4]` alone; testing [4:6] as a uint16 LE marker forced `block[5] == 0`, which is what capped the peak at one byte. (2) The **final block of the stream carries tail `9c 06 00 42`** instead of `1e 0a 00 00`, with arbitrary bytes at [21:23]; rejecting it dropped the last interval of nearly every histogram - often the one holding the event peak. Verified on 1211 production histograms vs their BW ASCII exports: **1211/1211 exact** (interval count + every per-interval peak) and 842,442 frequency comparisons with zero mismatches; was 1/1196. | RESOLVED | 2026-08-25 | Resolved 2026-08-25 | | **Micromate (UM-series) IDF decode is ~1000x low** — e.g. `UM11402_20260406130113.IDFW` decodes a Tran peak of 0.0009 in/s against a device-reported 1.1168. Distinct from the Thor IDF path, which decodes sanely. Suspect a different per-count LSB or a body offset that does not hold for UM-series files. | MEDIUM | 2026-08-25 | Still open | | **Thor IDF per-count LSB** — after the 32000 geo full-scale correction, series-4 Thor peaks sit at a median 0.983 of the device-reported peak (was 0.960 under 32768). Closer, but the residual ~1.7% suggests Thor uses its own per-count LSB rather than the BW 16-count/0.005 in/s convention. A code comment in `sfm/waveform_store.py` claims Thor's LSB is 0.0003 in/s, which would predict Thor reading *high* — the measurement shows the opposite, so that comment is unverified. | LOW | 2026-08-25 | Still open | diff --git a/docs/waveform_codec_re_status.md b/docs/waveform_codec_re_status.md index 1a0b864..6d9ff71 100644 --- a/docs/waveform_codec_re_status.md +++ b/docs/waveform_codec_re_status.md @@ -1,3 +1,22 @@ +> ## SUPERSEDED 2026-08-25 — the body is a RECORD CHAIN +> +> The tag-dispatch model described in this document — `40 NN` segment headers, +> tagless headers, channel rotation — is **wrong**. It produced nearly-correct +> output only because the block table happens to tile the data sections. +> +> The body is a chain of self-delimiting per-channel records: `off+2` is a +> uint16 BE length, `next = off + 2 + len`, and the chain ends on a record whose +> chan_id is `0x06`. A 3-valued mode enum at `off+8` selects delta / absolute / +> raw-12-bit semantics. `40 NN` is an ordinary int16 BE data block. +> +> See the record-chain section of `docs/instantel_protocol_reference.md` §7.6.1 +> and the implementation in `minimateplus/waveform_codec.py`. +> +> Result: all four channels equal length in 1388/1388 files (was 156/1388); +> ASCII sample-count exact 75/75, fully exact 73/75; device PPV 1306/1306. +> +> This document is retained as the reasoning trail. + # Waveform body codec — FULLY DECODED (2026-05-11) This is the **clean working note** for the body-codec reverse-engineering diff --git a/micromate/idf_file.py b/micromate/idf_file.py index f3db878..60937b8 100644 --- a/micromate/idf_file.py +++ b/micromate/idf_file.py @@ -47,7 +47,19 @@ from dataclasses import dataclass from pathlib import Path from typing import Optional, Union -from minimateplus.waveform_codec import decode_waveform_v2 +# Thor IDFW bodies are pinned to the SUPERSEDED tag-dispatch decoder. +# +# _find_waveform_body_offset() trial-decodes every candidate offset and keeps +# whichever yields the most samples. The series-3 record-chain decoder +# correctly returns None where the legacy walker returned garbage, which +# changes that heuristic's winner on 33 of 577 files. The net effect measured +# 2026-08-25 was positive (all-channels-equal 8/577 -> 506/577, mean abs PPV +# error 0.228 -> 0.173 in/s) but Thor has no ASCII ground truth in the corpus +# and its geo scaling is separately suspect, so the switch is deferred until +# the body-offset search is reworked to use the record chain directly. +from minimateplus.waveform_codec import ( + decode_waveform_legacy as decode_waveform_v2, +) from .models import IdfEvent, IdfPeaks, IdfReport diff --git a/minimateplus/event_file_io.py b/minimateplus/event_file_io.py index 5d3a01f..15b6641 100644 --- a/minimateplus/event_file_io.py +++ b/minimateplus/event_file_io.py @@ -27,6 +27,7 @@ from typing import Optional, Union from .models import Event, PeakValues, ProjectInfo, Timestamp from . import blastware_file as _bw # avoid circular reference at module load from .bw_ascii_report import BwAsciiReport +from . import waveform_codec as _wc from .waveform_codec import decode_waveform_v2, decoded_to_adc_counts from .histogram_codec import decode_histogram_body @@ -843,7 +844,13 @@ def read_blastware_file(path: Union[str, Path]) -> Event: # Footer: locate the 0e 08 marker, validating the year is in a sane range. body_start = _bw._WAVEFORM_HEADER_SIZE + 21 - footer_pos = -1 + # The 0e 08 + plausible-year footer signature can occur inside the sample + # stream. Collect every candidate and prefer the first whose body yields a + # waveform record chain terminating on the 0x06 marker; fall back to the + # first candidate otherwise. Blast radius measured 2026-08-25: changes the + # chosen footer on exactly 1 of 1,388 series-3 waveform files + # (BE17353/S353L4O5.OX0W, false positive at 3800, real footer at 8576). + footer_candidates = [] pos = body_start while True: pos = raw.find(b"\x0e\x08", pos) @@ -851,10 +858,24 @@ def read_blastware_file(path: Union[str, Path]) -> Event: break yr = (raw[pos + 4] << 8) | raw[pos + 5] if 2015 <= yr <= 2050: - footer_pos = pos - break + footer_candidates.append(pos) pos += 1 + footer_pos = -1 + for cand in footer_candidates: + cand_body = raw[body_start:cand] + try: + recs = _wc.walk_records(cand_body) + except Exception: + recs = [] + if recs: + tail = recs[-1]["end"] + if tail + 5 <= len(cand_body) and cand_body[tail + 4] == _wc.STREAM_END_ID: + footer_pos = cand + break + if footer_pos < 0 and footer_candidates: + footer_pos = footer_candidates[0] + if footer_pos < 0 and len(raw) >= 26: footer_pos = len(raw) - 26 if footer_pos < body_start: diff --git a/minimateplus/waveform_codec.py b/minimateplus/waveform_codec.py index 0c96422..9a74780 100644 --- a/minimateplus/waveform_codec.py +++ b/minimateplus/waveform_codec.py @@ -441,8 +441,16 @@ def decode_tran_initial(body: bytes) -> Optional[List[int]]: return out -def decode_waveform_v2(body: bytes) -> Optional[dict]: +def decode_waveform_legacy(body: bytes) -> Optional[dict]: """ + SUPERSEDED 2026-08-25 — the tag-dispatch / segment-header model. + + Retained because ``micromate/idf_file.py`` trial-decodes Thor IDFW bodies + at many candidate offsets and keeps whichever yields the most samples; + the record-chain decoder returns None where this one returned garbage, + which shifts that heuristic's winner. Thor is pinned here until its own + body-offset search is reworked. Do not use for series-3. + Decode the body into per-channel sample arrays. Status (2026-05-11 evening — channel-rotation hypothesis CONFIRMED): @@ -673,3 +681,231 @@ def decode_a5_frames(a5_frames) -> Optional[dict]: if decoded is None: return None return decoded_to_adc_counts(decoded) + + +# ── Record-chain body model (CONFIRMED 2026-08-25) ────────────────────────── +# +# The body is NOT a flat tag-dispatch stream with ``40 NN`` segment headers. +# It is a chain of self-delimiting per-channel RECORDS: +# +# off+0 field2 uint16 purpose unknown (not a length, not a checksum) +# off+2 len uint16 BE next_record = off + 2 + len <- authoritative +# off+4 chan_id 0x46 Tran / 0x47 Vert / 0x48 Long / 0x49 MicL +# 0x06 = end of waveform stream +# off+5 0x00 +# off+6 0x00 +# off+7 segment index +# off+8 mode 2 bytes, a 3-valued enum (see below) +# off+10 anchors 2 x int16 BE, ABSOLUTE — present only when mode is 02 00 +# +# Mode semantics, all ground-truth verified: +# 02 00 14-byte header; emit the 2 anchors, then blocks are CUMULATIVE deltas +# 01 00 10-byte header; no anchors; blocks carry ABSOLUTE sample values +# 00 03 10-byte header; NO TAGS AT ALL — the data section is raw 12-bit +# packed ABSOLUTE samples (6 bytes -> 4 samples) +# +# ``40 NN`` is an ordinary int16 BE DATA block (length 2*NN + 2), never a header. +# The previous model read it as a variable-width segment header of length +# 2*NN + 16, which is why walks drifted and channels came out unequal. +# +# Verified over the 1,388 series-3 waveform binaries in the production +# snapshot: the length chain terminates on a 0x06 record in 1,387 of them (the +# exception has an ambiguous footer, handled by the caller), and all four +# channels come out at identical length in 1,388/1,388 — against 156/1,388 +# under the superseded model. Against the 75 events with a preserved +# Blastware ASCII export: sample-count exact 72/75 -> 75/75, fully exact +# 70/75 -> 73/75. + +CHANNEL_IDS = {0x46: "Tran", 0x47: "Vert", 0x48: "Long", 0x49: "MicL"} +STREAM_END_ID = 0x06 + +MODE_DELTA = (0x02, 0x00) +MODE_ABSOLUTE = (0x01, 0x00) +MODE_RAW12 = (0x00, 0x03) +_MODES = (MODE_DELTA, MODE_ABSOLUTE, MODE_RAW12) + + +def _u16(b: bytes, p: int) -> int: + return (b[p] << 8) | b[p + 1] + + +def _i16(b: bytes, p: int) -> int: + v = _u16(b, p) + return v - 0x10000 if v >= 0x8000 else v + + +def data_block_len(body: bytes, p: int) -> Tuple[Optional[int], Optional[int]]: + """``(byte_length, n_samples)`` of the data block at *p*, or ``(None, None)``. + + Data-section blocks only — there is no segment-header tag in this model. + ``30 NN`` has no trailer-length fallback here; that fallback corrupted + records whose ``30 NN`` sat near a record boundary. + """ + if p + 2 > len(body): + return None, None + t0, t1 = body[p], body[p + 1] + hi = t0 & 0xF0 + nn = ((t0 & 0x0F) << 8) | t1 + if hi == 0x40: # int16 BE data block + return (None, None) if (nn == 0 or nn > 0x08) else (2 * nn + 2, nn) + if nn == 0 or nn % 4: + return None, None + if hi == 0x00: + return 2, nn # RLE hold + if hi == 0x10: + return nn // 2 + 2, nn # 4-bit nibble + if hi == 0x20: + return nn + 2, nn # int8 + if hi == 0x30: + return nn * 3 // 2 + 2, nn # 12-bit packed + return None, None + + +def unpack12(data: bytes) -> List[int]: + """Raw 12-bit packed samples: 6 bytes -> 4 signed values.""" + out: List[int] = [] + for g in range(len(data) // 6): + hi = (data[6 * g] << 8) | data[6 * g + 1] + for k in range(4): + x = (((hi >> (12 - 4 * k)) & 0xF) << 8) | data[6 * g + 2 + k] + out.append(x - 0x1000 if x >= 0x800 else x) + return out + + +def is_record(body: bytes, p: int) -> bool: + """True if a per-channel record header starts at *p*.""" + return (p + 10 <= len(body) + and body[p + 4] in CHANNEL_IDS + and body[p + 5] == 0x00 and body[p + 6] == 0x00 + and 8 <= _u16(body, p + 2) <= len(body) - p + and (body[p + 8], body[p + 9]) in _MODES) + + +def find_first_record(body: bytes) -> Optional[int]: + """Offset of the first record, or None. + + Under the normal ``00 02 00`` preamble the leading bytes are segment-0's + Tran blocks, so walk them. Under the ``00 00 03`` preamble that data is + raw 12-bit with no tags at all and cannot be block-walked — scan instead. + """ + if len(body) >= 3 and (body[1], body[2]) == MODE_RAW12: + scan_from = 3 + else: + i = 7 + while i < len(body): + if is_record(body, i): + nxt = i + 2 + _u16(body, i + 2) + if nxt + 5 <= len(body) and (is_record(body, nxt) + or body[nxt + 4] == STREAM_END_ID): + return i + length, _ = data_block_len(body, i) + if length is None: + return None + i += length + return None + for i in range(scan_from, max(scan_from, len(body) - 10)): + if is_record(body, i): + nxt = i + 2 + _u16(body, i + 2) + if nxt + 5 <= len(body) and (is_record(body, nxt) + or body[nxt + 4] == STREAM_END_ID): + return i + return None + + +def walk_records(body: bytes, first: Optional[int] = None) -> List[dict]: + """Follow the length chain from *first* to the ``0x06`` terminator.""" + if first is None: + first = find_first_record(body) + out: List[dict] = [] + if first is None: + return out + p, seen = first, set() + while p is not None and p + 10 <= len(body): + if p in seen: + break + seen.add(p) + cid = body[p + 4] + if cid == STREAM_END_ID or cid not in CHANNEL_IDS: + break + length = _u16(body, p + 2) + if length < 8 or p + 2 + length > len(body): + break + out.append({"offset": p, "channel": CHANNEL_IDS[cid], + "segment_index": body[p + 7], + "mode": (body[p + 8], body[p + 9]), + "end": p + 2 + length}) + p += 2 + length + return out + + +def decode_waveform_v2(body: bytes) -> Optional[dict]: + """Decode a Blastware waveform body into per-channel sample arrays. + + Returns ``{"Tran": [...], "Vert": [...], "Long": [...], "MicL": [...]}`` + in 16-count units (LSB = 0.005 in/s at Normal range), or None if *body* + is not a decodable waveform body. + + Implements the record-chain model documented above. + """ + if len(body) < 8 or body[0] != 0x00: + return None + preamble = (body[1], body[2]) + if preamble not in (MODE_DELTA, MODE_RAW12): + return None + first = find_first_record(body) + if first is None: + return None + + out: dict = {c: [] for c in ("Tran", "Vert", "Long", "MicL")} + + def run(channel: str, start: int, end: int, absolute: bool) -> None: + cur = out[channel][-1] if out[channel] else 0 + i = start + while i < end: + length, nn = data_block_len(body, i) + if length is None or i + length > end: + return # stop this record; the chain resyncs at end + hi = body[i] & 0xF0 + if hi == 0x00: + vals = [None] * nn + elif hi == 0x10: + vals = [] + for k in range(nn): + byte = body[i + 2 + k // 2] + v = (byte >> 4) if k % 2 == 0 else (byte & 0xF) + vals.append(v - 16 if v >= 8 else v) + elif hi == 0x20: + vals = [v - 256 if v >= 128 else v + for v in body[i + 2:i + 2 + nn]] + elif hi == 0x30: + vals = unpack12(body[i + 2:i + length]) + else: + vals = [_i16(body, i + 2 + 2 * k) for k in range(nn)] + for v in vals: + if v is None: + pass # RLE hold, in delta AND absolute modes + elif absolute: + cur = v + else: + cur += v + out[channel].append(cur) + i += length + + # Segment 0 is an implicit Tran record carried in the preamble. + if preamble == MODE_DELTA: + out["Tran"].extend([_i16(body, 3), _i16(body, 5)]) + run("Tran", 7, first, absolute=False) + else: + out["Tran"].extend(unpack12(body[3:first])) + + for rec in walk_records(body, first): + ch, off, mode, end = (rec["channel"], rec["offset"], + rec["mode"], rec["end"]) + if mode == MODE_DELTA: + out[ch].extend([_i16(body, off + 10), _i16(body, off + 12)]) + run(ch, off + 14, end, absolute=False) + elif mode == MODE_ABSOLUTE: + run(ch, off + 10, end, absolute=True) + elif mode == MODE_RAW12: + out[ch].extend(unpack12(body[off + 10:end])) + return out diff --git a/tests/test_waveform_codec.py b/tests/test_waveform_codec.py index 2d3290f..eebcf9d 100644 --- a/tests/test_waveform_codec.py +++ b/tests/test_waveform_codec.py @@ -14,6 +14,7 @@ import pytest from minimateplus.waveform_codec import ( WaveformBlock, + decode_waveform_legacy, decode_tran_initial, decode_waveform_v2, decoded_to_adc_counts, @@ -548,9 +549,18 @@ def test_walk_body_wide_rle_block(): assert blocks[0].length == 2 +# NOTE (2026-08-25): the four tests below assert the SUPERSEDED tag-dispatch +# model — `40 NN` as a variable-width segment header, tagless headers, channel +# from rotation. The body format is really a chain of self-delimiting records +# (see the record-chain section of waveform_codec.py), so `decode_waveform_v2` +# no longer behaves this way. They are retargeted at `decode_waveform_legacy`, +# which still implements the old model and is pinned by micromate/idf_file.py +# for Thor IDFW bodies. + + def test_decode_wide_rle_repeats_full_run(): """A wide RLE run repeats the running value NN times, not NN & 0xFF.""" - decoded = decode_waveform_v2(_synth(b"\x01\x0c")) + decoded = decode_waveform_legacy(_synth(b"\x01\x0c")) # 2 preamble anchors + 268 repeats assert len(decoded["Tran"]) == 2 + 268 assert set(decoded["Tran"]) == {0} @@ -589,7 +599,7 @@ def test_segment_header_anchors_track_header_width(nn, hdr_len): data[2 * nn + 8 : 2 * nn + 10] = b"\x02\x00" data[2 * nn + 10 : 2 * nn + 12] = (7).to_bytes(2, "big") # anchor 0 data[2 * nn + 12 : 2 * nn + 14] = (9).to_bytes(2, "big") # anchor 1 - decoded = decode_waveform_v2(_synth(bytes([0x40, nn]) + bytes(data))) + decoded = decode_waveform_legacy(_synth(bytes([0x40, nn]) + bytes(data))) assert decoded["Vert"][:2] == [7, 9] @@ -629,7 +639,7 @@ def test_tagless_header_carries_full_14_bytes_as_data(): def test_tagless_header_anchors_and_channel_id(): """Anchors decode from data[10:14]; the channel comes from the id byte.""" - decoded = decode_waveform_v2(_synth(_tagless(chan_id=0x48, a0=11, a1=13))) + decoded = decode_waveform_legacy(_synth(_tagless(chan_id=0x48, a0=11, a1=13))) assert decoded["Long"][:2] == [11, 13] # 0x48 → Long, not rotation assert decoded["Vert"] == [] @@ -642,10 +652,146 @@ def test_segment_channel_comes_from_id_not_rotation(chan_id, name): would put the second one on the next channel and corrupt both.""" body = _synth(_tagless(chan_id=chan_id, seg=1, a0=5, a1=6), _tagless(chan_id=chan_id, seg=2, a0=7, a1=8)) - decoded = decode_waveform_v2(body) + decoded = decode_waveform_legacy(body) # Tran additionally carries the body preamble's 2 anchors (both 0 here). expected = [0, 0, 5, 6, 7, 8] if name == "Tran" else [5, 6, 7, 8] assert decoded[name] == expected for other in ("Tran", "Vert", "Long", "MicL"): if other != name: assert decoded[other] == ([0, 0] if other == "Tran" else []) + + +# ── Record-chain body model (2026-08-25) ──────────────────────────────────── +# +# The body is a chain of self-delimiting per-channel records, not a flat +# tag-dispatch stream. Verified over 1,388 production series-3 waveform +# binaries: the chain terminates on a 0x06 record in 1,387 of them and all +# four channels come out at identical length in 1,388/1,388 (was 156/1,388). +# Against the 75 events with a preserved Blastware ASCII export: sample-count +# exact 72/75 -> 75/75, fully exact 70/75 -> 73/75. + +from minimateplus.waveform_codec import ( # noqa: E402 + CHANNEL_IDS, + MODE_ABSOLUTE, + MODE_DELTA, + MODE_RAW12, + STREAM_END_ID, + data_block_len, + find_first_record, + is_record, + unpack12, + walk_records, +) + + +def _rec(chan_id, mode, payload, seg=0, field2=b"\x00\x00", anchors=None): + """Build one self-delimiting record.""" + head = bytearray() + head += bytes([chan_id, 0x00, 0x00, seg]) + head += bytes(mode) + if anchors is not None: + for a in anchors: + head += int(a).to_bytes(2, "big", signed=True) + body = bytes(head) + payload + return field2 + (len(body) + 2).to_bytes(2, "big") + body + + +def _terminator(): + return b"\x00\x00" + (8).to_bytes(2, "big") + bytes([STREAM_END_ID, 0, 0, 0, 0, 0]) + + +def _body(*records, preamble=b"\x00\x02\x00", seg0=b"\x00\x00\x00\x00"): + return preamble + seg0 + b"".join(records) + _terminator() + + +def test_forty_nn_is_a_data_block_not_a_segment_header(): + """`40 NN` is an int16 BE data block of length 2*NN + 2. + + The superseded model read it as a segment header of length 2*NN + 16, + which is what made walks drift and channels come out unequal. + """ + assert data_block_len(b"\x40\x02\x00\x01\x00\x02", 0) == (6, 2) + assert data_block_len(b"\x40\x08" + bytes(16), 0) == (18, 8) + # NN > 8 is not a data block + assert data_block_len(b"\x40\x0c" + bytes(24), 0) == (None, None) + + +def test_record_chain_is_followed_by_length_not_by_tag_sniffing(): + payload = b"\x00\x04" # RLE hold x4 + body = _body(_rec(0x47, MODE_DELTA, payload, anchors=(3, 5))) + recs = walk_records(body) + assert len(recs) == 1 + assert recs[0]["channel"] == "Vert" + assert recs[0]["mode"] == MODE_DELTA + + +def test_chain_terminates_on_channel_id_06(): + body = _body(_rec(0x47, MODE_DELTA, b"\x00\x04", anchors=(1, 1)), + _rec(0x48, MODE_DELTA, b"\x00\x04", anchors=(2, 2))) + assert [r["channel"] for r in walk_records(body)] == ["Vert", "Long"] + assert not is_record(body, len(body) - 10) # the terminator is not a record + + +def test_mode_delta_emits_anchors_then_accumulates(): + # two anchors, then an int8 block of +1,+1,+1,+1 + body = _body(_rec(0x47, MODE_DELTA, b"\x20\x04\x01\x01\x01\x01", + anchors=(10, 11))) + d = decode_waveform_v2(body) + assert d["Vert"] == [10, 11, 12, 13, 14, 15] + + +def test_mode_absolute_replaces_rather_than_accumulates(): + """mode `01 00`: no anchors, and block values are ABSOLUTE samples.""" + body = _body(_rec(0x48, MODE_ABSOLUTE, b"\x20\x04\x0a\x0b\x0c\x0d")) + d = decode_waveform_v2(body) + assert d["Long"] == [10, 11, 12, 13], "01 00 blocks are absolute, not deltas" + + +def test_mode_absolute_rle_holds_the_previous_value(): + body = _body(_rec(0x48, MODE_ABSOLUTE, b"\x20\x04\x07\x07\x07\x07\x00\x04")) + d = decode_waveform_v2(body) + assert d["Long"] == [7, 7, 7, 7, 7, 7, 7, 7] + + +def test_mode_raw12_has_no_tags_at_all(): + """mode `00 03`: the whole data section is raw 12-bit absolute samples. + + Decoding these matters for the time base — skipping the record would + displace every later sample on that channel (observed on + BE9558/K558LOF2.820W, MicL shifted by exactly 512). + """ + packed = bytes([0x01, 0x23, 0x04, 0x05, 0x06, 0x07]) # 4 samples + body = _body(_rec(0x49, MODE_RAW12, packed)) + d = decode_waveform_v2(body) + assert d["MicL"] == unpack12(packed) + assert len(d["MicL"]) == 4 + + +def test_unpack12_sign_extends(): + assert unpack12(bytes([0x00, 0x00, 0x01, 0x02, 0x03, 0x04])) == [1, 2, 3, 4] + # high nibble 0x8 -> negative + assert unpack12(bytes([0x80, 0x00, 0x00, 0x00, 0x00, 0x00]))[0] == -2048 + + +def test_channel_comes_from_the_record_id(): + for cid, name in CHANNEL_IDS.items(): + body = _body(_rec(cid, MODE_ABSOLUTE, b"\x20\x04\x01\x02\x03\x04")) + d = decode_waveform_v2(body) + assert d[name][-4:] == [1, 2, 3, 4], f"{name} misrouted" + + +def test_raw12_preamble_is_scanned_not_block_walked(): + """A `00 00 03` preamble carries raw 12-bit data from body[3] with no tags, + so find_first_record must scan rather than block-walk. One production file + has this (BE13121/O121L4L1.KF0W); block-walking returns None on it.""" + packed = bytes([0x00, 0x00, 0x01, 0x02, 0x03, 0x04]) + body = _body(_rec(0x47, MODE_DELTA, b"\x00\x04", anchors=(1, 1)), + preamble=b"\x00\x00\x03", seg0=packed) + assert find_first_record(body) == 3 + len(packed) + d = decode_waveform_v2(body) + assert d["Tran"] == unpack12(packed) + + +def test_returns_none_when_no_record_chain(): + assert decode_waveform_v2(b"\x00\x02\x00" + bytes(40)) is None + assert decode_waveform_v2(b"") is None -- 2.54.0 From 4c58a532de66c0c04a4494d75d166f339d234bcf Mon Sep 17 00:00:00 2001 From: serversdown Date: Tue, 25 Aug 2026 22:23:10 +0000 Subject: [PATCH 26/30] fix(backfill): remove stale .h5 when nothing decodes; log the 415-file histogram variant backfill_sidecars.py skipped the .h5 write when a file produced no samples, with the stated intent of not replacing it with an empty placeholder. That silently preserved output from a superseded decoder. After the record-chain fix, 415 histogram files stopped decoding (216 on BE18193, 199 on BE9440) but kept .h5 files whose peaks ran up to 400x the device-reported PPV. Those were feeding charts and the false-trigger detector with nothing marking them. The .h5 is now removed in that case and the run reports stale_h5_removed. Store-wide effect, series-3, decoded peak vs device-reported PPV: waveform 1307/1307 (100%), mean abs ratio error 0.00000 histogram 4434/4435 (100%) Both were 99% with a tail of 18 and 25 wrong files respectively. The 415 files are a genuine unmapped format variant, not a regression: their bodies open `00 00 00 01 0a 00` (valid block header, marker 0a at [4], block_ctr 256) but block[28:32] matches neither known tail, and no stride from 8 to 64 bytes places a marker at [4] consistently. Bodies are very large (one is 360,573 bytes). They were previously being decoded by the WAVEFORM codec, which accepted them and returned garbage - so the gap pre-dates today's work; the fix only exposed it. Logged as an open question in the protocol reference. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HgTe8CamXAHcAmaQ6QNcog --- CHANGELOG.md | 9 +++++++++ docs/instantel_protocol_reference.md | 1 + scripts/backfill_sidecars.py | 23 +++++++++++++++++++---- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94bc028..cb7aa9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,15 @@ All notable changes to seismo-relay are documented here. ### Fixed +- **`backfill_sidecars.py` now removes a stale `.h5` when nothing decodes.** + It previously skipped the write "so we don't replace whatever's there with an + empty placeholder", which silently preserved output from a superseded + decoder. After the record-chain fix, 415 histogram files stopped decoding (an + unmapped block variant on BE18193 and BE9440) but kept `.h5` files whose peaks + ran up to **400× the device's own reported PPV** — garbage feeding the charts + and the false-trigger detector with nothing marking it. Reports a + `stale_h5_removed` count. + - **The series-3 waveform body is a RECORD CHAIN, not a tag stream — this supersedes the segment-header model, including the fixes made earlier the same day.** diff --git a/docs/instantel_protocol_reference.md b/docs/instantel_protocol_reference.md index 1242d6a..371e800 100644 --- a/docs/instantel_protocol_reference.md +++ b/docs/instantel_protocol_reference.md @@ -3083,6 +3083,7 @@ The `.bin` files produced by `s3_bridge` are **not raw wire bytes**. The logger | **RV55 DCD/DTR default** — newer Sierra Wireless RV55 firmware does not assert DCD/DTR by default, so the MiniMate Plus never detects TCP disconnect and stays idle instead of resuming monitoring. Root cause: RV55 ACEmanager `DCD Control` setting. Workaround not yet found. | MEDIUM | 2026-04-11 | Still open | | ~~**Variable-prefix segment descriptors**~~ - **RESOLVED 2026-08-25:** there is no variable prefix. The body is a chain of self-delimiting records (see S7.6.1); the 0/2/4/6/8-byte prefix was walker drift from reading `40 NN` as a segment header. All 25 affected files now chain cleanly to the `06` terminator. | RESOLVED | 2026-08-25 | Resolved 2026-08-25 | | ~~**Histogram codec drops trailing intervals (series-3)**~~ - **RESOLVED 2026-08-25.** Two errors, both in the block model. (1) The block is **uniformly big-endian**: peaks and half-periods are uint16 BE (`T_peak` [5:7], `T_halfperiod` [7:9], `V_peak` [9:11], ...); only `block_ctr` [2:4] is LE. The old uint8-peak + "annotation"-byte model silently clipped any peak above 1.275 in/s, and the "annotation" byte was really the half-period high byte - non-zero exactly on the sub-Hz intervals BW renders `<1.0`. The marker is `block[4]` alone; testing [4:6] as a uint16 LE marker forced `block[5] == 0`, which is what capped the peak at one byte. (2) The **final block of the stream carries tail `9c 06 00 42`** instead of `1e 0a 00 00`, with arbitrary bytes at [21:23]; rejecting it dropped the last interval of nearly every histogram - often the one holding the event peak. Verified on 1211 production histograms vs their BW ASCII exports: **1211/1211 exact** (interval count + every per-interval peak) and 842,442 frequency comparisons with zero mismatches; was 1/1196. | RESOLVED | 2026-08-25 | Resolved 2026-08-25 | +| **Unmapped histogram block variant (415 files, 2 units)** - after the 2026-08-25 record-chain fix these stop decoding entirely: 216 on BE18193 and 199 on BE9440, all histograms. Their bodies open `00 00 00 01 0a 00`, which is a valid block header (marker `0a` at [4], block_ctr 256 at [2:4] LE), but `block[28:32]` is neither the standard `1e 0a 00 00` nor the terminal `9c 06 00 42`, and no fixed stride between 8 and 64 bytes puts a marker at [4] consistently. Bodies are very large (one is 360,573 bytes). Previously these were being decoded by the WAVEFORM codec, which accepted them and returned garbage peaking up to 400x the device-reported PPV - so this is a pre-existing gap the fix merely exposed, not a regression. Their stale `.h5` files are now removed by `scripts/backfill_sidecars.py` rather than left behind. Example: `BE18193/T193L2X2.1J0H`. | MEDIUM | 2026-08-25 | Still open | | **Micromate (UM-series) IDF decode is ~1000x low** — e.g. `UM11402_20260406130113.IDFW` decodes a Tran peak of 0.0009 in/s against a device-reported 1.1168. Distinct from the Thor IDF path, which decodes sanely. Suspect a different per-count LSB or a body offset that does not hold for UM-series files. | MEDIUM | 2026-08-25 | Still open | | **Thor IDF per-count LSB** — after the 32000 geo full-scale correction, series-4 Thor peaks sit at a median 0.983 of the device-reported peak (was 0.960 under 32768). Closer, but the residual ~1.7% suggests Thor uses its own per-count LSB rather than the BW 16-count/0.005 in/s convention. A code comment in `sfm/waveform_store.py` claims Thor's LSB is 0.0003 in/s, which would predict Thor reading *high* — the measurement shows the opposite, so that comment is unverified. | LOW | 2026-08-25 | Still open | diff --git a/scripts/backfill_sidecars.py b/scripts/backfill_sidecars.py index 04789a5..366b854 100644 --- a/scripts/backfill_sidecars.py +++ b/scripts/backfill_sidecars.py @@ -136,6 +136,7 @@ def main(argv=None) -> int: db = SeismoDb(db_path) written = skipped = errors = 0 + stale_h5_removed = 0 for serial_dir in sorted(p for p in store_root.iterdir() if p.is_dir()): serial = serial_dir.name for path in sorted(serial_dir.iterdir()): @@ -388,9 +389,16 @@ def main(argv=None) -> int: # waveform_codec.decode_waveform_v2 or histogram_codec. # decode_histogram_body. If samples are still empty after # both codecs run, it's a genuine "we can't decode this - # file" case (truncated, malformed, or unknown mode); - # skip the .h5 write so we don't replace whatever's - # there with an empty placeholder. + # file" case (truncated, malformed, or unknown mode). + # + # In that case we REMOVE any existing .h5 rather than leave + # it. Leaving it was the old behaviour and it silently + # preserved output from a superseded decoder: after the + # 2026-08-25 record-chain fix, 415 histogram files stopped + # decoding (an unmapped block variant on BE18193/BE9440) but + # kept .h5 files whose peaks were up to 400x the device's own + # reported PPV — garbage that fed the charts and the + # false-trigger detector with nothing marking it. has_samples = bool( ev.raw_samples and any( ev.raw_samples.get(ch) for ch in ("Tran", "Vert", "Long", "MicL") @@ -398,6 +406,13 @@ def main(argv=None) -> int: ) hdf5_path = store.hdf5_path_for(serial, path.name) hdf5_filename = hdf5_path.name if hdf5_path.exists() else None + if not has_samples and hdf5_path.exists(): + log.warning("%s: no samples decode — removing stale %s", + path.name, hdf5_path.name) + if not args.dry_run: + hdf5_path.unlink() + hdf5_filename = None + stale_h5_removed += 1 hdf5_action = "kept" need_h5 = ( not args.skip_hdf5 @@ -458,7 +473,7 @@ def main(argv=None) -> int: log.error("backfill failed for %s: %s", path, exc, exc_info=args.verbose) errors += 1 - print(f"\nDone. written={written} skipped(uptodate)={skipped} errors={errors}") + print(f"\nDone. written={written} skipped(uptodate)={skipped} errors={errors} stale_h5_removed={stale_h5_removed}") return 0 if errors == 0 else 1 -- 2.54.0 From 306104354b595ad9691bbc1114def6dd9c215c3f Mon Sep 17 00:00:00 2001 From: serversdown Date: Wed, 26 Aug 2026 04:59:00 +0000 Subject: [PATCH 27/30] =?UTF-8?q?feat(histogram):=20decode=20multi-interva?= =?UTF-8?q?l=20blocks=20=E2=80=94=20recovers=20415=20files?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sub-minute histogram intervals are packed several to a block so that every block still covers exactly one minute of data: interval intervals/block stride 1 minute 1 32 <- the standard big-endian block 15 s 4 92 2 s 30 612 stride = 12 + n * 20 Block = [00][segment][ctr uint16 LE][0a][00], then n x 20-byte records of 8 x uint16 LITTLE-endian values (T_peak, T_halfp, V_peak, V_halfp, L_peak, L_halfp, M_peak, M_halfp) plus a 2-word tail whose first word is 0000 on every real interval, then a 6-byte block trailer. The standard 32-byte block is BIG-endian; this variant is LITTLE-endian. The tail-word check matters: a session ending mid-block leaves buffer garbage in the remaining interval slots, which decoded as peaks thousands of times the real value. Stride detection also requires at least 2 records, since a 1-record block would have stride 32 and collide with the standard block. Recovers 415 files that decoded to nothing: 216 on BE18193 (2 s intervals) and 199 on BE9440 (15 s). Before decoding to nothing they were being accepted by the WAVEFORM codec, which returned garbage peaking up to 400x the device-reported PPV. Ground truth BE9440/K440L3AQ.T70H (5,710 intervals) matches its Blastware ASCII export exactly: 17,130/17,130 geo peaks, 22,840/22,840 frequencies, 5,710/5,710 mic dB(L). Across all 455 affected files, 1,354/1,365 channel peaks (99.2%) match the device-reported PPV; the 11 that don't are under-reads on BE9440 where the walk stops early. Fixture (binary + ASCII) saved under tests/fixtures/, which is gitignored per repo practice — the ground-truth test skips when absent. Tests: 258 passed, failure list unchanged from baseline. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HgTe8CamXAHcAmaQ6QNcog --- CHANGELOG.md | 28 +++++ CLAUDE.md | 25 +++++ docs/instantel_protocol_reference.md | 3 +- minimateplus/histogram_codec.py | 119 +++++++++++++++++++- tests/test_histogram_codec.py | 161 +++++++++++++++++++++++++++ 5 files changed, 333 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb7aa9c..179ad17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,34 @@ All notable changes to seismo-relay are documented here. ### Fixed +- **Sub-minute histogram intervals are packed several to a block — 415 files + recovered.** The device always writes one minute of data per block, so a + shorter interval just means more intervals in a longer block: + + | interval | intervals/block | stride | + |---|---|---| + | 1 min | 1 | 32 (the standard block) | + | 15 s | 4 | 92 | + | 2 s | 30 | 612 | + + `stride = 12 + n * 20`. Each 20-byte record carries 8 × uint16 + **little**-endian values — peak and half-period per channel — plus a 2-word + tail whose first word is `0000` on every real interval (a session ending + mid-block leaves buffer garbage in the remaining slots, which decoded as + peaks thousands of times the real value until that check was added). + **The standard 32-byte block is big-endian; this variant is not.** + + These 415 files (216 on BE18193 at 2 s intervals, 199 on BE9440 at 15 s) + previously decoded to nothing at all — and before that were being accepted + by the *waveform* codec, which returned garbage peaking up to 400× the + device-reported PPV. + + Ground truth `BE9440/K440L3AQ.T70H` — 5,710 intervals — matches its + Blastware ASCII export on **17,130/17,130** geo peaks, **22,840/22,840** + frequencies and **5,710/5,710** mic dB(L) values. Across all 455 affected + files, **1,354/1,365 (99.2%)** channel peaks match the device-reported PPV; + the 11 that don't are under-reads on BE9440 where the walk stops early. + - **`backfill_sidecars.py` now removes a stale `.h5` when nothing decodes.** It previously skipped the write "so we don't replace whatever's there with an empty placeholder", which silently preserved output from a superseded diff --git a/CLAUDE.md b/CLAUDE.md index 0af777e..49c4a1d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -284,6 +284,31 @@ rotation and corrupts every channel after it. Corpus result, end to end through the production path: **exact 37 → 72, truncated 23 → 3, full-length value errors 15 → 0.** +### Histogram codec — multi-interval blocks (2026-08-26) + +Sub-minute histogram intervals are packed several to a block, so every +block still covers exactly one minute: + +| interval | intervals/block | stride | +|---|---|---| +| 1 min | 1 | 32 (the standard big-endian block) | +| 15 s | 4 | 92 | +| 2 s | 30 | 612 | + +`stride = 12 + n * 20`. Block = `[00][segment][ctr uint16 LE][0a][00]`, +then n x 20-byte records of 8 x uint16 **LITTLE**-endian values +(`T_peak, T_halfp, V_peak, V_halfp, L_peak, L_halfp, M_peak, M_halfp`) +plus a 2-word tail whose first word is `0000` on every real interval, +then a 6-byte block trailer. + +⚠ The standard 32-byte block is BIG-endian; this variant is LITTLE-endian. + +Recovers **415 files** (216 on BE18193, 199 on BE9440) that decoded to +nothing. Ground truth `BE9440/K440L3AQ.T70H` matches its BW ASCII export +on every one of 17,130 geo peaks, 22,840 frequencies and 5,710 mic dB(L) +values; across all 455 affected files 1,354/1,365 channel peaks (99.2%) +match the device-reported PPV. + ### Histogram codec — corrected 2026-08-25 The histogram block is **uniformly big-endian**, and the stream's final diff --git a/docs/instantel_protocol_reference.md b/docs/instantel_protocol_reference.md index 371e800..4fecbad 100644 --- a/docs/instantel_protocol_reference.md +++ b/docs/instantel_protocol_reference.md @@ -11,6 +11,7 @@ | Date | Section | Change | |---|---|---| +| 2026-08-26 | S7.6.2, S15 | **MULTI-INTERVAL HISTOGRAM BLOCKS - sub-minute intervals pack several per block.** The device always writes one minute of data per block, so a shorter histogram interval means more intervals packed into a longer block: 1 min -> 1 (the standard 32-byte block), 15 s -> 4 (stride 92), 2 s -> 30 (stride 612), with `stride = 12 + n * 20`. Each 20-byte record holds 8 x uint16 **little**-endian values (peak/half-period per channel) plus a 2-word tail whose first word is `0000` on every real interval. The standard block is big-endian - the variant is not. This recovers **415 files** (216 on BE18193 at 2 s, 199 on BE9440 at 15 s) that previously decoded to nothing, and before that were being accepted by the WAVEFORM codec and returning garbage up to 400x the device-reported PPV. | | 2026-08-25 (3) | S7.6.1, S15 | **THE WAVEFORM BODY IS A RECORD CHAIN, NOT A TAG STREAM — supersedes the segment-header model entirely.** Records are self-delimiting: `off+2` is a uint16 BE length and `next = off + 2 + len`; the chain ends on a record whose chan_id is `0x06`. `off+8` holds a 3-valued mode enum - `02 00` (14-byte header, anchors, cumulative deltas), `01 00` (10-byte, no anchors, ABSOLUTE values), `00 03` (10-byte, no tags at all, raw 12-bit absolute). **`40 NN` is an ordinary int16 BE data block of length 2*NN+2**, never a header; reading it as a 2*NN+16 header is what made walks drift, and the 'variable prefix' of 0/2/4/6/8 bytes reported earlier the same day was walker drift, exactly `4 - (old_stop - true_record_start)`. Verified: chain terminates on `06` in 1387/1388 files; all four channels equal length in **1388/1388** (was 156/1388); ASCII sample-count exact 72/75 -> **75/75**, fully exact 70/75 -> **73/75**; device PPV on a live decode **1306/1306** waveform (mean abs ratio error 0.00000) and **4458/4459** histogram. Also eliminated the walker-over-read class: 24 of those files were histograms the waveform codec was wrongly accepting. The superseded model is retained as `decode_waveform_legacy` because `micromate/idf_file.py` pins it for Thor IDFW body-offset search. | | 2026-08-25 (2) | S7.6.2, S15 | **HISTOGRAM BLOCK IS BIG-ENDIAN + terminal block tail.** The 32-byte histogram block's per-channel fields are uint16 **big-endian** (`T_peak` [5:7], `T_halfperiod` [7:9], `V_peak` [9:11], `V_halfperiod` [11:13], `L_peak` [13:15], `L_halfperiod` [15:17], `M_peak` [17:19], `M_halfperiod` [19:21]); only `block_ctr` [2:4] is little-endian. The marker is `block[4]` alone - the previous uint16 LE marker test at [4:6] forced `block[5] == 0` and thereby capped every geo peak at 255 counts (1.275 in/s), silently clipping larger peaks. The byte previously documented as a per-channel "annotation" is the high byte of the big-endian half-period, which is why it was non-zero exactly on sub-Hz intervals. Separately, the **final block of each stream carries tail `9c 06 00 42`** rather than `1e 0a 00 00` and holds arbitrary bytes at [21:23]; rejecting it dropped the last interval of nearly every histogram. Verified against 1211 production histograms paired with their Blastware ASCII exports: 1211/1211 decode exactly, plus 842,442 per-interval frequency comparisons with zero mismatches (previously 1 of 1196 files fully correct). | | 2026-08-25 | §7.6.1, §15, Appendix E (NEW) | **BODY CODEC + SCALE PASS — five findings, all verified against 75 production events paired with their preserved Blastware ASCII exports.** (1) **Geo full scale is 32000 ADC counts, not 32768** — one decoder unit (16 counts) is exactly 0.005 in/s, so 10.000 in/s = 32000 counts. Consumers dividing by 32768 read every geophone sample and derived peak **2.34% low**; the error scales with amplitude so it was invisible on quiet events and worst on loud ones. 216 per-channel comparisons: 32768 → 151/216 exact (worst 0.238 in/s on a 10 in/s event); 32000 → 216/216 exact, worst 1 LSB. Affects waveforms, histograms and series-4 alike. (2) **Wide-NN RLE `0X NN`** — the 12-bit NN encoding already known for `1X`/`2X` also applies to the `00 NN` RLE tag (runs > 252 samples). (3) **`30 NN` is not capped at NN=0x10** — data-section blocks reach at least 0x18; the length formula was already right. (4) **`40 NN` segment headers are variable width** — NN counts the previous-channel continuation deltas, so the header is `2*NN + 16` bytes; `40 01` and `40 03` occur alongside `40 02`. A header can also appear **tagless** (the NN=0 case): just the 14-byte tail. (5) **The header field documented as a "monotonic uint32 LE counter" is really `[channel_id][00][00][segment_index]`** with 0x46=Tran 0x47=Vert 0x48=Long 0x49=MicL — verified on 1697/1697 segment headers, zero disagreements. Decoders should take the channel from this field, not from rotation position. Items (2)–(5) each caused **silent channel truncation**: an unhandled tag ends the walk and the decoder returns short channels with no error. Corpus result end-to-end: exact 37 → 72, truncated 23 → 3, full-length value errors 15 → 0. New Appendix E documents the field-observed "offset" device fault. | @@ -3083,7 +3084,7 @@ The `.bin` files produced by `s3_bridge` are **not raw wire bytes**. The logger | **RV55 DCD/DTR default** — newer Sierra Wireless RV55 firmware does not assert DCD/DTR by default, so the MiniMate Plus never detects TCP disconnect and stays idle instead of resuming monitoring. Root cause: RV55 ACEmanager `DCD Control` setting. Workaround not yet found. | MEDIUM | 2026-04-11 | Still open | | ~~**Variable-prefix segment descriptors**~~ - **RESOLVED 2026-08-25:** there is no variable prefix. The body is a chain of self-delimiting records (see S7.6.1); the 0/2/4/6/8-byte prefix was walker drift from reading `40 NN` as a segment header. All 25 affected files now chain cleanly to the `06` terminator. | RESOLVED | 2026-08-25 | Resolved 2026-08-25 | | ~~**Histogram codec drops trailing intervals (series-3)**~~ - **RESOLVED 2026-08-25.** Two errors, both in the block model. (1) The block is **uniformly big-endian**: peaks and half-periods are uint16 BE (`T_peak` [5:7], `T_halfperiod` [7:9], `V_peak` [9:11], ...); only `block_ctr` [2:4] is LE. The old uint8-peak + "annotation"-byte model silently clipped any peak above 1.275 in/s, and the "annotation" byte was really the half-period high byte - non-zero exactly on the sub-Hz intervals BW renders `<1.0`. The marker is `block[4]` alone; testing [4:6] as a uint16 LE marker forced `block[5] == 0`, which is what capped the peak at one byte. (2) The **final block of the stream carries tail `9c 06 00 42`** instead of `1e 0a 00 00`, with arbitrary bytes at [21:23]; rejecting it dropped the last interval of nearly every histogram - often the one holding the event peak. Verified on 1211 production histograms vs their BW ASCII exports: **1211/1211 exact** (interval count + every per-interval peak) and 842,442 frequency comparisons with zero mismatches; was 1/1196. | RESOLVED | 2026-08-25 | Resolved 2026-08-25 | -| **Unmapped histogram block variant (415 files, 2 units)** - after the 2026-08-25 record-chain fix these stop decoding entirely: 216 on BE18193 and 199 on BE9440, all histograms. Their bodies open `00 00 00 01 0a 00`, which is a valid block header (marker `0a` at [4], block_ctr 256 at [2:4] LE), but `block[28:32]` is neither the standard `1e 0a 00 00` nor the terminal `9c 06 00 42`, and no fixed stride between 8 and 64 bytes puts a marker at [4] consistently. Bodies are very large (one is 360,573 bytes). Previously these were being decoded by the WAVEFORM codec, which accepted them and returned garbage peaking up to 400x the device-reported PPV - so this is a pre-existing gap the fix merely exposed, not a regression. Their stale `.h5` files are now removed by `scripts/backfill_sidecars.py` rather than left behind. Example: `BE18193/T193L2X2.1J0H`. | MEDIUM | 2026-08-25 | Still open | +| ~~**Unmapped histogram block variant (415 files, 2 units)**~~ - **RESOLVED 2026-08-26.** When the histogram interval is SHORTER than one minute the device packs several intervals into a single block so each block still covers exactly one minute: 1 min -> 1 interval (the standard 32-byte block), 15 s -> 4 (stride 92), 2 s -> 30 (stride 612). `stride = 12 + n * 20`. Block is `[00][segment][ctr uint16 LE][0a][00]` then n x 20-byte records of 8 x uint16 **LITTLE**-endian `T_peak,T_halfp,V_peak,V_halfp,L_peak,L_halfp,M_peak,M_halfp` plus a 2-word tail whose first word is `0000` on every real interval (a session ending mid-block leaves buffer garbage in the remaining slots), then a 6-byte block trailer. **Note the endianness flip** - the standard 32-byte block is big-endian. Ground truth `BE9440/K440L3AQ.T70H` (5,710 intervals at 15 s) decodes with 17,130/17,130 geo peaks, 22,840/22,840 frequencies and 5,710/5,710 mic dB(L) matching its Blastware ASCII export exactly; across all 455 affected files 1,354/1,365 channel peaks (99.2%) match the device-reported PPV. | RESOLVED | 2026-08-25 | Resolved 2026-08-26 | | **Micromate (UM-series) IDF decode is ~1000x low** — e.g. `UM11402_20260406130113.IDFW` decodes a Tran peak of 0.0009 in/s against a device-reported 1.1168. Distinct from the Thor IDF path, which decodes sanely. Suspect a different per-count LSB or a body offset that does not hold for UM-series files. | MEDIUM | 2026-08-25 | Still open | | **Thor IDF per-count LSB** — after the 32000 geo full-scale correction, series-4 Thor peaks sit at a median 0.983 of the device-reported peak (was 0.960 under 32768). Closer, but the residual ~1.7% suggests Thor uses its own per-count LSB rather than the BW 16-count/0.005 in/s convention. A code comment in `sfm/waveform_store.py` claims Thor's LSB is 0.0003 in/s, which would predict Thor reading *high* — the measurement shows the opposite, so that comment is unverified. | LOW | 2026-08-25 | Still open | diff --git a/minimateplus/histogram_codec.py b/minimateplus/histogram_codec.py index d60bf07..b6746ec 100644 --- a/minimateplus/histogram_codec.py +++ b/minimateplus/histogram_codec.py @@ -263,7 +263,7 @@ def decode_histogram_body(body: bytes) -> Optional[dict]: to get 1-count ADC values, then ``count / 32767 * 10.0`` for in/s) - Mic channel: use ``waveform_codec.mic_count_to_db(count)`` """ - records = walk_body(body) + records = walk_body(body) or walk_multi_interval_blocks(body) if not records: return None return { @@ -285,7 +285,7 @@ def decode_histogram_body_full(body: bytes) -> Optional[List[dict]]: Returns ``None`` if the body has no valid blocks. """ - records = walk_body(body) + records = walk_body(body) or walk_multi_interval_blocks(body) return records if records else None @@ -305,3 +305,118 @@ def half_period_to_hz(halfp: int) -> Optional[float]: def geo_count_to_ins(count: int) -> float: """Convert a histogram geo peak count to in/s at Normal range.""" return count * _GEO_LSB_INS + + +# ── Multi-interval block variant (CONFIRMED 2026-08-26) ───────────────────── +# +# When the histogram interval is SHORTER than one minute, the device packs +# several intervals into a single block so that every block still covers +# exactly one minute of data: +# +# interval size intervals/block stride +# 1 minute 1 32 <- the standard block above +# 15 seconds 4 92 +# 2 seconds 30 612 +# +# stride = 12 + n_intervals * 20 +# +# Block layout: +# [0] 0x00 +# [1] segment_id (256 blocks per segment, same as the standard block) +# [2:4] block_ctr uint16 LE (0x0100.., resets each segment) +# [4] 0x0a marker +# [5] 0x00 +# [6 ...] n x 20-byte interval records, each carrying 8 x uint16 +# LITTLE-endian values: +# T_peak, T_halfperiod, V_peak, V_halfperiod, +# L_peak, L_halfperiod, M_peak, M_halfperiod +# then 2 more words; the first is 0x0000 on every real interval. +# [-6:] 6-byte block trailer +# +# ⚠ ENDIANNESS: the standard 32-byte block is BIG-endian. This variant is +# LITTLE-endian. Do not share the accessor. +# +# These files previously decoded to nothing at all — 415 of them in the +# production snapshot, 216 on BE18193 (2 s intervals) and 199 on BE9440 +# (15 s). Before that they were being accepted by the WAVEFORM codec, which +# returned garbage peaking up to 400x the device-reported PPV. +# +# Ground truth: BE9440/K440L3AQ.T70H (15 s intervals, 5,710 of them) decodes +# against its Blastware ASCII export with 17,130/17,130 geo peak counts, +# 22,840/22,840 frequencies and 5,710/5,710 mic dB(L) values matching exactly. + +_MULTI_HEADER_LEN = 6 +_MULTI_RECORD_LEN = 20 +_MULTI_TRAILER_LEN = 6 +# At least 2 records: a 1-record block would have stride 12 + 20 = 32, which +# collides with the standard big-endian block and mis-decodes it. +_MULTI_MIN_RECORDS = 2 +_MULTI_MAX_RECORDS = 64 + + +def _is_multi_header(body: bytes, off: int) -> bool: + return (off + _MULTI_HEADER_LEN <= len(body) + and body[off] == 0x00 + and body[off + 4] == 0x0A + and body[off + 5] == 0x00) + + +def detect_multi_interval_stride(body: bytes) -> Optional[int]: + """Block stride of a multi-interval histogram body, or None. + + Found by locating the second block header; validated against + ``stride = 12 + n * 20`` and confirmed on a third block where present. + """ + if not _is_multi_header(body, 0): + return None + lo = _MULTI_HEADER_LEN + _MULTI_TRAILER_LEN + _MULTI_RECORD_LEN * _MULTI_MIN_RECORDS + hi = _MULTI_HEADER_LEN + _MULTI_TRAILER_LEN + _MULTI_RECORD_LEN * _MULTI_MAX_RECORDS + for stride in range(lo, min(hi, len(body)) + 1, 2): + if (stride - 12) % _MULTI_RECORD_LEN: + continue + if not _is_multi_header(body, stride): + continue + # confirm on a third block when the body is long enough + if 2 * stride + _MULTI_HEADER_LEN <= len(body) and not _is_multi_header(body, 2 * stride): + continue + return stride + return None + + +def walk_multi_interval_blocks(body: bytes, + stride: Optional[int] = None) -> List[dict]: + """Decode a multi-interval histogram body into per-interval records.""" + if stride is None: + stride = detect_multi_interval_stride(body) + if not stride: + return [] + n_per_block = (stride - _MULTI_HEADER_LEN - _MULTI_TRAILER_LEN) // _MULTI_RECORD_LEN + if n_per_block < 1: + return [] + + def u16le(p: int) -> int: + return body[p] | (body[p + 1] << 8) + + out: List[dict] = [] + for off in range(0, len(body) - stride + 1, stride): + if not _is_multi_header(body, off): + break # end of the block run; trailer follows + for k in range(n_per_block): + q = off + _MULTI_HEADER_LEN + _MULTI_RECORD_LEN * k + # The first word of each record's 2-word tail is 0x0000 on every + # real interval. A session ending mid-block leaves the remaining + # slots filled with whatever was in the buffer; emitting those + # produced peaks thousands of times the device-reported PPV. + if u16le(q + 16) != 0: + return out + out.append({ + "segment_id": body[off + 1], + "block_ctr": u16le(off + 2), + "t_peak": u16le(q), "t_halfp": u16le(q + 2), + "v_peak": u16le(q + 4), "v_halfp": u16le(q + 6), + "l_peak": u16le(q + 8), "l_halfp": u16le(q + 10), + "m_peak": u16le(q + 12), "m_halfp": u16le(q + 14), + "meta_var": bytes(body[q + 16:q + 20]), + "is_terminal": False, + }) + return out diff --git a/tests/test_histogram_codec.py b/tests/test_histogram_codec.py index e37f60e..9d24817 100644 --- a/tests/test_histogram_codec.py +++ b/tests/test_histogram_codec.py @@ -471,3 +471,164 @@ def test_marker_is_single_byte_not_uint16(): r = decode_histogram_body_full(_mk_block(t_peak=0x0676)) assert r is not None, "block[5] != 0 must not disqualify the block" assert r[0]["t_peak"] == 0x0676 + + +# ── Multi-interval block variant (2026-08-26) ─────────────────────────────── +# +# When the histogram interval is SHORTER than one minute the device packs +# several intervals into one block, so that every block still covers exactly +# one minute of data: +# +# interval intervals/block stride +# 1 min 1 32 (the standard big-endian block) +# 15 s 4 92 +# 2 s 30 612 +# +# stride = 12 + n_intervals * 20 +# +# Block layout: +# [0] 0x00 +# [1] segment_id (256 blocks per segment) +# [2:4] block_ctr uint16 LE +# [4] 0x0a marker +# [5] 0x00 +# [6 ...] n x 20-byte interval records, each 8 x uint16 LITTLE-endian: +# T_peak, T_halfp, V_peak, V_halfp, +# L_peak, L_halfp, M_peak, M_halfp +# followed by 2 words (first is 0x0000) +# [-6:] 6-byte block trailer +# +# NOTE the endianness flip: the standard 32-byte block is big-endian, this +# variant is little-endian. +# +# Ground truth: BE9440/K440L3AQ.T70H (15 s intervals, 5,710 of them) decodes +# with 17,130/17,130 geo peak counts, 22,840/22,840 frequencies and +# 5,710/5,710 mic dB(L) values matching its Blastware ASCII export exactly. + +from minimateplus.histogram_codec import ( # noqa: E402 + detect_multi_interval_stride, + walk_multi_interval_blocks, +) + + +def _mk_multi_block(intervals, seg=0, ctr=256): + """Build one multi-interval block from a list of 8-tuples.""" + b = bytearray() + b += bytes([0x00, seg]) + b += int(ctr).to_bytes(2, "little") + b += bytes([0x0A, 0x00]) + for iv in intervals: + for v in iv: + b += int(v).to_bytes(2, "little") + b += (0).to_bytes(2, "little") + b += (5).to_bytes(2, "little") + b += bytes(6) + assert len(b) == 12 + 20 * len(intervals) + return bytes(b) + + +def test_stride_is_twelve_plus_twenty_per_interval(): + for n in (4, 30): + ivs = [(1, 1, 2, 2, 3, 3, 4, 4)] * n + body = _mk_multi_block(ivs, ctr=256) + _mk_multi_block(ivs, ctr=257) + assert detect_multi_interval_stride(body) == 12 + 20 * n + + +def test_multi_interval_block_decodes_all_four_channels(): + ivs = [(1, 1, 2, 2, 3, 3, 4, 4), (5, 6, 7, 8, 9, 10, 11, 12)] + body = _mk_multi_block(ivs) + _mk_multi_block(ivs, ctr=257) + recs = walk_multi_interval_blocks(body) + assert len(recs) == 4 + r = recs[0] + assert (r["t_peak"], r["v_peak"], r["l_peak"], r["m_peak"]) == (1, 2, 3, 4) + assert (r["t_halfp"], r["v_halfp"], r["l_halfp"], r["m_halfp"]) == (1, 2, 3, 4) + assert recs[1]["t_peak"] == 5 and recs[1]["m_halfp"] == 12 + + +def test_multi_interval_values_are_little_endian(): + """The standard 32-byte block is big-endian; this variant is not. + + A peak of 0x0100 must decode as 256, not 1. + """ + ivs = [(0x0100, 1, 1, 1, 1, 1, 1, 1), (1, 1, 1, 1, 1, 1, 1, 1)] + body = _mk_multi_block(ivs) + _mk_multi_block(ivs, ctr=257) + assert walk_multi_interval_blocks(body)[0]["t_peak"] == 0x0100 + + +def test_decode_histogram_body_falls_back_to_the_variant(): + ivs = [(1, 1, 2, 2, 3, 3, 4, 4)] * 4 + body = _mk_multi_block(ivs) + _mk_multi_block(ivs, ctr=257) + ch = decode_histogram_body(body) + assert ch is not None + assert len(ch["Tran"]) == 8 + assert ch["Long"][0] == 3 + + +def test_standard_blocks_still_take_precedence(): + """A body of standard 32-byte blocks must not be re-read as the variant.""" + std = _mk_block(t_peak=7) + _mk_block(t_peak=9, ctr=257, + tail=b"\x9c\x06\x00\x42") + ch = decode_histogram_body(std) + assert ch is not None and ch["Tran"] == [7, 9] + + +# ── Ground truth for the multi-interval variant ───────────────────────────── +# Fixture is gitignored (like the rest of tests/fixtures); skips when absent. + +_MULTI_FIXTURE = os.path.join( + os.path.dirname(__file__), "fixtures", "histogram-multi-interval", + "K440L3AQ.T70H", +) + + +@pytest.mark.skipif(not os.path.exists(_MULTI_FIXTURE), + reason="multi-interval fixture not present") +def test_multi_interval_matches_blastware_ascii_exactly(): + """BE9440/K440L3AQ.T70H — 5,710 intervals at 15 s, 4 per 92-byte block. + + Every geo peak, every frequency and every mic dB(L) in the file matches + the Blastware ASCII export: 17,130 / 22,840 / 5,710 values, zero + mismatches. Before this decoder the file produced nothing at all. + """ + import math + import re as _re + from minimateplus import blastware_file as _bwf + + raw = open(_MULTI_FIXTURE, "rb").read() + bs = _bwf._WAVEFORM_HEADER_SIZE + 21 + pos, fp = bs, -1 + while True: + pos = raw.find(b"\x0e\x08", pos) + if pos < 0 or pos + 26 > len(raw): + break + if 2015 <= ((raw[pos + 4] << 8) | raw[pos + 5]) <= 2050: + fp = pos + break + pos += 1 + recs = walk_multi_interval_blocks(raw[bs:fp]) + + rows = [] + for line in open(_MULTI_FIXTURE + "_ASCII.TXT", errors="replace"): + p = [x.strip() for x in line.split("\t")] + if len(p) >= 11 and _re.match(r"^\d{2}:\d{2}:\d{2}$", p[0]): + rows.append(p) + + assert len(recs) == len(rows) == 5710 + + def want_count(x): + return round(float(x) / 0.005) + + for rec, row in zip(recs, rows): + assert rec["t_peak"] == want_count(row[1]) + assert rec["v_peak"] == want_count(row[3]) + assert rec["l_peak"] == want_count(row[5]) + # mic dB(L) + assert abs((81.94 + 20 * math.log10(rec["m_peak"])) - float(row[9])) <= 0.06 + # frequency: half-period <= 5 is BW's ">100 Hz" sentinel + for hp, cell in ((rec["t_halfp"], row[2]), (rec["v_halfp"], row[4]), + (rec["l_halfp"], row[6]), (rec["m_halfp"], row[10])): + hz = None if hp <= 5 else 512.0 / hp + if cell.startswith(">"): + assert hz is None + elif not cell.startswith("<"): + assert hz is not None and abs(hz - float(cell)) <= max(0.55, float(cell) * 0.02) -- 2.54.0 From a3b69a62a642f6d87a069bf9fc067c003f7cca57 Mon Sep 17 00:00:00 2001 From: serversdown Date: Wed, 26 Aug 2026 05:55:46 +0000 Subject: [PATCH 28/30] =?UTF-8?q?fix(histogram):=20three=20defects=20found?= =?UTF-8?q?=20by=20a=20full=20series-3=20sweep=20=E2=80=94=2011603/11603?= =?UTF-8?q?=20clean?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Swept every series-3 binary with the live decoder against five independent checks: decode exceptions, zero samples, unequal geo channel lengths, peaks above range full scale, decoded peak vs device-reported PPV, and waveform length vs declared record time. 1. block[22] is NOT a constant and must not be tested. Documented as always 0x00, it carries data on loud blocks, and rejecting those threw away the interval holding the event peak. BE18350/T350L7HR.NL0H block 92 has block[22]=0x26 and a Tran peak of 0x0563 = 1379 counts = 6.895 in/s — exactly the device-reported PPV — while the file decoded to 0.015 in/s. block[0]==0, block[4]==0x0A and the 4-byte tail are six bytes of constraint, which is what keeps trailer content out. 2. Block-model dispatch now goes on signature strength rather than on whichever decoder returns first. A multi-interval body also yields scattered standard-tail blocks by coincidence, so "first non-empty" handed 193 BE18193 files to the standard walker and produced peaks of 149 in/s against a 10 in/s full scale. 3. Multi-interval stride detection requires the block counter to increment by exactly 1. Without it the detector false-positives on ordinary standard-block bodies: they carry a header every 32 bytes, and 192 = 12 + 20*9 and 512 = 12 + 20*25 are both multiples of 32, so a stride "fits" while skipping 6 or 16 real blocks. That misrouted 9,082 files. Partial-block garbage is trimmed within the final block only, stopping at the first slot with a non-zero tail word or a geo peak above full scale (2000 counts in 16-count units). Trimming purely from the end left garbage stranded behind a slot that happened to have a zero tail word; trimming on the tail word alone truncated four BE9440 files by up to 2,800 intervals. Result: 11,603 / 11,603 series-3 binaries clean on every check. Ground truth unchanged: 1211/1211 histograms exact per-interval, 75/75 waveform sample counts exact (73/75 fully exact, the 2 differ by 1 LSB on rail samples), and the multi-interval fixture still matches its BW ASCII export on all 45,680 values. Tests: 259 passed, failure list unchanged from baseline. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HgTe8CamXAHcAmaQ6QNcog --- CHANGELOG.md | 35 ++++++++++++++ minimateplus/histogram_codec.py | 85 +++++++++++++++++++++++++++------ tests/test_histogram_codec.py | 21 ++++++-- 3 files changed, 122 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 179ad17..253e875 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,41 @@ All notable changes to seismo-relay are documented here. ### Fixed +- **Series-3 health sweep: 11,603 / 11,603 binaries now clean on every check.** + Swept every series-3 file with the live decoder against five independent + checks — decode exceptions, zero samples, unequal geo channel lengths, peaks + above range full scale, decoded peak vs the device-reported PPV, and waveform + length vs the declared record time. Three real defects surfaced and were + fixed: + + - **`block[22]` is not a constant and must not be tested.** It was documented + as always `0x00` but carries data on loud blocks, and rejecting those threw + away the interval holding the event peak. + `BE18350/T350L7HR.NL0H` block 92 has `block[22]=0x26` and a Tran peak of + `0x0563` = 1379 counts = **6.895 in/s** — exactly the device-reported PPV — + while the file as a whole decoded to 0.015 in/s. `block[0]==0x00`, + `block[4]==0x0A` and the 4-byte tail are six bytes of constraint, which is + what keeps trailer content out. + + - **Block-model dispatch now goes on signature strength, not on whichever + decoder returns first.** A multi-interval body also yields scattered + standard-tail blocks by coincidence; dispatching on "first non-empty" + handed 193 BE18193 files to the standard walker and produced peaks of + 149 in/s against a 10 in/s full scale. + + - **Multi-interval stride detection requires the block counter to increment + by exactly 1.** Without it the detector false-positives on ordinary + standard-block bodies: those carry a header every 32 bytes, and + `192 = 12 + 20×9` and `512 = 12 + 20×25` are both multiples of 32, so a + stride "fits" while actually skipping 6 or 16 real blocks. That misrouted + 9,082 files. + + Partial-block garbage is now trimmed within the final block only, stopping at + the first slot with a non-zero tail word or a geo peak above full scale. + Trimming purely from the end left garbage stranded behind one slot that + happened to have a zero tail word; trimming on the tail word alone truncated + four BE9440 files by up to 2,800 intervals. + - **Sub-minute histogram intervals are packed several to a block — 415 files recovered.** The device always writes one minute of data per block, so a shorter interval just means more intervals in a longer block: diff --git a/minimateplus/histogram_codec.py b/minimateplus/histogram_codec.py index b6746ec..d60f853 100644 --- a/minimateplus/histogram_codec.py +++ b/minimateplus/histogram_codec.py @@ -170,10 +170,14 @@ def _is_data_block(block: bytes) -> bool: return False if block[4] != _BLOCK_MARKER: return False - tail = block[28:32] - if tail == _BLOCK_TAIL: - return block[22] == 0x00 - return tail == _BLOCK_TAIL_TERMINAL + # The 4-byte tail plus block[0]==0 and block[4]==0x0A is already six bytes + # of constraint — enough to keep trailer content out. There is NO extra + # test on block[22]: it was documented as a constant 0x00 but carries data + # on loud blocks, and rejecting those threw away the interval holding the + # event peak. BE18350/T350L7HR.NL0H is the proof: its block 92 has + # block[22]=0x26 and a Tran peak of 0x0563 = 1379 counts = 6.895 in/s, + # exactly the device-reported PPV, while the file decoded to 0.015 in/s. + return block[28:32] in (_BLOCK_TAIL, _BLOCK_TAIL_TERMINAL) def _decode_block(block: bytes) -> Optional[dict]: @@ -248,6 +252,23 @@ def walk_body(body: bytes) -> List[dict]: return records +def _walk_auto(body: bytes) -> List[dict]: + """Pick the block model by signature strength, not by which returns first. + + The multi-interval variant announces itself with consecutive block headers + at an exact ``12 + 20*n`` stride — far stronger evidence than a handful of + scattered standard-tail blocks, which a multi-interval body will also yield + by coincidence. Dispatching on "whichever decoder returns something" + handed 193 BE18193 files to the standard walker and produced peaks of + 149 in/s against a 10 in/s full scale. + """ + if detect_multi_interval_stride(body): + recs = walk_multi_interval_blocks(body) + if recs: + return recs + return walk_body(body) + + def decode_histogram_body(body: bytes) -> Optional[dict]: """Decode a histogram-mode body into per-channel peak-sample arrays. @@ -263,7 +284,7 @@ def decode_histogram_body(body: bytes) -> Optional[dict]: to get 1-count ADC values, then ``count / 32767 * 10.0`` for in/s) - Mic channel: use ``waveform_codec.mic_count_to_db(count)`` """ - records = walk_body(body) or walk_multi_interval_blocks(body) + records = _walk_auto(body) if not records: return None return { @@ -285,7 +306,7 @@ def decode_histogram_body_full(body: bytes) -> Optional[List[dict]]: Returns ``None`` if the body has no valid blocks. """ - records = walk_body(body) or walk_multi_interval_blocks(body) + records = _walk_auto(body) return records if records else None @@ -353,6 +374,10 @@ _MULTI_TRAILER_LEN = 6 _MULTI_MIN_RECORDS = 2 _MULTI_MAX_RECORDS = 64 +# Geo full scale in 16-count units: 10.000 in/s / 0.005 = 2000. A peak above +# this is physically impossible and marks buffer garbage in a partial block. +_GEO_MAX_COUNTS = 2000 + def _is_multi_header(body: bytes, off: int) -> bool: return (off + _MULTI_HEADER_LEN <= len(body) @@ -376,9 +401,24 @@ def detect_multi_interval_stride(body: bytes) -> Optional[int]: continue if not _is_multi_header(body, stride): continue - # confirm on a third block when the body is long enough - if 2 * stride + _MULTI_HEADER_LEN <= len(body) and not _is_multi_header(body, 2 * stride): + # DECISIVE CHECK: consecutive blocks differ by exactly 1 in block_ctr. + # Without it this false-positives on ordinary standard-block bodies: + # those carry a header every 32 bytes, and 192 = 12 + 20*9 and + # 512 = 12 + 20*25 are both multiples of 32, so a stride "fits" while + # actually skipping 6 or 16 real blocks. Sampling a standard body at + # stride 192 handed 9,082 files to the wrong decoder and produced peaks + # of 149 in/s against a 10 in/s full scale. + def _ctr(o: int) -> int: + return body[o + 2] | (body[o + 3] << 8) + + if (_ctr(stride) - _ctr(0)) & 0xFFFF != 1: continue + # confirm on a third block when the body is long enough + if 2 * stride + _MULTI_HEADER_LEN <= len(body): + if not _is_multi_header(body, 2 * stride): + continue + if (_ctr(2 * stride) - _ctr(stride)) & 0xFFFF != 1: + continue return stride return None @@ -403,13 +443,8 @@ def walk_multi_interval_blocks(body: bytes, break # end of the block run; trailer follows for k in range(n_per_block): q = off + _MULTI_HEADER_LEN + _MULTI_RECORD_LEN * k - # The first word of each record's 2-word tail is 0x0000 on every - # real interval. A session ending mid-block leaves the remaining - # slots filled with whatever was in the buffer; emitting those - # produced peaks thousands of times the device-reported PPV. - if u16le(q + 16) != 0: - return out out.append({ + "_tail0": u16le(q + 16), "segment_id": body[off + 1], "block_ctr": u16le(off + 2), "t_peak": u16le(q), "t_halfp": u16le(q + 2), @@ -419,4 +454,26 @@ def walk_multi_interval_blocks(body: bytes, "meta_var": bytes(body[q + 16:q + 20]), "is_terminal": False, }) + # A session ending mid-block leaves the remaining slots of the FINAL block + # filled with whatever was in the buffer. Those decoded as peaks thousands + # of times the device-reported PPV, so they have to go — but only from the + # final block: a non-zero tail word occurs mid-file on real intervals, and + # trimming on that alone truncated four BE9440 files by up to 2,800 + # intervals, while trimming purely from the end left garbage stranded + # behind one slot that happened to have a zero tail word. + # + # Within the final block, stop at the first slot that is not plausibly + # real: a non-zero tail word, or a geo peak above full scale. 16-count + # units put Normal-range full scale (10.000 in/s) at 2000 counts, so + # anything beyond that is physically impossible. + if out: + last_block_start = ((len(out) - 1) // n_per_block) * n_per_block + for i in range(last_block_start, len(out)): + r = out[i] + if (r["_tail0"] != 0 + or max(r["t_peak"], r["v_peak"], r["l_peak"]) > _GEO_MAX_COUNTS): + del out[i:] + break + for r in out: + r.pop("_tail0", None) return out diff --git a/tests/test_histogram_codec.py b/tests/test_histogram_codec.py index 9d24817..558a0c9 100644 --- a/tests/test_histogram_codec.py +++ b/tests/test_histogram_codec.py @@ -457,11 +457,22 @@ def test_terminal_block_exempt_from_byte22_constraint(): assert ch is not None and len(ch["Tran"]) == 2 -def test_standard_block_still_requires_byte22_zero(): - """The [22] == 0x00 constraint is what keeps trailer content out, so it - must still apply to standard-tail blocks.""" - ch = decode_histogram_body(_mk_block(b22=0x01)) - assert ch is None +def test_standard_block_accepts_nonzero_byte22(): + """block[22] is NOT a constant and must not be tested. + + It was documented as always 0x00, but it carries data on loud blocks. + Rejecting those threw away the interval holding the event peak: + BE18350/T350L7HR.NL0H block 92 has block[22]=0x26 and a Tran peak of + 0x0563 = 1379 counts = 6.895 in/s — exactly the device-reported PPV — + while the file as a whole decoded to 0.015 in/s. + + block[0]==0x00, block[4]==0x0A and the 4-byte tail are six bytes of + constraint, which is what keeps trailer content out. + """ + ch = decode_histogram_body(_mk_block(t_peak=1379, b22=0x26)) + assert ch is not None + assert ch["Tran"] == [1379] + assert geo_count_to_ins(ch["Tran"][0]) == pytest.approx(6.895) def test_marker_is_single_byte_not_uint16(): -- 2.54.0 From b2ef02ebccc394e292663709718c6b27699f99d4 Mon Sep 17 00:00:00 2001 From: serversdown Date: Thu, 27 Aug 2026 16:35:47 +0000 Subject: [PATCH 29/30] =?UTF-8?q?chore(release):=20v0.26.0=20=E2=80=94=20s?= =?UTF-8?q?eries-3=20decode=20correctness?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two body-model rewrites, a systematic scale error affecting every geophone reading the system ever produced, a recovered file format, and two artifact-hygiene bugs where stale files outlived the decodes that made them. - geo full scale is 32000 ADC counts, not 32768 (every reading 2.34% low) - the waveform body is a record chain, not a tag stream - the histogram block is big-endian, with a terminal tail - sub-minute intervals pack several per block (415 files recovered) - three more defects found by a full-corpus sweep, each masking the next - stale .h5 files and stale shape_* columns are now cleared, not left All 11,603 series-3 binaries in the production snapshot pass every check. Ground truth: 1,211/1,211 histograms exact per-interval, 75/75 waveform sample counts exact, multi-interval fixture exact on all 45,680 values. Also corrects a changelog note that went stale within the same day: the "3 of 75 events still truncate" item was resolved by the record-chain rewrite, and the remaining open items are now listed explicitly. CLAUDE.md gains a "Where things stand" block at the top — the header had been reading v0.21.0, four releases behind, which is the first thing you see when picking the project back up. Tests: 259 passed; the 16 failures are pre-existing (gitignored fixtures) and unchanged from baseline. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HgTe8CamXAHcAmaQ6QNcog --- CHANGELOG.md | 31 ++++++++++++++++++++++++++----- CLAUDE.md | 29 ++++++++++++++++++++++++++++- pyproject.toml | 2 +- sfm/server.py | 2 +- 4 files changed, 56 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 253e875..5284dc6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,16 @@ All notable changes to seismo-relay are documented here. ## [Unreleased] +--- + +## v0.26.0 — 2026-08-27 + +**Series-3 decode correctness.** Two body-model rewrites, a systematic +scale error affecting every geophone reading ever produced, a recovered +file format, and two artifact-hygiene bugs where stale files outlived the +decodes that made them. All 11,603 series-3 binaries in the production +snapshot now pass every check. + ### Fixed - **Series-3 health sweep: 11,603 / 11,603 binaries now clean on every check.** @@ -261,11 +271,22 @@ gitignored fixtures). the ACH queue with garbage events. Store-wide it affects 2 units of 21 across 6 episodes; see `scratch/offset_candidates.csv` and the project memory notes. -- **Still open:** 3 of 75 ground-truth events truncate at a segment-header - variant with a variable-width prefix before the channel-id field (2, 4 or 6 - bytes observed) and an `01 00` marker instead of `02 00`. See the protocol - reference, "Unmapped: variable-prefix segment descriptors". Examples: - `BE12599/N599LPNB.JF0W` at body offset 1155, `BE9558/K558LOF2.820W` at 1485. +- ~~**Still open:** 3 of 75 ground-truth events truncate at a segment-header + variant with a variable-width prefix.~~ **Resolved later the same day** — the + record-chain rewrite (above) showed there is no variable prefix; it was + walker drift. All 75 are now sample-count exact. + +- **Still open after this release:** + - **Series-4 (Thor / Micromate) is not verified** — UM-series sits at ~48% + against device peaks with a ~1.7% systematic bias and a near-zero tail. + Thor IDFW is pinned to `decode_waveform_legacy` deliberately. + - **14 sensitive-range files** show a decoded/truth ratio of exactly 8.0 + (= 10.0/1.25) — a units bug, not a codec one. Never chased. + - **`backfill_sidecars.py --force` also inserts DB rows** for store files + that have none (1,286 on the snapshot; one-time per store), and the + dry-run does not report that count before you commit to it. + - **Verification is uneven:** per-sample proof on the 11% of files with a + preserved `_ASCII.TXT`, peak-and-structure consistency on the other 89%. --- diff --git a/CLAUDE.md b/CLAUDE.md index 49c4a1d..f2c7e75 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,7 +2,34 @@ Ground-up Python replacement for **Blastware**, Instantel's Windows-only software for managing MiniMate Plus seismographs. Connects over direct RS-232 or cellular modem -(Sierra Wireless RV50 / RV55). Current version: **v0.21.0**. +(Sierra Wireless RV50 / RV55). Current version: **v0.26.0**. + +--- + +## Where things stand (updated 2026-08-27) + +Read this first when picking the project back up. + +- **Series-3 decode is correct and verified.** All 11,603 series-3 binaries in + the prod snapshot pass every check (channel lengths, peaks vs the device's + own reported PPV, nothing above full scale, length vs declared record time). + Ground truth: 1,211/1,211 histograms exact per-interval and 75/75 waveform + sample counts exact against preserved Blastware ASCII exports. + ⚠ That is per-sample proof on 11% of files and peak-only consistency on the + other 89% — see `docs/instantel_protocol_reference.md` §7.6.1. +- **Series-4 (Thor / Micromate) is NOT verified.** UM-series sits at ~48% + against device peaks with a ~1.7% systematic bias and a near-zero tail. + Thor IDFW is pinned to `decode_waveform_legacy` deliberately. +- **Open, not blocking:** 14 sensitive-range files show an exact 8x + (= 10.0/1.25) units discrepancy; `scripts/backfill_sidecars.py --force` also + inserts DB rows for store files that have none (one-time per store) and the + dry-run does not report that count. +- **After any codec change, regenerate the store** — `backfill_sidecars.py + --force` then `backfill_event_shape.py`, DB backup first. Stored `.h5` files + do not update themselves. +- **Parked:** the "offset" hardware-fault investigation (Appendix E of the + protocol reference) pending the multi-year BW archive. + When new information about the protocol is discovered, please update the instantel_protocol_reference.md with the findings in addition to this document diff --git a/pyproject.toml b/pyproject.toml index ed0cc90..24a4e1b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "seismo-relay" -version = "0.25.0" +version = "0.26.0" description = "Python client and REST server for MiniMate Plus seismographs" requires-python = ">=3.10" dependencies = [ diff --git a/sfm/server.py b/sfm/server.py index eabc45a..0004702 100644 --- a/sfm/server.py +++ b/sfm/server.py @@ -90,7 +90,7 @@ app = FastAPI( "Implements the minimateplus RS-232 protocol library.\n" "Proxied by terra-view at /api/sfm/*." ), - version="0.25.0", + version="0.26.0", ) # Allow requests from the waveform viewer opened as a local file (file://) -- 2.54.0 From dedf1f02c9ba9ac656f30721123276f7218e0b58 Mon Sep 17 00:00:00 2001 From: serversdown Date: Thu, 27 Aug 2026 17:22:16 +0000 Subject: [PATCH 30/30] =?UTF-8?q?fix(release):=20bump=20TOOL=5FVERSION=20t?= =?UTF-8?q?o=200.26.0=20=E2=80=94=20sidecar=20staleness=20was=20inert?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TOOL_VERSION had been frozen at 0.21.1 for four releases despite its own comment saying "Bump this constant and CHANGELOG.md together at release time". It is not cosmetic: backfill_sidecars.py decides whether to regenerate with ver_ok = sidecar.source.tool_version >= event_file_io.TOOL_VERSION so with the constant stuck at 0.21.1 and every sidecar stamped 0.21.1, a backfill WITHOUT --force skipped the entire store. That is precisely the failure the check exists to prevent, and it means every sidecar regenerated during the 0.26.0 decode work is stamped 0.21.1 while having been produced by 0.26.0 code. Verified: a non-force dry-run over the snapshot now reports written=11603 skipped(uptodate)=0, where before it would have skipped all 11,603. Prod therefore does not need --force to pick up the decode corrections — the version difference alone is enough. Note the installed dist metadata reads 0.12.0, older than the constant, so the best-effort "prefer installed metadata when newer" path correctly defers to TOOL_VERSION. Also bumps the README header, which still read v0.22.0. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HgTe8CamXAHcAmaQ6QNcog --- README.md | 2 +- minimateplus/event_file_io.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 791f0b3..02f7644 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# seismo-relay `v0.22.0` +# seismo-relay `v0.26.0` A ground-up replacement for **Blastware** — Instantel's aging Windows-only software for managing seismographs. Supports both the **MiniMate Plus diff --git a/minimateplus/event_file_io.py b/minimateplus/event_file_io.py index 15b6641..b0393d8 100644 --- a/minimateplus/event_file_io.py +++ b/minimateplus/event_file_io.py @@ -50,7 +50,7 @@ SIDECAR_KIND = "sfm.event" # bumped without a `pip install` re-run — leading to confusing stale # version stamps in sidecars. Bump this constant and CHANGELOG.md # together at release time. -TOOL_VERSION = "0.21.1" +TOOL_VERSION = "0.26.0" try: # Best-effort: prefer the installed metadata when it's NEWER than the -- 2.54.0